tools - a quick reference for common linux tools

This is a handy list of tools I use, mostly to investigate performance issues, on a Linux machine. Much credit due to Brendan Gregg.

Errors & Logs

# kernel events
dmesg | tail
# systemd kernel logs
journalctl -k

# systemd, show logs for specific unit
journalctl -fu <unit>

# logs since boot
journalctl -b

# log time filtering, multiplexing
journalctl --since "2015-01-10 17:15:00"  --until "2015-01-11 03:00"
journalctl --since yesterday
journalctl -u nginx.service -u php-fpm.service --since today

# no wrapping
journalctl --no-full

# last 10 entries
journalctl -n 10
Great journalctl notes

Processes (finding, killing)

# make pgrep useful by showing the command line and args
pgrep -lf <additional filters>

# process with a file descriptor for the given filesystem path
lsof /var/foo

Resource Consumption

CPU

# 1, 5, 15 minute damped average runqueues
uptime
# 1Hz samples of runnable processes and their CPU usage
pidstat 1
# 1Hz samples of system-wide CPU, memory, I/O, context switches, runqueue
vmstat 1
# 1Hz view of multi-core load balance
mpstat -P -ALL 1
# sort by CPU with 'P'
htop
# 1Hz samples of runqueue length
sar -q

Memory

# 1Hz samples of system-wide CPU, memory, I/O, context switches, runqueue
vmstat 1
# used, free, buffers, cache, swap usage (in MB)
free -m
# sort by memory with 'M'
htop

Swap

# swap location, usage
swapon -s
# 1Hz samples of swap and I/O (si, so) in kB/sec
vmstat 1
# used, free, buffers, cache, swap usage (in MB)
free -m

Filesystems

# disk usage for filesystems
df -h
# disk usage within directory
du --max-depth 1

I/O

# 1Hz samples of swap and I/O (si, so) in kB/sec
vmstat 1
# cache, buffers, swap
free -m
# 1Hz per-task kB read, kB write, i/o delay
pidstat -d 1
# 1Hz per-device i/o scheduler behavior, kB/sec read and write, saturation
iostat -xz 1
# per-process B/s read, write, swap
iotop

Network

# 1Hz per-interface packets/sec, kB/sec, saturation
sar -n DEV 1
# 1Hz TCP segments and connections /sec, and errored (attempted)
sar -n TCP,ETCP 1
# interface rx, tx, queue length statistics
netstat -ie

Sockets/Connections

# listening sockets (omitted by default) for TCP
ss -lt

# TCP timer information
ss -o

# memory associated with sockets
ss -m

# PID associated with sockets
ss -p

# established connections for SSH
ss -o state established '( dport = :ssh or sport = :ssh )'
# lsof requires `-a` to logically AND filters
# internet sockets for a specific PID, port 80
lsof -a -i :80 -p <pid>
# listening TCP sockets, with PID, don't resolve
netstat -ltpn

Network Configuration

IP layer, routing

# interface IPs
ip a

# hosts without iproute2
ifconfig

# show routing tables
ip route show dev eth0 table main
ip route show dev eth0 table local
netstat -r
linux-ip.net

iptables

# list all rules for a given table (default is the 'filter' table)
iptables -vL -t <(filter|nat|mangle|raw|security)>

# drop inbound traffic from a remote host:port
iptables -t filter -A INPUT -p tcp --sport ${remoteport} --source ${remotehost} -j DROP

# remove the above rule
iptables -t filter -D INPUT -p tcp --sport ${remoteport} --source ${remotehost} -j DROP

tcpdump

# capture 5 //full// packets and print them in ASCII. Filter for
# TCP packets where tcp-push header flag is set
tcpdump -v -c 5 -s 0 -A dst 10.188.3.14 and dst port 31905 and "(tcp[tcpflags] & tcp-push != 0)"

Kernel Configuration

# list in-use modules
lsmod
ls /proc/modules
# kernel parameters, like TCP syn retries (example)
sysctl net.ipv4.tcp_syn_retries
# live kernel config location
/proc/config.gz

# also often stored as a file copy at /boot/config*

Hardware

Block devices

# list block devices and their mount points
lsblk

USB

# summary of USB devices
lsusb

# detailed characteristics of devices
lsusb -v

udev

# listen to kernel uevents and print the udev event after rule processing
udevadm monitor --environment --udev

# pick up new udev rule changes
udevadm control --reload-rules

# query udev database for device information (in this case, a graphics device)
udevadm info --name=/dev/dri/card0

PCI

# list PCI devices
lspci

NIC

# 1Hz per-interface packets/sec, kB/sec, saturation
sar -n DEV 1
# ethernet (layer 2) modes, including negotiated line / duplex speed
ethtool <interface>
# interface rx, tx, queue length statistics
netstat -ie

Generating Load

# create a file of 6*4096 byte blocks in size
dd if=/dev/zero of=file.txt count=6 bs=4096
# peg a core to ~100%
yes | /dev/null

Golang

# run a local godoc server
godoc -http=":6060"
# clone a repo into $GOPATH but don't install its binaries
read repo && go get -d -u ${repo}

Bash

# read a file and operate on each line
while read line; do echo ${line}; done <"$filename"

vim

# jump back
ctrl+O

# jump forward
ctrl+I
# open/close quickfix
:copen
:ccl

# next quickfix
:cn
# open/close location list
:lop
:lcl
# tell gitgutter to show comparison to a different base SHA/commit
let g:gitgutter_diff_base='<commit SHA>'

git

# show a pretty one-liner history of SHAs between commits
git log --graph --abbrev-commit --pretty=format:'%h %s <%ae>'

Compression & Encoding

# zstd compress and decompress
echo 'foo' | zstd -c | zstd -d
# gzip compress and decompress
echo 'foo' | gzip -f | gzip -d
# zlib compress and decompress
echo 'foo' | openssl zlib | openssl zlib -d
# xz compress and decompress
echo 'foo' | xz | xz -d
# UTF-8 to hex and back
echo 'foo' | xxd -p | xxd -p -r