tools - a quick reference for common linux tools
Thu, Jul 13, 2017This is a handy list of tools I use, mostly to investigate performance issues, on a Linux machine. Much credit due to Brendan Gregg.
Errors & Logs
# kernel events
dmesg | tail
# systemd kernel logs
journalctl -k
# systemd, show logs for specific unit
journalctl -fu <unit>
# logs since boot
journalctl -b
# log time filtering, multiplexing
journalctl --since "2015-01-10 17:15:00" --until "2015-01-11 03:00"
journalctl --since yesterday
journalctl -u nginx.service -u php-fpm.service --since today
# no wrapping
journalctl --no-full
# last 10 entries
journalctl -n 10
Processes (finding, killing)
# make pgrep useful by showing the command line and args
pgrep -lf <additional filters>
# process with a file descriptor for the given filesystem path
lsof /var/foo
Resource Consumption
CPU
# 1, 5, 15 minute damped average runqueues
uptime
# 1Hz samples of runnable processes and their CPU usage
pidstat 1
# 1Hz samples of system-wide CPU, memory, I/O, context switches, runqueue
vmstat 1
# 1Hz view of multi-core load balance
mpstat -P -ALL 1
# sort by CPU with 'P'
htop
# 1Hz samples of runqueue length
sar -q
Memory
# 1Hz samples of system-wide CPU, memory, I/O, context switches, runqueue
vmstat 1
# used, free, buffers, cache, swap usage (in MB)
free -m
# sort by memory with 'M'
htop
Swap
# swap location, usage
swapon -s
# 1Hz samples of swap and I/O (si, so) in kB/sec
vmstat 1
# used, free, buffers, cache, swap usage (in MB)
free -m
Filesystems
# disk usage for filesystems
df -h
# disk usage within directory
du --max-depth 1
I/O
# 1Hz samples of swap and I/O (si, so) in kB/sec
vmstat 1
# cache, buffers, swap
free -m
# 1Hz per-task kB read, kB write, i/o delay
pidstat -d 1
# 1Hz per-device i/o scheduler behavior, kB/sec read and write, saturation
iostat -xz 1
# per-process B/s read, write, swap
iotop
Network
# 1Hz per-interface packets/sec, kB/sec, saturation
sar -n DEV 1
# 1Hz TCP segments and connections /sec, and errored (attempted)
sar -n TCP,ETCP 1
# interface rx, tx, queue length statistics
netstat -ie
Sockets/Connections
# listening sockets (omitted by default) for TCP
ss -lt
# TCP timer information
ss -o
# memory associated with sockets
ss -m
# PID associated with sockets
ss -p
# established connections for SSH
ss -o state established '( dport = :ssh or sport = :ssh )'
# lsof requires `-a` to logically AND filters
# internet sockets for a specific PID, port 80
lsof -a -i :80 -p <pid>
# listening TCP sockets, with PID, don't resolve
netstat -ltpn
Network Configuration
IP layer, routing
# interface IPs
ip a
# hosts without iproute2
ifconfig
# show routing tables
ip route show dev eth0 table main
ip route show dev eth0 table local
netstat -r
iptables
# list all rules for a given table (default is the 'filter' table)
iptables -vL -t <(filter|nat|mangle|raw|security)>
# drop inbound traffic from a remote host:port
iptables -t filter -A INPUT -p tcp --sport ${remoteport} --source ${remotehost} -j DROP
# remove the above rule
iptables -t filter -D INPUT -p tcp --sport ${remoteport} --source ${remotehost} -j DROP
tcpdump
# capture 5 //full// packets and print them in ASCII. Filter for
# TCP packets where tcp-push header flag is set
tcpdump -v -c 5 -s 0 -A dst 10.188.3.14 and dst port 31905 and "(tcp[tcpflags] & tcp-push != 0)"
Kernel Configuration
# list in-use modules
lsmod
ls /proc/modules
# kernel parameters, like TCP syn retries (example)
sysctl net.ipv4.tcp_syn_retries
# live kernel config location
/proc/config.gz
# also often stored as a file copy at /boot/config*
Hardware
Block devices
# list block devices and their mount points
lsblk
USB
# summary of USB devices
lsusb
# detailed characteristics of devices
lsusb -v
udev
# listen to kernel uevents and print the udev event after rule processing
udevadm monitor --environment --udev
# pick up new udev rule changes
udevadm control --reload-rules
# query udev database for device information (in this case, a graphics device)
udevadm info --name=/dev/dri/card0
PCI
# list PCI devices
lspci
NIC
# 1Hz per-interface packets/sec, kB/sec, saturation
sar -n DEV 1
# ethernet (layer 2) modes, including negotiated line / duplex speed
ethtool <interface>
# interface rx, tx, queue length statistics
netstat -ie
Generating Load
# create a file of 6*4096 byte blocks in size
dd if=/dev/zero of=file.txt count=6 bs=4096
# peg a core to ~100%
yes | /dev/null
Golang
# run a local godoc server
godoc -http=":6060"
# clone a repo into $GOPATH but don't install its binaries
read repo && go get -d -u ${repo}
Bash
# read a file and operate on each line
while read line; do echo ${line}; done <"$filename"
vim
# jump back
ctrl+O
# jump forward
ctrl+I
# open/close quickfix
:copen
:ccl
# next quickfix
:cn
# open/close location list
:lop
:lcl
# tell gitgutter to show comparison to a different base SHA/commit
let g:gitgutter_diff_base='<commit SHA>'
git
# show a pretty one-liner history of SHAs between commits
git log --graph --abbrev-commit --pretty=format:'%h %s <%ae>'
Compression & Encoding
# zstd compress and decompress
echo 'foo' | zstd -c | zstd -d
# gzip compress and decompress
echo 'foo' | gzip -f | gzip -d
# zlib compress and decompress
echo 'foo' | openssl zlib | openssl zlib -d
# xz compress and decompress
echo 'foo' | xz | xz -d
# UTF-8 to hex and back
echo 'foo' | xxd -p | xxd -p -r