gdb -tui ./binary # Start with TUI enabled
gdb ./binary # Start normally, then:
(gdb) tui enable # Enable TUI
(gdb) layout src # Source view
(gdb) layout asm # Assembly view
(gdb) layout regs # Register view
(gdb) layout split # Source + assembly split
(gdb) tui reg general # General purpose registers
(gdb) tui reg float # Float registers
(gdb) focus cmd # Focus on command window
(gdb) focus src # Focus on source window
Ctrl-L # Refresh TUI display
Ctrl-X A # Toggle TUI on/off
Ctrl-X 2 # Cycle through layouts
(gdb) break main # Break at function
(gdb) break file.c:42 # Break at line
(gdb) break *0x401234 # Break at address
(gdb) break foo if x > 10 # Conditional breakpoint
(gdb) break foo if strcmp(s, "bad")==0 # String condition
(gdb) rbreak ^my_prefix_ # Regex breakpoint (all matching)
(gdb) tbreak main # Temporary breakpoint (once)
(gdb) hbreak foo # Hardware breakpoint
(gdb) info breakpoints # List all breakpoints
(gdb) disable 2 # Disable breakpoint 2
(gdb) enable 2 # Re-enable
(gdb) delete 2 # Delete breakpoint 2
(gdb) ignore 2 5 # Skip breakpoint 2 five times
(gdb) condition 2 x == 42 # Add/change condition on bp 2
(gdb) commands 2 # Commands to run on bp 2
> print x
> continue
> end
(gdb) watch var # Break on var write
(gdb) rwatch var # Break on var read
(gdb) awatch var # Break on var read or write
(gdb) watch *0xdeadbeef # Watch memory address
(gdb) watch -l var # Watch location (alias)
(gdb) info watchpoints # List watchpoints
(gdb) target record-full # Start recording execution
(gdb) run # Run (now recorded)
(gdb) reverse-continue # Run backwards to prev breakpoint
(gdb) reverse-step # Step backwards one source line
(gdb) reverse-next # Step backwards over calls
(gdb) reverse-finish # Run backwards out of function
(gdb) reverse-stepi # Step backwards one instruction
(gdb) set exec-direction reverse # Make continue/step go backward
(gdb) set exec-direction forward # Restore forward execution
gdb -p 12345 # Attach by PID
gdb ./binary 12345 # Attach with binary for symbols
sudo gdb -p $(pgrep myproc) # Attach (root for other users)
(gdb) detach # Detach without killing
(gdb) quit # Detach and quit
(gdb) info threads # List all threads
(gdb) thread 3 # Switch to thread 3
(gdb) thread apply all bt # Backtrace all threads
(gdb) thread apply all bt full # Full backtrace all threads
(gdb) thread apply 1 2 3 print x # Command on specific threads
(gdb) set scheduler-locking on # Only run current thread
(gdb) set scheduler-locking off # All threads run (default)
(gdb) set scheduler-locking step # Lock during step only
(gdb) break foo thread 2 # Break only in thread 2
(gdb) checkpoint # Save current execution state
(gdb) info checkpoints # List checkpoints
(gdb) restart 1 # Restore to checkpoint 1
(gdb) delete checkpoint 1 # Remove checkpoint
(gdb) python print(gdb.selected_frame().name())
(gdb) python gdb.execute("bt")
(gdb) python gdb.parse_and_eval("x + y")
(gdb) source script.py # Load a Python GDB script
# script.py - example GDB Python script
import gdb
class PrintOnBreak(gdb.Breakpoint):
def stop(self):
val = gdb.parse_and_eval("my_var")
print(f"my_var = {val}")
return False # don't actually stop
PrintOnBreak("main")
gdb.execute("run")
# STL pretty printers (usually installed with gdb)
(gdb) info pretty-printer # List active printers
(gdb) disable pretty-printer global libstdc++ # Disable one
(gdb) print myVector # Shows contents if printer active
# Load custom pretty printer
(gdb) source /path/to/printers.py
# Custom pretty printer example
import gdb.printing
class MyStructPrinter:
def __init__(self, val):
self.val = val
def to_string(self):
return f"MyStruct(x={self.val['x']}, y={self.val['y']})"
def build_printer(obj):
pp = gdb.printing.RegexpCollectionPrettyPrinter("my_lib")
pp.add_printer("MyStruct", "^MyStruct$", MyStructPrinter)
return pp
gdb.printing.register_pretty_printer(gdb.current_objfile(), build_printer)
(gdb) set pagination off # No "Press Enter" prompts
(gdb) set print pretty on # Indent structs
(gdb) set print array on # Pretty-print arrays
(gdb) set print array-indexes on # Show array indices
(gdb) set print elements 0 # Print unlimited array elements
(gdb) x/20xb 0xdeadbeef # Examine 20 hex bytes at address
(gdb) x/s 0xdeadbeef # Examine as string
(gdb) x/10i $pc # Disassemble 10 instructions from PC
(gdb) disassemble /m foo # Disassemble with source interleaved
(gdb) info registers # All register values
(gdb) print $rax # Specific register
(gdb) call foo(1, 2) # Call function from gdb
(gdb) set var x = 42 # Change a variable
(gdb) generate-core-file # Dump core from live process
(gdb) save breakpoints bps.txt # Save breakpoints to file
(gdb) source bps.txt # Restore breakpoints
set history save on
set history size 10000
set print pretty on
set print array on
set print array-indexes on
set print elements 0
set pagination off
set disassembly-flavor intel
strace ./binary # Trace all syscalls
strace -p 12345 # Attach to running process
strace -f ./binary # Follow forks (child processes too)
strace -ff -o out ./binary # Separate output file per process
strace -o trace.log ./binary # Write to file
strace -e trace=file ./binary # File-related syscalls
strace -e trace=network ./binary # Network syscalls
strace -e trace=memory ./binary # mmap, brk, mprotect, etc.
strace -e trace=process ./binary # fork, exec, wait, etc.
strace -e trace=signal ./binary # Signal-related
strace -e trace=ipc ./binary # IPC (pipes, sockets, etc.)
strace -e trace=desc ./binary # File descriptor operations
strace -e openat,read,write ./binary # Specific syscalls only
strace -e trace=!futex ./binary # Exclude futex calls
strace -c ./binary # Summary: count, time, errors per syscall
strace -C ./binary # Summary + live trace
strace -T ./binary # Show time spent in each syscall
strace -t ./binary # Prefix timestamps (HH:MM:SS)
strace -tt ./binary # Microsecond timestamps
strace -ttt ./binary # Unix epoch + microseconds
strace -r ./binary # Relative timestamps between syscalls
strace -yy ./binary # Decode fd paths for all FDs
strace -y ./binary # Decode fd path for file syscalls only
# Output shows: openat(AT_FDCWD, "/etc/passwd", ...) = 3</dev/null>
strace ./binary 2>&1 | grep ENOENT # Missing files
strace ./binary 2>&1 | grep EACCES # Permission denied
strace ./binary 2>&1 | grep ECONNREFS # Connection refused
strace ./binary 2>&1 | grep -E "= -1" # All failed syscalls
strace -e openat -z ./binary # Only failed calls (-z shows errors)
strace -e openat -Z ./binary # Only successful calls
# What files does this program open?
strace -e trace=openat -o /dev/stderr ./binary 2>&1 | grep -v ENOENT
# Why is this program slow?
strace -c -T ./binary
# What network connections?
strace -e trace=network -yy ./binary 2>&1 | grep connect
# Trace already-running process, follow children
strace -fp $(pgrep myproc) -e trace=file -yy
ltrace ./binary # Trace library calls
ltrace -p 12345 # Attach to process
ltrace -f ./binary # Follow forks
ltrace -e malloc ./binary # Only trace malloc calls
ltrace -e malloc+free ./binary # Trace malloc and free
ltrace -e @libssl.so ./binary # All calls into libssl
ltrace -l /lib/libc.so.6 ./binary # Calls into specific library
ltrace -c ./binary # Summary count per call
ltrace -C ./binary # Demangle C++ symbol names
ltrace -n 2 ./binary # Indent nested calls 2 spaces
ltrace -o trace.log ./binary # Write to file
ltrace -T ./binary # Show time in each call
# Trace specific calls with arguments
ltrace -e malloc -e free -e realloc ./binary
# C++ demangling
ltrace -C ./binary 2>&1 | grep operator
perf record ./binary # Sample CPU at 1000 Hz
perf record -g ./binary # With call graph (frame pointers)
perf record --call-graph dwarf ./binary # DWARF-based call graph (no -fomit-frame-pointer needed)
perf record -F 99 ./binary # Sample at 99 Hz
perf record -p 12345 # Profile running process
perf record -a sleep 10 # System-wide for 10 seconds
perf report # Interactive report from perf.data
perf report --stdio # Non-interactive text output
perf report --no-children # Hide aggregated children costs
perf top # Live CPU usage by symbol
perf top -p 12345 # Live profile of process
perf top -g # With call graph
perf top --stdio # Non-interactive
perf stat ./binary # Hardware counters summary
perf stat -r 5 ./binary # Repeat 5 times, show variance
perf stat -a sleep 1 # System-wide for 1 second
perf stat -e cache-misses,cache-references ./binary # Cache metrics
perf stat -e branch-misses,branch-instructions ./binary # Branch prediction
perf stat -e context-switches,cpu-migrations ./binary # Scheduler events
perf stat -e cycles,instructions,ipc ./binary # IPC
perf stat -e L1-dcache-loads,L1-dcache-load-misses ./binary # L1 detail
perf stat -e LLC-loads,LLC-load-misses ./binary # LLC detail
perf record -g ./binary && perf report # Find hot function, then:
perf annotate my_hot_function # Interleaved source+asm with hit counts
perf annotate --stdio # Non-interactive
perf script # Dump all samples as text
perf script | head -100 # Preview
perf list # All available events
perf record -e 'syscalls:sys_enter_*' ./binary # Trace all syscall enters
perf record -e 'sched:sched_switch' -a sleep 5 # Scheduler switches
# Install Flamegraph (one-time)
git clone https://github.com/brendangregg/FlameGraph /opt/FlameGraph
# Record
perf record -F 99 -g ./binary
# or: perf record -F 99 --call-graph dwarf ./binary
# Generate
perf script | /opt/FlameGraph/stackcollapse-perf.pl | \
/opt/FlameGraph/flamegraph.pl > cpu.svg
# View
firefox cpu.svg
# Capture baseline
perf record -F 99 -g ./binary_before
perf script | /opt/FlameGraph/stackcollapse-perf.pl > before.folded
# Capture after
perf record -F 99 -g ./binary_after
perf script | /opt/FlameGraph/stackcollapse-perf.pl > after.folded
# Generate differential (red = regression, blue = improvement)
/opt/FlameGraph/difffolded.pl before.folded after.folded | \
/opt/FlameGraph/flamegraph.pl > diff.svg
# Off-CPU time (blocked in kernel, sleeping, I/O wait)
perf record -e sched:sched_switch -a -g sleep 10
perf script | /opt/FlameGraph/stackcollapse-perf.pl | \
/opt/FlameGraph/flamegraph.pl --title "Off-CPU" --colors io > offcpu.svg
cargo install flamegraph
cargo flamegraph # Profile default binary
cargo flamegraph --bin mybinary # Profile specific binary
cargo flamegraph -- arg1 arg2 # Pass arguments
# Output: flamegraph.svg in current directory
bpftrace -e 'profile:hz:99 /pid == 12345/ { @[ustack] = count(); }' \
> stacks.txt
# Then: flamegraph.pl stacks.txt > out.svg (if using bpftrace folded output)
valgrind ./binary # Default: memcheck
valgrind --leak-check=full ./binary # Full leak report
valgrind --leak-check=full --show-leak-kinds=all ./binary # All leak types
valgrind --track-origins=yes ./binary # Track origin of uninit values
valgrind --error-exitcode=1 ./binary # Non-zero exit on errors
valgrind --gen-suppressions=all ./binary # Generate suppression file
valgrind --suppressions=supp.txt ./binary # Use suppression file
valgrind --log-file=vg.log ./binary # Write report to file
valgrind --num-callers=30 ./binary # Deeper call stacks
valgrind -v ./binary # Verbose output
valgrind --tool=cachegrind ./binary # Simulate L1/LL cache
ls cachegrind.out.* # Find output file
cg_annotate cachegrind.out.12345 # Annotate by function
cg_annotate --auto=yes cachegrind.out.12345 # Auto-find source
cg_diff cachegrind.out.before cachegrind.out.after # Compare runs
valgrind --tool=callgrind ./binary # Profile with call graph
valgrind --tool=callgrind --callgrind-out-file=cg.out ./binary
callgrind_annotate cg.out # Text report
kcachegrind cg.out # GUI viewer (install kcachegrind)
callgrind_control -i on # Toggle instrumentation live
callgrind_control -d # Dump results now
valgrind --tool=callgrind --instr-atstart=no ./binary # Start disabled
valgrind --tool=massif ./binary # Heap profiling
valgrind --tool=massif --pages-as-heap=yes ./binary # Include mmap
ms_print massif.out.12345 # Text report (ASCII chart)
massif-visualizer massif.out.12345 # GUI viewer
valgrind --tool=massif --time-unit=B ./binary # Time by bytes allocated
valgrind --tool=helgrind ./binary # Data races and lock errors
valgrind --tool=helgrind --history-level=full ./binary # Full history
valgrind --tool=drd ./binary # Faster than helgrind for races
valgrind --tool=drd --check-stack-var=yes ./binary # Also check stack
valgrind --tool=drd --segment-merging=no ./binary # More precise (slower)
# Compilation
gcc -fsanitize=address -fno-omit-frame-pointer -g -O1 -o binary source.c
clang -fsanitize=address -fno-omit-frame-pointer -g -O1 -o binary source.c
# Options at runtime
ASAN_OPTIONS=halt_on_error=0 ./binary # Don't stop on first error
ASAN_OPTIONS=abort_on_error=1 ./binary # Abort (get core dump)
ASAN_OPTIONS=log_path=/tmp/asan.log ./binary # Write to file
ASAN_OPTIONS=detect_leaks=1 ./binary # Also detect leaks (default on Linux)
ASAN_OPTIONS=detect_stack_use_after_return=1 ./binary # Slower, catches more
ASAN_OPTIONS=strict_string_checks=1 ./binary # Extra string checking
ASAN_OPTIONS=check_initialization_order=1 ./binary # Init order fiasco
ASAN_OPTIONS=symbolize=1 ASAN_SYMBOLIZER_PATH=$(which llvm-symbolizer) ./binary
# Suppress specific errors
ASAN_OPTIONS=suppressions=asan.supp ./binary
# Compilation
gcc -fsanitize=undefined -g -O1 -o binary source.c
gcc -fsanitize=undefined,integer -g -O1 -o binary source.c # Include int overflow
# All UBSan checks
-fsanitize=undefined,integer,float-divide-by-zero,float-cast-overflow,\
pointer-overflow,bounds,alignment,null,nonnull-attribute
# Options
UBSAN_OPTIONS=halt_on_error=1 ./binary # Stop on first error (default is continue)
UBSAN_OPTIONS=print_stacktrace=1 ./binary # Print full stack trace
UBSAN_OPTIONS=log_path=/tmp/ubsan.log ./binary
# Compilation
gcc -fsanitize=thread -g -O1 -o binary source.c
# CRITICAL: Never combine TSan with ASan or MSan
# TSan can be combined with UBSan only
# Options
TSAN_OPTIONS=halt_on_error=1 ./binary # Stop on first race
TSAN_OPTIONS=log_path=/tmp/tsan.log ./binary
TSAN_OPTIONS=history_size=7 ./binary # More history (default 2, max 7)
TSAN_OPTIONS=second_deadlock_stack=1 ./binary # Extra deadlock info
TSAN_OPTIONS=suppressions=tsan.supp ./binary
# Compilation (requires all deps also built with MSan, usually use with clang)
clang -fsanitize=memory -fno-omit-frame-pointer -g -O1 -o binary source.c
clang -fsanitize=memory -fsanitize-memory-track-origins=2 -g source.c # Track origins
# Options
MSAN_OPTIONS=halt_on_error=1 ./binary
MSAN_OPTIONS=log_path=/tmp/msan.log ./binary
# CRITICAL: Never combine MSan with ASan or TSan
ASan + UBSan = OK
TSan + UBSan = OK
ASan + TSan = NEVER (incompatible)
ASan + MSan = NEVER (incompatible)
MSan + TSan = NEVER (incompatible)
ulimit -c unlimited # Allow unlimited core dump size (current shell)
ulimit -c unlimited && ./binary # One-liner
# Make permanent for session
echo "ulimit -c unlimited" >> ~/.bashrc
# systemd-coredump (usually default on modern systems)
cat /proc/sys/kernel/core_pattern # Shows current core pattern
# If using systemd-coredump: |/usr/lib/systemd/systemd-coredump %P %u %g %s %t %c %h
# Set core file name pattern
sudo bash -c 'echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern'
# %e = executable name, %p = pid, %t = timestamp
coredumpctl list # List all recorded core dumps
coredumpctl info # Info about most recent
coredumpctl info -1 # Most recent
coredumpctl info 12345 # By PID
coredumpctl info myprogram # By executable name
coredumpctl debug # Open most recent in GDB
coredumpctl debug 12345 # Open specific in GDB
coredumpctl dump -o core.file # Extract core to file
coredumpctl dump myprogram -o core.file
gdb ./binary core # Load binary + core
gdb ./binary core.12345 # Named core file
(gdb) bt # Backtrace at crash point
(gdb) thread apply all bt # All threads at crash
(gdb) info registers # Register state at crash
(gdb) frame 3 # Jump to specific frame
(gdb) up / down # Navigate frames
(gdb) print myvar # Inspect variables
(gdb) x/20x $sp # Inspect stack memory
# systemd stores cores compressed
ls /var/lib/systemd/coredump/
# Journal has crash metadata
journalctl -t systemd-coredump
journalctl -b -1 | grep -i coredump # Crashes from last boot
PID=12345
# Identity
cat /proc/$PID/cmdline | tr '\0' ' ' # Command line with args
cat /proc/$PID/exe # Symlink to executable (or readlink)
readlink /proc/$PID/exe
cat /proc/$PID/environ | tr '\0' '\n' # Environment variables
readlink /proc/$PID/cwd # Current working directory
# Status
cat /proc/$PID/status # Summary: state, memory, threads, uid
cat /proc/$PID/stat # Raw stats (see proc(5) for field meaning)
cat /proc/$PID/statm # Memory pages
cat /proc/$PID/limits # Resource limits (ulimit values)
cat /proc/$PID/syscall # Current/last syscall being made
# Memory
cat /proc/$PID/maps # Virtual memory map (addr, perms, file)
cat /proc/$PID/smaps # Detailed per-mapping stats
cat /proc/$PID/smaps_rollup # Summary of smaps totals
cat /proc/$PID/mem # Read process memory (use with maps)
# File Descriptors
ls -la /proc/$PID/fd/ # All open file descriptors
readlink /proc/$PID/fd/3 # What fd 3 points to
ls -la /proc/$PID/fdinfo/ # Per-fd flags, position
# Networking
cat /proc/$PID/net/tcp # TCP sockets (hex addresses)
cat /proc/$PID/net/udp # UDP sockets
# I/O
cat /proc/$PID/io # Bytes read/written to storage
# Threads
ls /proc/$PID/task/ # One dir per thread (TID)
cat /proc/$PID/task/*/status # Status of all threads
# Namespace
ls -la /proc/$PID/ns/ # Namespace memberships
# Stack (kernel)
cat /proc/$PID/stack # Kernel stack trace (requires root)
cat /proc/$PID/wchan # What kernel function waiting in
# If a process has a deleted file open:
ls -la /proc/$PID/fd/ | grep deleted # Find deleted fd
cp /proc/$PID/fd/3 /tmp/recovered # Copy it back
# Recover deleted shared library
find /proc/*/fd -ls 2>/dev/null | grep "deleted" | grep "\.so"
# By process
lsof -p 12345 # All files for PID
lsof -p 12345,67890 # Multiple PIDs
lsof -c myproc # By process name
# By file/directory
lsof /var/log/syslog # Who has this file open
lsof +D /var/log/ # Who has any file in this dir open
lsof /dev/sda1 # Who is using this device
# By network
lsof -i # All network connections
lsof -i :8080 # By port
lsof -i tcp:8080 # TCP on port 8080
lsof -i udp:53 # UDP on port 53
lsof -i @192.168.1.1 # To/from specific host
lsof -i 4 # IPv4 only
lsof -i 6 # IPv6 only
lsof -i tcp -s tcp:LISTEN # Listening TCP sockets only
lsof -i tcp -s tcp:ESTABLISHED # Established TCP only
# Unix sockets
lsof -U # All unix sockets
lsof /tmp/myapp.sock # Who has this socket
# Deleted files still held open (space leak)
lsof | grep deleted
lsof | grep "(deleted)"
# Repeat mode
lsof -r 2 -i :8080 # Refresh every 2 seconds
# Combine with grep
lsof -p 12345 | grep REG # Regular files only
lsof -p 12345 | grep CHR # Character devices
lsof -p 12345 | grep DIR # Directories
lsof -p 12345 | grep PIPE # Pipes
# Which processes are preventing unmount
lsof +D /mnt/usb
ss -tlnp # TCP listening with PID/process
ss -ulnp # UDP listening
ss -tlnp sport = :80 # Filter by source port
ss -tlnp dport = :443 # Filter by destination port
ss -t state established # Only established TCP
ss -t state time-wait # TIME_WAIT sockets
ss -t state close-wait # CLOSE_WAIT (potential leak)
ss -tnp dst 10.0.0.1 # Connections to host
ss -tnp src 192.168.1.5 # Connections from specific local addr
ss -s # Summary statistics
ss -o # Show timer info
ss -ti # Detailed TCP info (cwnd, rtt, etc.)
ss -tlnp 'sport > 1024' # Ports above 1024
ss -xnp # Unix sockets with PID
# Show TCP internal info (RTT, send buffer, etc.)
ss -ti dst 10.0.0.1
# Filter syntax
ss -tn 'dport = :https or dport = :http'
ss -tn 'dport >= :8000 and dport <= :9000'
ip addr show # All interfaces and addresses
ip addr show eth0 # Specific interface
ip link show # Link layer info
ip route show # Routing table
ip route get 8.8.8.8 # Which route will be used for this dest
ip neigh show # ARP/neighbor table
ip -s link show eth0 # Interface statistics (packets, errors)
# Monitor (live events)
ip monitor # All netlink events
ip monitor route # Route changes only
ip monitor addr # Address changes only
ip monitor link # Link state changes
# Count syscalls by process
bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'
# Count syscalls by name system-wide
bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[ksym(args->id)] = count(); }'
# Trace file opens with path
bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s %s\n", comm, str(args->filename)); }'
# Profile CPU stacks (flamegraph input)
bpftrace -e 'profile:hz:99 { @[ustack] = count(); }' > stacks.txt
# Trace disk I/O latency
bpftrace -e 'tracepoint:block:block_rq_issue { @start[args->dev, args->sector] = nsecs; }
tracepoint:block:block_rq_complete { @usecs = hist((nsecs - @start[args->dev, args->sector]) / 1000); delete(@start[args->dev, args->sector]); }'
# TCP connection latency (connect time)
bpftrace -e 'kprobe:tcp_connect { @start[tid] = nsecs; }
kretprobe:tcp_connect { @ms = hist((nsecs - @start[tid]) / 1000000); delete(@start[tid]); }'
# malloc size distribution
bpftrace -e 'uprobe:/lib/libc.so.6:malloc { @bytes = hist(arg0); }'
# malloc latency
bpftrace -e 'uprobe:/lib/libc.so.6:malloc { @start[tid] = nsecs; }
uretprobe:/lib/libc.so.6:malloc { @ns = hist(nsecs - @start[tid]); delete(@start[tid]); }'
# Signal delivery
bpftrace -e 'tracepoint:signal:signal_generate { printf("sig %d to pid %d (%s)\n", args->sig, args->pid, args->comm); }'
# Page faults by process
bpftrace -e 'software:page-faults:1 { @[comm] = count(); }'
# New process execution
bpftrace -e 'tracepoint:syscalls:sys_enter_execve { printf("%s -> %s\n", comm, str(args->filename)); }'
# Count writes by size bucket
bpftrace -e 'tracepoint:syscalls:sys_enter_write { @[comm] = hist(args->count); }'
# Track specific PID's syscalls
bpftrace -e 'tracepoint:raw_syscalls:sys_enter /pid == 12345/ { @[ksym(args->id)] = count(); }'
# OOM killer invocations
bpftrace -e 'tracepoint:oom:oom_score_adj_update { printf("OOM adj: pid=%d (%s) score=%d\n", args->pid, str(args->comm), args->oom_score_adj); }'
# Kernel function duration (example: tcp_sendmsg)
bpftrace -e 'kprobe:tcp_sendmsg { @start[tid] = nsecs; }
kretprobe:tcp_sendmsg { @us = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'
dmesg # Kernel ring buffer
dmesg -T # Human-readable timestamps
dmesg -T | tail -50 # Last 50 messages with timestamps
dmesg -w # Follow (like tail -f)
dmesg -Tw # Follow with timestamps
dmesg -l err # Only errors
dmesg -l warn # Only warnings
dmesg -l err,warn # Errors and warnings
dmesg -l crit,alert,emerg # Critical and above
dmesg -f kern # Kernel facility only
dmesg -f daemon # Daemon facility
dmesg --human # Colored, decoded output
dmesg -c # Print and clear (root)
dmesg | grep -i "oom" # OOM killer events
dmesg | grep -i "segfault" # Segfaults
dmesg | grep -i "killed" # OOM kill events
dmesg | grep -i "oom_kill" # OOM kill
dmesg | grep "Call Trace" # Kernel panics / bugs
dmesg | grep -i "usb" # USB events
dmesg | grep -i "error" # All errors
dmesg | grep -i "i/o error" # Disk errors
dmesg -T | grep -A 20 "Out of memory"
# Shows:
# Killed process 1234 (myapp), UID 1000, total-vm:102400kB, anon-rss:98304kB
# oom_score_adj: 0
# Call trace of the oom kill path
dmesg | grep segfault
# Shows: myproc[1234]: segfault at 0 ip 00007f... sp 00007fff... error 6 in mylib.so[...]
# error codes: 4=user, 2=write, 1=protection fault (vs not-present)
objdump -d binary # Disassemble all code sections
objdump -d -M intel binary # Intel syntax (default is AT&T)
objdump -d -S binary # Interleave source (needs -g)
objdump -d -S --no-show-raw-insn binary # Without hex bytes
objdump -d --start-address=0x401234 --stop-address=0x401280 binary
objdump --disassemble=my_function binary # Disassemble specific function
objdump -x binary # All headers
objdump -h binary # Section headers (sizes, offsets)
objdump -s -j .rodata binary # Dump .rodata section contents
objdump -s -j .data binary # Dump .data section
objdump -p binary # Private headers (dynamic section, needed libs)
objdump -R binary # Dynamic relocations (PLT entries)
objdump -t binary # Symbol table
objdump -T binary # Dynamic symbol table
readelf -h binary # ELF header
readelf -S binary # Section headers
readelf -l binary # Program headers (segments)
readelf -s binary # Symbol table
readelf -d binary # Dynamic section (NEEDED libs, RPATH)
readelf -r binary # Relocations
readelf -n binary # Notes (build ID, ABI)
readelf -a binary # All of the above
readelf --debug-dump=info binary # DWARF debug info
readelf --debug-dump=frames binary # Frame info (CFI)
readelf -Wi binary # Wide output for DWARF
nm binary # Symbol table
nm -D binary # Dynamic symbols only
nm -C binary # Demangle C++ names
nm -u binary # Undefined symbols only (external deps)
nm -g binary # External symbols only
nm --size-sort binary # Sort by size (largest last)
nm --size-sort -r binary # Largest first
nm -S binary # Show symbol sizes
nm -l binary # Show source file/line (needs -g)
nm -f posix binary # POSIX output format (name value type size)
ldd binary # Shared library dependencies
ldd -v binary # Verbose (show all version requirements)
readelf -d binary | grep NEEDED # Same via readelf (no resolution)
objdump -p binary | grep NEEDED # Same via objdump
ldconfig -v 2>/dev/null | grep libname # Confirm library is in cache
LD_PRELOAD=/path/to/lib.so ./binary # Load lib.so before everything
LD_PRELOAD="lib1.so lib2.so" ./binary # Multiple libraries
# Trace malloc/free calls
LD_PRELOAD=/usr/lib/libmpatrol.so ./binary
# Suppress fsync for speed (testing/dev only, data loss risk)
LD_PRELOAD=libeatmydata.so ./binary # Install: apt install libeatmydata / pacman -S libeatmydata
# Fake system time
LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 FAKETIME="2025-01-01 00:00:00" ./binary
# Install: libfaketime
# Catch segfaults with backtrace
LD_PRELOAD=/lib/libSegFault.so SEGFAULT_SIGNALS="all" ./binary
# Interpose malloc (minimal example)
cat > mymalloc.c << 'EOF'
#define _GNU_SOURCE
#include <dlfcn.h>
#include <stdio.h>
void *malloc(size_t size) {
static void *(*real_malloc)(size_t) = NULL;
if (!real_malloc) real_malloc = dlsym(RTLD_NEXT, "malloc");
void *ptr = real_malloc(size);
fprintf(stderr, "malloc(%zu) = %p\n", size, ptr);
return ptr;
}
EOF
gcc -shared -fPIC -o mymalloc.so mymalloc.c -ldl
LD_PRELOAD=./mymalloc.so ./binary
// trace_connect.c
#define _GNU_SOURCE
#include <dlfcn.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdio.h>
int connect(int sockfd, const struct sockaddr *addr, socklen_t len) {
static int (*real_connect)() = NULL;
if (!real_connect) real_connect = dlsym(RTLD_NEXT, "connect");
if (addr->sa_family == AF_INET) {
struct sockaddr_in *in = (struct sockaddr_in *)addr;
fprintf(stderr, "connect to %s:%d\n",
inet_ntoa(in->sin_addr), ntohs(in->sin_port));
}
return real_connect(sockfd, addr, len);
}
gcc -shared -fPIC -o trace_connect.so trace_connect.c -ldl
LD_PRELOAD=./trace_connect.so ./binary
# LD_PRELOAD is ignored for setuid binaries
# To debug LD_PRELOAD loading:
LD_DEBUG=libs LD_PRELOAD=./mylib.so ./binary 2>&1 | grep mylib
# Record
rr record ./binary arg1 arg2 # Record execution
rr record -n ./binary # No signal for record noise
sudo sysctl kernel.perf_event_paranoid=1 # May be needed
# Replay
rr replay # Replay most recent recording
rr replay -d gdb # Use specific GDB
rr replay /path/to/trace/dir # Replay specific trace
# Inside GDB replay session:
(gdb) continue # Run forward
(gdb) reverse-continue # Run backward to prev event
(gdb) reverse-step # Step backward
(gdb) reverse-next # Step backward over calls
(gdb) reverse-finish # Return backward from function
(gdb) watch -l myvar # Watchpoint works both directions
(gdb) reverse-continue # Find where myvar was last written
rr record --chaos ./binary # Randomize scheduling to expose races
# Run multiple times until race reproduces, then it's deterministically replayable
for i in $(seq 1 100); do
rr record --chaos ./binary && break
done
rr replay # Now replay the exact failing run
rr record ./binary
rr replay
(gdb) break my_function
(gdb) continue
(gdb) watch -l my_struct->field # Set watchpoint
(gdb) reverse-continue # Find last write to field
(gdb) bt # Who wrote it?
rr has approximately 1.2x overhead vs native execution.
Much less than valgrind (~20x) or full instruction simulation.
Works on x86-64 Linux only.
Requires hardware performance counters (may need paranoid=1).
rr ls # List recorded traces
rr pack # Pack trace for sharing
rr replay /path/to/trace # Replay specific trace
Add to ~/.bashrc or ~/.config/fish/config.fish:
# GDB shortcuts
alias gdb-tui='gdb -tui'
alias gdb-attach='gdb -p'
alias gdb-core='gdb' # usage: gdb-core ./binary core
# strace shortcuts
alias st='strace -f -T -yy'
alias st-file='strace -e trace=file -yy'
alias st-net='strace -e trace=network -yy'
alias st-mem='strace -e trace=memory'
alias st-stat='strace -c'
alias st-err='strace -f 2>&1 | grep "= -1"'
alias st-attach='strace -p'
# ltrace shortcuts
alias lt='ltrace -C'
alias lt-stat='ltrace -c'
# perf shortcuts
alias perf-rec='perf record -g --call-graph dwarf'
alias perf-top='perf top -g'
alias perf-stat='perf stat -r 3'
alias perf-cache='perf stat -e cache-misses,cache-references,L1-dcache-load-misses,LLC-load-misses'
alias perf-branch='perf stat -e branch-misses,branch-instructions'
alias perf-sched='perf stat -e context-switches,cpu-migrations'
# Flamegraph
FGDIR=/opt/FlameGraph
alias flamegraph-cpu='perf record -F 99 -g --call-graph dwarf && perf script | $FGDIR/stackcollapse-perf.pl | $FGDIR/flamegraph.pl > cpu.svg && firefox cpu.svg'
# Valgrind shortcuts
alias vg='valgrind --leak-check=full --track-origins=yes --num-callers=30'
alias vg-fast='valgrind --leak-check=summary'
alias vg-callgrind='valgrind --tool=callgrind'
alias vg-massif='valgrind --tool=massif'
alias vg-helgrind='valgrind --tool=helgrind'
alias vg-drd='valgrind --tool=drd'
# Sanitizer build shortcuts (call as: san-asan gcc source.c -o binary)
alias san-asan='gcc -fsanitize=address -fno-omit-frame-pointer -g -O1'
alias san-ubsan='gcc -fsanitize=undefined -g -O1'
alias san-tsan='gcc -fsanitize=thread -g -O1'
alias san-clang-msan='clang -fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1'
# Core dump shortcuts
alias coredump-list='coredumpctl list'
alias coredump-debug='coredumpctl debug'
alias coredump-info='coredumpctl info'
# /proc shortcuts
alias proc-maps='cat /proc/$1/maps'
alias proc-fds='ls -la /proc/$1/fd'
alias proc-env='cat /proc/$1/environ | tr "\0" "\n"'
proc-info() { cat /proc/$1/status; }
proc-cmdline() { cat /proc/$1/cmdline | tr '\0' ' '; echo; }
# lsof shortcuts
alias lsof-port='lsof -i'
alias lsof-pid='lsof -p'
alias lsof-deleted='lsof | grep deleted'
alias lsof-listen='lsof -i tcp -s tcp:LISTEN'
# ss shortcuts
alias ss-listen='ss -tlnp'
alias ss-conn='ss -tnp state established'
alias ss-wait='ss -tn state time-wait | wc -l'
alias ss-close-wait='ss -tn state close-wait'
alias ss-stat='ss -s'
alias ss-detail='ss -ti'
# nm shortcuts
alias nm-big='nm --size-sort -r -S' # Biggest symbols first
alias nm-undef='nm -u -C' # Undefined symbols, demangled
alias nm-dyn='nm -D -C' # Dynamic symbols, demangled
# objdump shortcuts
alias dis='objdump -d -M intel -S'
alias dis-func='objdump --disassemble -M intel' # usage: dis-func myfunc binary
# bpftrace shortcuts
alias bpf-syscalls='bpftrace -e "tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }"'
alias bpf-opens='bpftrace -e "tracepoint:syscalls:sys_enter_openat { printf(\"%s %s\n\", comm, str(args->filename)); }"'
alias bpf-execs='bpftrace -e "tracepoint:syscalls:sys_enter_execve { printf(\"%s -> %s\n\", comm, str(args->filename)); }"'
# rr shortcuts
alias rr-rec='rr record'
alias rr-chaos='rr record --chaos'
alias rr-rep='rr replay'
alias rr-list='rr ls'
# dmesg shortcuts
alias dmesg-time='dmesg -T'
alias dmesg-follow='dmesg -Tw'
alias dmesg-err='dmesg -T -l err,warn'
alias dmesg-oom='dmesg -T | grep -i "out of memory\|oom\|killed process"'
alias dmesg-seg='dmesg -T | grep segfault'
# GDB
abbr --add gdb-tui 'gdb -tui'
# strace
abbr --add st 'strace -f -T -yy'
abbr --add st-file 'strace -e trace=file -yy'
abbr --add st-net 'strace -e trace=network -yy'
abbr --add st-stat 'strace -c'
# lsof
abbr --add lsof-listen 'lsof -i tcp -s tcp:LISTEN'
abbr --add lsof-deleted 'lsof | grep deleted'
# ss
abbr --add ss-listen 'ss -tlnp'
abbr --add ss-conn 'ss -tnp state established'
# perf
abbr --add perf-rec 'perf record -g --call-graph dwarf'
abbr --add perf-cache 'perf stat -e cache-misses,cache-references,L1-dcache-load-misses'
# valgrind
abbr --add vg 'valgrind --leak-check=full --track-origins=yes --num-callers=30'
# dmesg
abbr --add dmesg-err 'dmesg -T -l err,warn'
abbr --add dmesg-oom 'dmesg -T | grep -i oom'
# rr
abbr --add rr-chaos 'rr record --chaos'
abbr --add rr-rep 'rr replay'
# coredumpctl
abbr --add cdl 'coredumpctl list'
abbr --add cdd 'coredumpctl debug'
| Task | Command |
|---|---|
| Segfault with backtrace | valgrind ./bin or ASAN_OPTIONS=abort_on_error=1 ./bin-asan |
| Memory leak | valgrind --leak-check=full ./bin |
| Data race | ./bin-tsan or valgrind --tool=helgrind ./bin |
| Use-after-free | ./bin-asan |
| Integer overflow | ./bin-ubsan |
| Uninitialized read | ./bin-msan or valgrind --track-origins=yes ./bin |
| What files is this opening? | strace -e trace=openat -yy ./bin 2>&1 | grep -v ENOENT |
| What network calls? | strace -e trace=network -yy ./bin |
| Syscall timing breakdown | strace -c ./bin |
| CPU hotspot | perf record -g ./bin && perf report |
| Cache misses | perf stat -e cache-misses,cache-references ./bin |
| CPU flamegraph | perf record -F 99 -g ./bin && perf script | stackcollapse-perf.pl | flamegraph.pl > out.svg |
| Who holds port 8080 | ss -tlnp sport = :8080 or lsof -i :8080 |
| Deleted file still open | lsof | grep deleted |
| What writes to this var? | rr record ./bin && rr replay then watch var + reverse-continue |
| Deterministic race replay | rr record --chaos ./bin && rr replay |
| Kernel errors since boot | dmesg -T -l err,warn |
| OOM kills | dmesg -T | grep -i "killed process" |
| Process memory layout | cat /proc/$PID/maps |
| All open FDs with paths | ls -la /proc/$PID/fd |
| Library dependencies | ldd ./bin or readelf -d ./bin | grep NEEDED |
| Biggest symbols | nm --size-sort -r -S ./bin | head -20 |
| Count syscalls by process | bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }' |