# Download
wget https://raw.githubusercontent.com/samatild/LinuxAiOPerf/main/build/linux_aio_perfcheck.sh

# Make executable
chmod +x linux_aio_perfcheck.sh

# Run (interactive mode)
sudo ./linux_aio_perfcheck.sh

# 1. Overall load and "is anything obviously wrong?"
uptime
dmesg -T | tail -50

# 2. CPU + run queue + swap + I/O wait, sampled every second
vmstat 1 5

# 3. Per-CPU breakdown (is one core pegged? is it user, system, or iowait?)
mpstat -P ALL 1 3

# 4. Top processes by CPU and memory
pidstat -u 1 3
pidstat -r 1 3

# 5. Disk pressure
iostat -xz 1 3

# 6. Memory truth
free -h

ps aux                              # All processes: user, CPU, MEM, state, command
ps auxf                             # Same, with ASCII process tree
ps -eo pid,ppid,stat,comm           # Custom columns: PID, parent, state, name
ps -eo pid,ppid,stat,wchan,args     # Add wait-channel (what kernel fn it's blocked in)

# Filter by name
ps aux | grep nginx

# Show a specific PID
ps -p <PID> -o pid,ppid,stat,vsz,rss,comm

# Find processes in bad states
ps aux | awk '$8 ~ /^[DZ]/'         # D-state (uninterruptible) or zombies

# Process status (state, memory, threads, signals)
cat /proc/<PID>/status

# Key fields
grep -E 'State|VmSize|VmRSS|VmSwap|Threads|SigCgt' /proc/<PID>/status

# What kernel function is the process sleeping in?
cat /proc/<PID>/wchan

# Full command line
cat /proc/<PID>/cmdline | tr '\0' ' '

# Open file descriptors
ls -la /proc/<PID>/fd

# Memory map (virtual address space)
cat /proc/<PID>/maps

# Detailed per-region memory accounting
cat /proc/<PID>/smaps_rollup

# System-wide memory
grep -E 'MemTotal|MemAvailable|SwapTotal|SwapFree' /proc/meminfo

vmstat 1            # Update every 1 second (forever)
vmstat 1 30         # 30 samples, 1 second apart
vmstat -s           # Summary totals
vmstat -m           # Slab allocator stats

procs --------memory--------- --swap- ---io-- -system- ------cpu-----
 r  b   swpd   free  buff cache  si  so   bi   bo   in   cs us sy id wa

# Install sysstat if needed
sudo apt-get install -y sysstat        # Debian/Ubuntu
sudo dnf install -y sysstat            # RHEL/Fedora
sudo zypper install -y sysstat         # SUSE

pidstat -u 1            # CPU usage
pidstat -r 1            # Memory (RSS, VSZ)
pidstat -d 1            # Disk I/O per process
pidstat -w 1            # Context switches
pidstat -urd 1          # All metrics combined
pidstat -p <PID> -u 1   # Filter for one PID

iostat -xz 1          # Extended stats, skip idle devices, 1s interval
iostat -xz 1 10       # 10 samples
iostat -h 1           # Human-readable throughput

free -h               # Human-readable
watch -n1 free -h     # Refresh every second

top                   # Classic interactive view
htop                  # Colourful interactive (sudo apt install htop)

# What syscalls is this process making right now?
strace -p <PID>

# Count syscalls (summary mode)
strace -c -p <PID>       # Ctrl-C after a few seconds

# Trace specific syscalls only
strace -e trace=read,write,open,openat,close -p <PID>

# Timestamps + time per syscall
strace -tt -T -p <PID>

# Who has this file/socket open?
lsof -p <PID>
lsof -i :443
lsof /var/log/syslog

kill <PID>            # SIGTERM (15) — polite request
kill -KILL <PID>      # SIGKILL (9) — cannot be caught
kill -STOP <PID>      # Freeze (SIGSTOP — cannot be caught)
kill -CONT <PID>      # Resume
pkill nginx           # Kill by name
kill -l               # List all signals

# Download
wget https://raw.githubusercontent.com/samatild/LinuxAiOPerf/main/build/linux_aio_perfcheck.sh

# Make executable
chmod +x linux_aio_perfcheck.sh

# Run (interactive mode)
sudo ./linux_aio_perfcheck.sh

# Pick a run directory
RUN=/var/log/perfrun-$(date +%Y%m%d-%H%M%S)
mkdir -p "$RUN"

# CPU + memory + I/O + run queue, every 5s
vmstat -t 5     | tee "$RUN/vmstat.log"   &

# Per-CPU breakdown
mpstat -P ALL 5 | tee "$RUN/mpstat.log"   &

# Per-process CPU, mem and I/O
pidstat -urd 5  | tee "$RUN/pidstat.log"  &

# Disk extended stats
iostat -xtz 5   | tee "$RUN/iostat.log"   &

# Network interfaces
sar -n DEV 5    | tee "$RUN/sar-net.log"  &

# Periodic process snapshot
while true; do
  date '+=== %F %T ===' >> "$RUN/ps.log"
  ps -eo pid,ppid,stat,pcpu,pmem,rss,vsz,wchan:25,args --sort=-pcpu \
    | head -50 >> "$RUN/ps.log"
  sleep 30
done &

# Enable and start the collector
sudo systemctl enable --now sysstat

# On most distros, tighten the sample interval:
sudo sed -i 's|^\*/10 |*/1 |' /etc/cron.d/sysstat 2>/dev/null || true
# (On systems using the systemd timer, edit sysstat-collect.timer instead)

# CPU history for a specific day (day-of-month)
sar -u -f /var/log/sysstat/sa15

# Restrict to the window of interest
sar -u -s 03:00:00 -e 04:00:00 -f /var/log/sysstat/sa15

# Memory, swap, I/O, network, run queue, context switches
sar -r -f /var/log/sysstat/sa15
sar -S -f /var/log/sysstat/sa15
sar -b -f /var/log/sysstat/sa15
sar -n DEV -f /var/log/sysstat/sa15
sar -q -f /var/log/sysstat/sa15
sar -w -f /var/log/sysstat/sa15

THRESHOLD=10
OUT=/var/log/perf-trigger
mkdir -p "$OUT"

while true; do
  L=$(awk '{print int($1)}' /proc/loadavg)
  if [ "$L" -ge "$THRESHOLD" ]; then
    TS=$(date +%Y%m%d-%H%M%S)
    {
      echo "=== $TS load=$L ==="
      uptime
      ps -eo pid,stat,pcpu,pmem,rss,wchan:25,args --sort=-pcpu | head -40
      cat /proc/meminfo
      vmstat 1 5
      iostat -xz 1 3
    } >> "$OUT/triggers-$TS.log"
    sleep 60   # cool-down so we don't spam
  fi
  sleep 5
done

# When did we swap?
awk 'NR>2 && ($7>0 || $8>0)' vmstat.log

# When did %iowait spike?
awk 'NR>3 && $16>20' vmstat.log     # column index depends on vmstat version

# Which processes had the biggest RSS over time?
awk '$6 > 500000' pidstat.log

# Which disks went past 20ms await?
awk '$10 > 20' iostat.log

# 1. What's using CPU right now?
pidstat -u 1 60 | tee /tmp/cpu.log

# 2. What's using memory?
pidstat -r 1 30 | tee /tmp/mem.log
awk '$6 > 100000' /tmp/mem.log          # RSS > 100 MB

# 3. Is the system swapping?
vmstat 1 10                              # watch si / so

# 4. Is disk I/O the bottleneck?
iostat -xz 1 10                          # watch await, %util

# 5. Find processes in bad states
ps aux | awk '$8 ~ /^[DZ]/'              # D-state or zombies

# 6. What is a specific process waiting on?
cat /proc/<PID>/wchan
grep State /proc/<PID>/status

# 7. How much memory is a process really using?
grep -E 'VmRSS|VmSwap|VmSize' /proc/<PID>/status
cat /proc/<PID>/smaps_rollup

# 8. Is a process leaking memory? Watch RSS over time.
watch -n2 'grep VmRSS /proc/<PID>/status'

# 9. Was there an OOM kill / hung task / disk error?
dmesg -T | grep -iE 'oom|killed process|hung task|i/o error|blocked for'
journalctl -k --since "1 hour ago" | grep -iE 'oom|hung|error'