Kubernetes debug pod cheat-sheet

Pod CrashLoopBackOff jam 2 malam. Image distroless, gak bisa exec ke shell. Pengen liat network connectivity, file system, env var. Snippet ini cheat-sheet kubectl debug yang sering dipakai SRE Indonesia — ephemeral container, port-forward, log streaming, dan ad-hoc curl test dari dalam cluster.

Kode

#!/usr/bin/env bash
# /usr/local/bin/kdebug.sh — wrapper untuk pattern debugging umum
# Pakai: kdebug <subcommand> <pod-name> [namespace]

set -euo pipefail

readonly DEFAULT_NS="${KDEBUG_NAMESPACE:-default}"
readonly DEBUG_IMAGE="${KDEBUG_IMAGE:-nicolaka/netshoot:latest}"

usage() {
  cat <<EOF
Pakai: $(basename "$0") <command> <pod> [namespace]

Commands:
  shell        Spawn shell debug di pod (ephemeral container)
  shell-copy   Copy pod dengan image debug (untuk distroless)
  logs         Stream log + sebelum + history crash
  network      Cek network connectivity dari pod
  env          Dump environment variables
  forward      Port forward (interactive)
  exec         Direct exec ke container existing
  describe     Verbose describe + events

Env override:
  KDEBUG_NAMESPACE=stagings $(basename "$0") shell pod-xxx
  KDEBUG_IMAGE=busybox:1.36 $(basename "$0") network pod-xxx
EOF
}

if [ $# -lt 2 ]; then
  usage
  exit 1
fi

readonly CMD=$1
readonly POD=$2
readonly NS="${3:-${DEFAULT_NS}}"

cmd_shell() {
  echo ">> Attach ephemeral container ke pod ${POD}"
  echo "   Image: ${DEBUG_IMAGE}"
  kubectl debug -n "${NS}" "${POD}" \
    --image="${DEBUG_IMAGE}" \
    --target="$(get_main_container)" \
    --share-processes \
    --stdin \
    --tty
}

cmd_shell_copy() {
  echo ">> Copy pod ${POD} dengan image debug (no in-place modify)"
  local debug_pod="${POD}-debug-$(date +%s)"
  kubectl debug -n "${NS}" "${POD}" \
    --copy-to="${debug_pod}" \
    --image="${DEBUG_IMAGE}" \
    --container="$(get_main_container)" \
    --stdin --tty -- bash
}

cmd_logs() {
  echo ">> Container: $(get_main_container)"
  echo ""
  echo "=== Current logs (last 200 lines) ==="
  kubectl logs -n "${NS}" "${POD}" --tail=200 || true
  echo ""
  echo "=== Previous crash logs (kalau ada) ==="
  kubectl logs -n "${NS}" "${POD}" --previous --tail=200 2>/dev/null || \
    echo "(tidak ada previous container)"
  echo ""
  echo "=== Stream live (Ctrl+C untuk stop) ==="
  kubectl logs -n "${NS}" "${POD}" -f --tail=50
}

cmd_network() {
  echo ">> Network test dari ephemeral container"
  kubectl debug -n "${NS}" "${POD}" \
    --image="${DEBUG_IMAGE}" \
    --target="$(get_main_container)" \
    --share-processes \
    --stdin --tty \
    -- bash -c '
echo "=== Network info ===";
ip addr show eth0 | grep inet;
echo "";
echo "=== DNS resolution ===";
nslookup kubernetes.default.svc.cluster.local || echo "DNS fail";
echo "";
echo "=== Routing ===";
ip route;
echo "";
echo "=== Connection test ===";
echo "Ke kubernetes API:";
nc -zv kubernetes.default.svc.cluster.local 443 2>&1;
echo "";
echo "Pakai curl untuk service tertentu:";
echo "  curl -v http://nama-service:port/path";
bash
'
}

cmd_env() {
  echo ">> Environment variables pod ${POD}"
  kubectl exec -n "${NS}" "${POD}" -c "$(get_main_container)" -- printenv \
    2>/dev/null | sort \
    || kubectl debug -n "${NS}" "${POD}" \
        --image="${DEBUG_IMAGE}" \
        --target="$(get_main_container)" \
        --share-processes \
        -- cat /proc/1/environ | tr '\0' '\n' | sort
}

cmd_forward() {
  echo ">> Port forward pod ${POD}"
  echo "   Container ports:"
  kubectl get pod -n "${NS}" "${POD}" -o jsonpath='{.spec.containers[*].ports[*]}' \
    | jq -r '"\(.containerPort)/\(.protocol // "TCP") (\(.name // "unnamed"))"' \
    2>/dev/null || echo "(tidak ada port declared)"
  echo ""
  read -p "Local port: " local_port
  read -p "Pod port: " pod_port

  kubectl port-forward -n "${NS}" "${POD}" "${local_port}:${pod_port}"
}

cmd_exec() {
  local container
  container=$(get_main_container)
  echo ">> Exec ke ${POD}/${container}"
  kubectl exec -n "${NS}" -it "${POD}" -c "${container}" -- \
    sh -c 'if command -v bash > /dev/null; then bash; else sh; fi'
}

cmd_describe() {
  echo "=== Pod describe ==="
  kubectl describe pod -n "${NS}" "${POD}"
  echo ""
  echo "=== Events terkait ==="
  kubectl get events -n "${NS}" \
    --field-selector "involvedObject.name=${POD}" \
    --sort-by='.lastTimestamp' \
    -o custom-columns=TIME:.lastTimestamp,TYPE:.type,REASON:.reason,MSG:.message
}

get_main_container() {
  kubectl get pod -n "${NS}" "${POD}" \
    -o jsonpath='{.spec.containers[0].name}'
}

case "${CMD}" in
  shell)      cmd_shell ;;
  shell-copy) cmd_shell_copy ;;
  logs)       cmd_logs ;;
  network)    cmd_network ;;
  env)        cmd_env ;;
  forward)    cmd_forward ;;
  exec)       cmd_exec ;;
  describe)   cmd_describe ;;
  *)          usage; exit 1 ;;
esac

# Quick reference inline commands (tanpa wrapper script)

# === Debug pod CrashLoopBackOff ===
# 1. Cek event terkini
kubectl get events --sort-by=.lastTimestamp -n production | tail -20

# 2. Log container yang crash
kubectl logs <pod> -n production --previous

# 3. Describe pod cek reason
kubectl describe pod <pod> -n production | grep -A20 "State:"

# === Debug pod yang running tapi ngambek ===
# 1. Ephemeral debug container (k8s 1.25+)
kubectl debug <pod> -n production \
  --image=nicolaka/netshoot \
  --target=app-container \
  --share-processes \
  -it

# 2. Copy pod + replace image jadi debug-able
kubectl debug <pod> -n production \
  --copy-to=<pod>-debug \
  --container=app-container \
  --image=nicolaka/netshoot \
  -it -- bash

# === Forward port untuk test ===
# Port forward 8080 local → 8080 pod
kubectl port-forward -n production <pod> 8080:8080

# === Resource usage ===
kubectl top pod -n production --containers
kubectl top node

# === Multi-container log streaming ===
kubectl logs <pod> -n production --all-containers --prefix -f

Pemakaian

# Setup wrapper script
chmod +x /usr/local/bin/kdebug.sh
ln -sf /usr/local/bin/kdebug.sh /usr/local/bin/kdebug

# Pemakaian
kdebug shell api-tokopedia-67d8f-xyz production
kdebug logs api-tokopedia-67d8f-xyz production
kdebug network api-tokopedia-67d8f-xyz production
kdebug describe api-tokopedia-67d8f-xyz production

# Debug pod distroless yang gak punya shell
kdebug shell-copy api-distroless-abc production

# Pattern: copy file dari pod ke local untuk analisis
kubectl cp production/<pod>:/tmp/heapdump.hprof ./heapdump.hprof

# Pattern: exec dengan env var override (test config change)
kubectl exec <pod> -n production -- \
  env DATABASE_URL=postgres://test... ./app --dry-run

# Pattern: snapshot logs semua pod dengan label
kubectl logs -l app=tokopedia -n production --tail=1000 --prefix \
  > /tmp/all-pods-$(date +%s).log

# Investigasi: pod restart 10 kali dalam 1 jam
POD=api-tokopedia-67d8f-xyz
NS=production

# 1. Cek restart count
kubectl get pod -n $NS $POD \
  -o jsonpath='{.status.containerStatuses[*].restartCount}'

# 2. Exit code dari last termination
kubectl get pod -n $NS $POD \
  -o jsonpath='{.status.containerStatuses[*].lastState.terminated}' \
  | jq .

# 3. Resource limit + actual usage
kubectl get pod -n $NS $POD \
  -o jsonpath='{.spec.containers[*].resources}' | jq .
kubectl top pod -n $NS $POD --containers

# 4. Cek host node — kalau OOMKilled, kemungkinan limit terlalu kecil
kubectl describe pod -n $NS $POD | grep -A5 "Last State"

Kapan dipakai

• Pod crash di production tengah malam.
• Investigasi konektivitas antar service (network policy issue).
• Test patch konfigurasi sebelum apply.
• Forensic — collect log + thread dump dari pod yang akan di-delete.

Catatan

• kubectl debug butuh k8s 1.25+ untuk ephemeral container. Versi lama pakai kubectl run debug pod terpisah.
• share-processes wajib supaya bisa lihat process container utama dari debug container (ps aux, /proc/1).
• nicolaka/netshoot lengkap network tool: dig, nslookup, nc, tcpdump, mtr, curl. Untuk image lebih kecil pakai busybox atau alpine.
• —copy-to safe untuk production — gak ganggu pod asli, debug di pod copy. Hapus pod copy setelah selesai.
• —previous flag logs crucial untuk crash investigation — log container baru pasti kosong, yang dibutuhkan log container yang crash.
• Port forward bukan untuk production traffic — single connection, gak load balance, ke satu pod saja. Pakai untuk debug doang.
• RBAC — beberapa subcommand butuh pod/portforward, pod/exec permission. Pastikan service account dev punya akses ini.

Saat panik di outage, simpan log + describe + events ke file dulu sebelum action. Pod yang di-restart hapus container state. kubectl logs --previous cuma simpan 1 generation terakhir.