On-call incident response checklist (script)

Jam 2 pagi, service down, PagerDuty bunyi. Pertama yang dilakukan biasanya: kubectl delete pod restart. Tapi setelah restart, state hilang — gak bisa investigate kemudian. Snippet ini script collect forensic data dulu (10 detik) sebelum take action. Auto-save ke S3 / shared volume.

Kode

#!/usr/bin/env bash
# /opt/oncall/collect_forensic.sh
# Usage: collect_forensic.sh <namespace> <pod-name> [output-dir]

set -uo pipefail   # NOTE: no -e, kita mau lanjut walau ada step fail

readonly NS="${1:?Pakai: $0 <namespace> <pod> [output-dir]}"
readonly POD="${2:?Pakai: $0 <namespace> <pod> [output-dir]}"
readonly OUTPUT_BASE="${3:-/tmp/oncall}"
readonly TIMESTAMP=$(date -u +"%Y%m%dT%H%M%SZ")
readonly OUTPUT_DIR="${OUTPUT_BASE}/${NS}-${POD}-${TIMESTAMP}"
readonly S3_BUCKET="${S3_BUCKET:-s3://oncall-forensic}"

mkdir -p "${OUTPUT_DIR}"

log() {
  echo "[$(date -u +%H:%M:%S)] $*" | tee -a "${OUTPUT_DIR}/collection.log"
}

main() {
  log "=== Forensic collection: ${NS}/${POD} ==="
  log "Output: ${OUTPUT_DIR}"

  # ==========================================
  # 1. POD METADATA & STATE
  # ==========================================
  log "1. Collecting pod metadata"
  kubectl get pod -n "${NS}" "${POD}" -o yaml > "${OUTPUT_DIR}/pod.yaml" 2>&1 || true
  kubectl describe pod -n "${NS}" "${POD}" > "${OUTPUT_DIR}/pod-describe.txt" 2>&1 || true

  # Events terkait
  kubectl get events -n "${NS}" \
    --field-selector "involvedObject.name=${POD}" \
    --sort-by='.lastTimestamp' \
    > "${OUTPUT_DIR}/events.txt" 2>&1 || true

  # Node info
  local node
  node=$(kubectl get pod -n "${NS}" "${POD}" -o jsonpath='{.spec.nodeName}' 2>/dev/null)
  if [ -n "${node}" ]; then
    log "   Pod on node: ${node}"
    kubectl describe node "${node}" > "${OUTPUT_DIR}/node-describe.txt" 2>&1 || true
  fi

  # ==========================================
  # 2. LOG — current + previous (kalau ada crash)
  # ==========================================
  log "2. Collecting logs"
  local containers
  containers=$(kubectl get pod -n "${NS}" "${POD}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null)

  for container in ${containers}; do
    log "   Container: ${container}"
    kubectl logs -n "${NS}" "${POD}" -c "${container}" --tail=2000 \
      > "${OUTPUT_DIR}/log-${container}.txt" 2>&1 || true
    kubectl logs -n "${NS}" "${POD}" -c "${container}" --previous --tail=2000 \
      > "${OUTPUT_DIR}/log-${container}-previous.txt" 2>&1 || \
      rm -f "${OUTPUT_DIR}/log-${container}-previous.txt"
  done

  # ==========================================
  # 3. THREAD DUMP / HEAP (kalau JVM)
  # ==========================================
  log "3. Checking JVM / process state"
  local pid
  pid=$(kubectl exec -n "${NS}" "${POD}" -- pgrep java 2>/dev/null | head -1)
  if [ -n "${pid}" ]; then
    log "   JVM detected, PID=${pid}"

    # Thread dump (jstack)
    kubectl exec -n "${NS}" "${POD}" -- \
      jstack "${pid}" > "${OUTPUT_DIR}/threaddump.txt" 2>&1 \
      || log "   WARN: jstack fail"

    # Heap summary (jmap)
    kubectl exec -n "${NS}" "${POD}" -- \
      jmap -histo:live "${pid}" 2>/dev/null | head -100 \
      > "${OUTPUT_DIR}/heap-histo.txt" || log "   WARN: jmap fail"

    # GC stats
    kubectl exec -n "${NS}" "${POD}" -- \
      jstat -gc "${pid}" > "${OUTPUT_DIR}/gc-stats.txt" 2>&1 \
      || log "   WARN: jstat fail"

  else
    # Process list umum
    kubectl exec -n "${NS}" "${POD}" -- ps auxf 2>/dev/null \
      > "${OUTPUT_DIR}/processes.txt" || true

    # /proc info untuk PID 1
    kubectl exec -n "${NS}" "${POD}" -- sh -c '
      echo "=== /proc/1/status ==="; cat /proc/1/status;
      echo ""; echo "=== /proc/1/io ==="; cat /proc/1/io;
      echo ""; echo "=== /proc/1/limits ==="; cat /proc/1/limits;
    ' > "${OUTPUT_DIR}/proc-info.txt" 2>&1 || true
  fi

  # ==========================================
  # 4. NETWORK STATE
  # ==========================================
  log "4. Network state"
  kubectl exec -n "${NS}" "${POD}" -- sh -c '
    echo "=== Connection states ==="
    ss -tan 2>/dev/null || netstat -tan 2>/dev/null || echo "no ss/netstat"
    echo ""
    echo "=== Routes ==="
    ip route 2>/dev/null || route -n 2>/dev/null
    echo ""
    echo "=== DNS test ==="
    nslookup kubernetes.default 2>&1 | head -10
  ' > "${OUTPUT_DIR}/network.txt" 2>&1 || true

  # ==========================================
  # 5. METRICS dari Prometheus
  # ==========================================
  log "5. Querying Prometheus for last 15 min metric"
  if [ -n "${PROMETHEUS_URL:-}" ]; then
    local end
    end=$(date +%s)
    local start=$((end - 900))   # 15 menit terakhir

    # Query metric umum
    cat > "${OUTPUT_DIR}/metrics-queries.txt" <<EOF
=== CPU usage (15 min) ===
$(curl -s "${PROMETHEUS_URL}/api/v1/query_range" \
  --data-urlencode "query=rate(container_cpu_usage_seconds_total{pod=\"${POD}\"}[1m])" \
  --data-urlencode "start=${start}" \
  --data-urlencode "end=${end}" \
  --data-urlencode "step=15s")

=== Memory usage (15 min) ===
$(curl -s "${PROMETHEUS_URL}/api/v1/query_range" \
  --data-urlencode "query=container_memory_working_set_bytes{pod=\"${POD}\"}" \
  --data-urlencode "start=${start}" \
  --data-urlencode "end=${end}" \
  --data-urlencode "step=15s")

=== Request rate ===
$(curl -s "${PROMETHEUS_URL}/api/v1/query_range" \
  --data-urlencode "query=sum(rate(http_requests_total{pod=\"${POD}\"}[1m]))" \
  --data-urlencode "start=${start}" \
  --data-urlencode "end=${end}" \
  --data-urlencode "step=15s")
EOF
  else
    log "   Prometheus URL tidak set, skip metric"
  fi

  # ==========================================
  # 6. RELATED PODS (same deployment)
  # ==========================================
  log "6. Sibling pods"
  local labels
  labels=$(kubectl get pod -n "${NS}" "${POD}" -o jsonpath='{.metadata.labels.app}' 2>/dev/null)
  if [ -n "${labels}" ]; then
    kubectl get pod -n "${NS}" -l "app=${labels}" -o wide \
      > "${OUTPUT_DIR}/sibling-pods.txt" 2>&1 || true
  fi

  # ==========================================
  # 7. PACKAGE & UPLOAD
  # ==========================================
  log "7. Packaging & upload"
  local archive="/tmp/forensic-${NS}-${POD}-${TIMESTAMP}.tar.gz"
  tar czf "${archive}" -C "$(dirname "${OUTPUT_DIR}")" "$(basename "${OUTPUT_DIR}")"
  log "   Archive: ${archive} ($(du -h "${archive}" | cut -f1))"

  if command -v aws &> /dev/null && [ -n "${S3_BUCKET}" ]; then
    if aws s3 cp "${archive}" "${S3_BUCKET}/" --quiet; then
      log "   Uploaded ke ${S3_BUCKET}/$(basename "${archive}")"
    else
      log "   WARN: S3 upload fail, archive masih ada lokal"
    fi
  fi

  log "=== Collection selesai ==="
  log ""
  log "Next steps:"
  log "  1. Review log dan event di ${OUTPUT_DIR}"
  log "  2. Restart pod / take action"
  log "  3. Post di #incident channel: archive link ${S3_BUCKET}/$(basename "${archive}")"
  log "  4. Schedule postmortem"
}

main "$@"

Pemakaian

# Setup
chmod +x /opt/oncall/collect_forensic.sh

# Saat insiden — collect dulu, baru action
PROMETHEUS_URL=http://prometheus.observability.svc:9090 \
S3_BUCKET=s3://tangerang-forensic \
/opt/oncall/collect_forensic.sh production api-tokopedia-67d8f-xyz

# Output:
# [10:32:00] === Forensic collection: production/api-tokopedia-67d8f-xyz ===
# [10:32:00] Output: /tmp/oncall/production-api-tokopedia-67d8f-xyz-20260718T033200Z
# [10:32:00] 1. Collecting pod metadata
# [10:32:01] 2. Collecting logs
# [10:32:02]    Container: api
# [10:32:03] 3. Checking JVM / process state
# [10:32:03]    JVM detected, PID=1
# [10:32:08] 4. Network state
# [10:32:09] 5. Querying Prometheus
# [10:32:10] 6. Sibling pods
# [10:32:11] 7. Packaging & upload
# [10:32:12]    Archive: /tmp/forensic-...-20260718T033200Z.tar.gz (4.2M)
# [10:32:14]    Uploaded ke s3://tangerang-forensic/forensic-...
# [10:32:14] === Collection selesai ===

# Baru aman take action
kubectl delete pod -n production api-tokopedia-67d8f-xyz

# Pattern: incident handler runbook
# 1. Page diterima → ack di PagerDuty
# 2. Jump ke #incident-XXX Slack channel
# 3. Collect forensic:
#    /opt/oncall/collect_forensic.sh <ns> <pod>
# 4. Identifikasi root cause kandidat:
#    - OOMKilled? cek heap-histo.txt
#    - Deadlock? cek threaddump.txt
#    - Network? cek network.txt + sibling-pods.txt
#    - External dep? cek log untuk error pattern
# 5. Mitigation:
#    - Rollback deployment kalau baru deploy
#    - Restart pod kalau resource issue
#    - Failover DB kalau dep issue
# 6. Communication update setiap 15 menit di channel
# 7. Setelah resolve, schedule postmortem dalam 48 jam

# Helm chart annotation untuk auto-collect saat pod terminating
# spec:
#   template:
#     metadata:
#       annotations:
#         oncall.kodekarawaci.id/pre-stop-collect: "true"
#     spec:
#       containers:
#         - name: app
#           lifecycle:
#             preStop:
#               exec:
#                 command:
#                   - /bin/sh
#                   - -c
#                   - |
#                     if [ "${POD_TERMINATING_FORENSIC:-false}" = "true" ]; then
#                       jstack 1 > /shared/threaddump-$(date +%s).txt 2>&1
#                     fi
#                     sleep 5

Kapan dipakai

• Production insident — collect dulu, restart kedua.
• Investigasi flaky pod yang restart sporadic.
• Post-mortem material gathering — evidence untuk timeline.
• Recurring issue — bandingkan forensic dari beberapa kejadian.

Catatan

• set -uo pipefail tanpa -e — sengaja. Kita mau collect sebanyak mungkin walau ada step fail. Setiap command pakai || true.
• kubectl logs —previous crucial — log container yang crash sudah hilang dari current. Previous masih ada satu generasi.
• Thread dump tepat waktu — kalau pod sudah stuck deadlock, thread dump akan show. Setelah restart, hilang.
• jstack butuh JVM tool. Untuk container Java production, install JDK (bukan JRE saja) atau pakai sidecar pattern.
• Upload ke shared storage — local filesystem hilang saat pod terminate. S3 / NFS shared volume aman.
• Prometheus query — capture metric historic 15 menit. Berguna kalau dashboard Grafana down juga.
• Postmortem template — link archive S3 di postmortem doc. Evidence reusable untuk training.

Forensic collection takes 30-60 detik. Kalau ada SLA strict (5 menit recovery), parallelize step yang independent. Atau pakai auto-snapshot via Kubernetes pre-stop hook.