S080 · Incident Response — Runbooks, Postmortems, On-Call

Now a Python simulator to practice the incident lifecycle:

```python
#!/usr/bin/env python3
"""incident_tabletop.py — tabletop incident simulator.

Prints a scenario, walks through the phases, prompts for decisions,
and simulates the outcome. Great for team practice; run this together
with your team once a quarter.

Run: python incident_tabletop.py
"""
from __future__ import annotations

import random
import time
from dataclasses import dataclass


@dataclass
class Scenario:
    name: str
    trigger: str
    hidden_root_cause: str
    good_actions: list[str]  # keywords in a good response
    time_to_mitigate_ideal: int  # minutes


SCENARIOS = [
    Scenario(
        name="Deploy-triggered 500s",
        trigger="Alert: 5xx rate 12% on checkout-api, started 2m ago.",
        hidden_root_cause="A refactor of the payment client shipped in the last deploy has a null-check bug.",
        good_actions=["rollback", "revert", "previous", "argocd", "deploy"],
        time_to_mitigate_ideal=5,
    ),
    Scenario(
        name="Database saturation",
        trigger="Alert: p99 latency 8s on user-service. CPU on the DB is 100%.",
        hidden_root_cause="A background job started 15m ago is running an unindexed query on the users table.",
        good_actions=["kill", "pg_terminate", "cancel", "job", "query"],
        time_to_mitigate_ideal=10,
    ),
    Scenario(
        name="Third-party outage",
        trigger="Alert: SendGrid webhook failures 100%. Emails are queuing in dead-letter queue.",
        hidden_root_cause="SendGrid is having an outage (confirmed on their status page).",
        good_actions=["status page", "third-party", "wait", "communicate", "queue"],
        time_to_mitigate_ideal=15,
    ),
]


def role_check(role: str) -> None:
    print(f"\n>>> Confirm your role: {role}")
    input("[press ENTER to continue] ")


def phase(name: str, prompt: str) -> str:
    print(f"\n{'='*60}\nPHASE: {name}\n{'='*60}")
    print(prompt)
    return input("Your action: ").lower().strip()


def evaluate_action(action: str, expected_keywords: list[str]) -> bool:
    return any(k in action for k in expected_keywords)


def run_scenario(s: Scenario) -> None:
    print(f"\n\n{'#'*60}\nSCENARIO: {s.name}\n{'#'*60}")
    print(f"\n[t=0]  🚨 {s.trigger}")

    # Phase 1: Detect + declare
    role_check("Incident Commander")
    action = phase("Declare",
                   "Is this an incident? What's your first move? "
                   "(hint: assign roles, open channel, page comms)")
    if any(k in action for k in ["declare", "channel", "comms", "role"]):
        print("✅ Good — you declared and set up roles.")
    else:
        print("⚠️  You should DECLARE and assign roles before diving in.")

    # Phase 2: Diagnose
    action = phase("Diagnose",
                   "What do you check first? (hint: recent deploys? "
                   "dashboards? logs? status page?)")
    if any(k in action for k in ["deploy", "dashboard", "log", "status"]):
        print("✅ Good — you looked at real data.")

    # Phase 3: Mitigate
    print(f"\n[t=+8min] Root cause found: {s.hidden_root_cause}")
    action = phase("Mitigate",
                   f"How do you STOP THE BLEEDING right now? "
                   f"(remember: mitigate first, root-cause later)")
    if evaluate_action(action, s.good_actions):
        outcome_time = s.time_to_mitigate_ideal
        print(f"✅ Mitigation successful. TTM = {outcome_time} min.")
    else:
        outcome_time = s.time_to_mitigate_ideal * 3
        print(f"❌ That's not the fastest mitigation. TTM = {outcome_time} min "
              f"(3x ideal — customers hurt for longer).")

    # Phase 4: Postmortem outline
    print(f"\n[t=+{outcome_time}min] Service recovered. Schedule postmortem.")
    action = phase("Postmortem outline",
                   "Name three action items you'd include. "
                   "(examples: fix root cause, improve runbook, add alert)")
    if "," in action or "\n" in action or "and" in action:
        print("✅ You have concrete action items with clear scope.")
    else:
        print("⚠️  Try to have 3+ specific action items, each with an owner and due date.")


def main() -> None:
    print("=" * 60)
    print("INCIDENT TABLETOP EXERCISE")
    print("=" * 60)
    print("\nYou'll run through one randomly-selected scenario.")
    print("Answer as if this were a real incident.\n")

    scenario = random.choice(SCENARIOS)
    run_scenario(scenario)

    print("\n\n" + "=" * 60)
    print("Debrief")
    print("=" * 60)
    print("- What went well?")
    print("- What was harder than expected?")
    print("- What would you add to your team's runbook after this?\n")


if __name__ == "__main__":
    main()