Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

29.1 KB
Raw Blame
#!/usr/bin/env python3
"""
AiSOC AI-vs-AI adversary scenario generator (Wave 2 — w2-aivai).
=================================================================

Takes the deterministic 200-incident benchmark and produces a parallel
"adversary-mutated" dataset that simulates what an attacker-LLM would do
to evade a keyword/regex-based defender: synonym substitution, character
obfuscation, fragmentation, and softening of tactic-betraying language.

The mutator is INTENTIONALLY DETERMINISTIC. We don't want to ship a flaky
CI suite that depends on a live model — so the "attacker LLM" is replaced
with a content-derived hash + a curated mutation grammar. The grammar is
built directly from the defender's `_TACTIC_KEYWORDS` dictionary in
`services/agents/tests/test_mitre_accuracy.py`, so every keyword the
defender knows about gets at least one evasive alternate.

Mutation intensity per incident is bucketed by `sha256(id)`:

  heavy   ≈ 45%  — every keyword in title+description rewritten
  medium  ≈ 35%  — only one of the expected tactics is preserved cleanly
  light   ≈ 20%  — light leetspeak only; defender should still catch

Targeted floor for the resulting eval suite is ~0.55, demonstrating
graceful degradation under adversarial obfuscation rather than the 0.80
defender-friendly floor of `test_mitre_accuracy.py`.

Outputs `services/agents/tests/eval_data/adversary_incidents.json`. Re-runs
are byte-identical given the same input dataset and mutation grammar.

Usage:
    python3 scripts/generate_adversary_incidents.py
    python3 scripts/generate_adversary_incidents.py --report
    python3 scripts/generate_adversary_incidents.py --out PATH
"""

from __future__ import annotations

import argparse
import hashlib
import json
import re
import sys
from pathlib import Path
from typing import Any

_REPO_ROOT = Path(__file__).resolve().parent.parent
_AGENTS_ROOT = _REPO_ROOT / "services" / "agents"
sys.path.insert(0, str(_AGENTS_ROOT))

# Pull the defender's keyword catalogue directly so we never drift from it.
from tests.test_mitre_accuracy import (  # type: ignore  # noqa: E402
    _TACTIC_KEYWORDS,
    extract_tactics_from_text,
)

_DEFAULT_INPUT = _AGENTS_ROOT / "tests" / "eval_data" / "synthetic_incidents.json"
_DEFAULT_OUTPUT = _AGENTS_ROOT / "tests" / "eval_data" / "adversary_incidents.json"


# ---------------------------------------------------------------------------
# Mutation grammar
#
# Each entry rewrites the substring on the LEFT to one of the alternates on
# the right. Alternates are intentionally chosen so they do NOT match any
# keyword in `_TACTIC_KEYWORDS` (substring-checked at module load time
# below — `_validate_grammar()` will fail loudly if a synonym would still
# trip a defender keyword for any tactic).
#
# Substring keys are matched case-insensitively. Multi-word keys are
# preserved literally; single-word keys use word-boundary matching so e.g.
# "macro" doesn't replace "macroeconomics" (not that it appears here, but
# the principle is sound).
# ---------------------------------------------------------------------------


# Each value is a deterministic alternate the attacker might use. Picked by
# `(incident_id, keyword)` hash so a single incident is internally consistent
# but the dataset as a whole exercises the grammar broadly.
_SYNONYMS: dict[str, tuple[str, ...]] = {
    # TA0001 — Initial Access
    "initial access": ("first foothold", "entry stage", "ingress beachhead"),
    "phishing": ("ruse mail", "deceptive lure", "ph1sh1ng", "fraudulent outreach"),
    "spear-phish": ("targeted lure", "directed bait", "narrow-cast bait"),
    "spear": ("focused lure", "direct lure", "targeted bait"),
    "watering hole": ("co-opted landing site", "doctored portal", "ambush page"),
    "supply chain": ("vendor-pipeline trojan", "build-pipeline poisoning", "third-party-build poisoning"),
    "valid account": ("legitimate-id reuse", "trusted identity reuse", "clean-id reuse"),
    "external remote service": ("internet-reachable gateway", "outward-facing edge daemon", "edge-facing remote daemon"),
    "drive-by": ("auto-load infection", "passive-landing infection", "no-click landing"),
    "usb": ("rem0vable media", "thumb-stick vector", "portable storage stick"),
    "oauth consent": ("3rd-party-app grant", "delegated-grant abuse", "tenant grant abuse"),
    "social engineering": ("operator deception", "human-factor pretexting", "operator manipulation"),
    "social-engineering": ("operator deception", "human-factor pretexting", "operator manipulation"),
    "vpn login": ("corp tunnel sign-in", "remote-access portal entry", "tunnelled sign-in"),
    "new geography": ("unseen origin region", "anomalous origin", "first-time origin"),
    "compromised npm": ("trojanized js-libs release", "doctored js-libs build", "polluted js-libs feed"),
    "container image": ("workload bundle", "runtime workload artifact", "compute workload bundle"),
    "wire transfer initial": ("ach instruction", "outgoing remittance", "treasury debit instruction"),
    # TA0002 — Execution
    "execution": ("payload delivery step", "binary launch", "code activation"),
    "powershell": ("p0wer$hell", "p\u200bow\u200bersh\u200bell", "PoSh"),
    "cmd.exe": ("command interpreter binary", "cm\u200bd.exe", "shell-host.exe"),
    "script": ("scr1pt", "automation snippet", "in-line code blob"),
    "macro": ("m@cro", "off1ce-automation routine", "embedded automation routine"),
    "python": ("p\u200bython", "py interpreter", "py-runtime"),
    "node": ("n0de runtime", "n\u200bode.js", "j$-runtime daemon"),
    "certutil": ("cert\u200butil", "windows cert tool", "cert-handling lolbin"),
    "wmi": ("management infra channel", "WM\u200bI", "windows-mgmt-fabric"),
    "wmic": ("wm\u200bic", "mgmt-fabric cli", "windows-mgmt-cli"),
    "mshta": ("ms\u200bhta", "mshta lolbin alt", "trusted-host runner"),
    "process hollow": ("memory-cavity injection", "image substitution into target proc", "in-place process swap"),
    "process create": ("spawn syscall", "child-proc launch", "new-proc materialization"),
    "npm": ("n\u200bpm", "j$-pkg manager", "js dependency manager"),
    "post-install": ("install-time hook", "hook stage after pkg add", "pkg-install hook"),
    "container with": ("workload containing", "image carrying", "bundle containing"),
    "docker run": ("orchestrator launch", "runc launch", "containerd run-cmd"),
    "javascript": ("j$ runtime payload", "es-module payload", "browser-side code"),
    "stage-2 payload": ("second-tier dropper", "follow-on artifact", "next-tier blob"),
    "vba": ("v\u200bba", "off1ce-automation language", "doc-embedded autom"),
    "browser exploitation": ("client-side rendering abuse", "html-engine abuse", "DOM-engine abuse"),
    # TA0003 — Persistence
    "persistence": ("re-establish footing", "reboot survival", "long-term foothold"),
    "registry run": ("hkcu autostart key", "winreg autorun stanza", "reg-autostart entry"),
    "scheduled task": ("schtasks job", "taskscheduler entry", "win-periodic job"),
    "startup": ("st@rtup", "boot-time launch slot", "boot autoload"),
    "service": ("d@emon entry", "long-running unit", "background unit"),
    "implant": ("imp1ant", "long-residence agent", "resident dropper"),
    "backdoor": ("b@ckdoor", "covert callback link", "covert reverse access"),
    "rootkit": ("r00tkit", "kernel-resident hider", "stealth kernel module"),
    "firmware": ("f1rmware", "low-level board code", "below-os layer"),
    "uefi": ("ue\u200bfi", "early-boot f1rmware", "platform pre-boot f1rmware"),
    "cron": ("cr0n", "unix periodic job", "unix sched entry"),
    "wmi event": ("wm\u200bi event", "mgmt-fabric stream", "management-event stream"),
    "permanent wmi": ("perm wm\u200bi", "always-on mgmt stream", "indefinite mgmt stream"),
    "office add-in": ("off1ce plugin slot", "productivity-suite extension", "doc-suite extension"),
    "vsto": ("vs\u200bto", "off1ce add-in tooling", "off1ce plugin tooling"),
    "outlook startup": ("mailer load-time slot", "messaging-app boot slot", "mail-client boot slot"),
    # TA0004 — Privilege Escalation
    "privilege escalation": ("rights step-up", "rights uplift", "level-up of identity"),
    "escalation": ("rights step-up", "level-up", "rights uplift"),
    "escalate": ("step up rights", "gain higher rights", "uplift rights"),
    "container escape": ("workload sandbox break", "namespace breakout", "isolation-layer break"),
    "privileged pod": ("p0werful pod", "host-shared workload", "high-cap pod"),
    "bypass": ("circumvent", "go around", "side-step"),
    "elevation": ("rights step-up", "rights uplift", "perm step-up"),
    "sudo": ("su\u200bdo", "root-shim", "uplifted-shim"),
    "uac bypass": ("ua\u200bc circumvention", "consent-prompt sidestep", "security-prompt sidestep"),
    "uac": ("ua\u200bc", "consent prompt", "security prompt"),
    "fodhelper": ("f0dhelper", "trusted helper binary", "feature-on-demand helper"),
    "suid": ("su\u200bid", "setuser bit", "setid-perm bit"),
    "metadata service": ("im\u200bds endpoint", "m\u200betadata svc", "instance-info endpoint"),
    # TA0005 — Defense Evasion
    "defense evasion": ("av-d0dging", "control-skirting", "telemetry-d0dging"),
    "obfuscat": ("scr@mbl", "deob h@rd'n", "morph"),
    "fileless": ("on-disk-less", "no-disk-write", "ram-resident-only"),
    "memory-only": ("ram-resident", "in-mem-only", "no on-disk artifact"),
    "memory only": ("ram resident", "in mem only", "no on-disk artifact"),
    "hollowing": ("h0llowing", "memory-cavity swap", "in-place image swap"),
    "side-load": ("s1de-load", "hijack legit-binary lookup chain", "search-order load t@mpering"),
    "dll": ("d\u200bll", "dynamic library", "loadable code module"),
    "masquerad": ("disguis", "imitat", "sp00f"),
    "encode": ("base64-wrap", "byte-wrap", "transform"),
    "pack": ("p@ck", "c0mpress", "wr@p"),
    "log cleared": ("logs purged", "audit history erased", "telemetry erased"),
    "wevtutil": ("we\u200bvtutil", "win event tool", "win-log cli"),
    "journalctl": ("j0urnalctl", "systemd journal cli", "systemd log cli"),
    "vacuum": ("v@cuum", "c0mpact db", "c0mpress-truncate"),
    "indicator removal": ("evidence cleanup", "trace cleanup", "footprint cleanup"),
    "tampering": ("altering", "modify-on-the-fly", "d0ctoring"),
    "stop defender": ("disable mde-agent", "halt windows av", "kill av-agent"),
    "stop crowdstrike": ("disable cs-falcon", "halt edr-agent", "kill edr-sensor"),
    "stop sysmon": ("disable sysmon-agent", "halt sysmon-collector", "kill sysmon-svc"),
    "secure boot": ("s\u200becure boot", "verified boot chain", "boot integrity"),
    # TA0006 — Credential Access
    "credential": ("cred", "auth-token", "login secret"),
    "brute force": ("repeated guess", "iterative login attempt", "dictionary attack"),
    "brute-force": ("repeated-guess", "iterative-login", "dict-attempt"),
    "kerberoast": ("kerb-ticket abuse", "kerb-ticket roast", "kerb-ticket grind"),
    "dcsync": ("dc-replica-pull abuse", "directory replica abuse", "msrpc replica-pull abuse"),
    "lsass": ("l$ass", "auth subsystem proc", "session-mgmt proc"),
    "minidump": ("mini-d\u200bump", "process snapshot", "proc-snapshot"),
    "dump": ("d\u200bump", "snapshot of proc memory", "proc-mem snapshot"),
    "harvest": ("scrap", "skim", "lift"),
    "password reset": ("pw rotation request", "self-serve cred change", "cred rotation request"),
    "password spray": ("low-and-slow login attempt", "quiet cred probe", "spread-out login attempts"),
    "credential spray": ("cred low-and-slow", "auth-token probe spread", "spread-out cred probe"),
    "saml": ("s@ml", "federation token", "fed-assertion"),
    "oauth": ("0auth", "tenant grant", "delegated grant"),
    "ntlm": ("n\u200btlm", "ms-auth legacy", "challenge-response auth"),
    "mimikatz": ("m1m1katz", "secrets extractor", "auth-secrets puller"),
    "pat": ("p@t", "github-token", "ci-token"),
    "personal access token": ("dev-portal token", "self-issued bearer token", "user-issued api token"),
    "refresh token": ("re-fresh tok\u200ben", "long-lived auth handle", "session-extension token"),
    "imds": ("im\u200bds", "instance metadata svc", "instance-info endpoint"),
    "session token": ("ses\u200bsion token", "auth-cookie", "bearer for active session"),
    "stolen session": ("hijacked sess1on", "co-opted active sess", "session-takeover"),
    "tgs": ("tg\u200bs", "kerb svc ticket", "svc-ticket"),
    # TA0007 — Discovery
    "discovery": ("scout-stage", "look-around stage", "environment-mapping"),
    "enumerat": ("listing-out", "walking the directory", "iterating across"),
    "scan": ("sweep", "probe-pass", "fingerprint-pass"),
    "account discovery": ("user-listing", "principal-listing", "identity-listing"),
    "network share": ("smb mount", "remote folder", "fileshare mount"),
    "recon": ("rec0n", "look-around stage", "environment-mapping"),
    "replication": ("dir-sync abuse", "ad-sync abuse", "directory replica use"),
    "ldap": ("l\u200bdap", "directory protocol", "x.500 query"),
    "bloodhound": ("bl0odhound", "ad-graph mapper", "ad-relationship mapper"),
    "smb share": ("smb mount", "remote folder mount", "fileshare mount"),
    "share enum": ("mount listing", "fileshare listing", "smb-volume listing"),
    "system info": ("host fingerprinting", "host-detail pull", "machine-detail pull"),
    # TA0008 — Lateral Movement
    "lateral movement": ("east-west pivot", "horizontal hop", "side-to-side traversal"),
    "lateral": ("e-w hop", "horizontal", "side-to-side"),
    "rdp": ("r\u200bdp", "remote desktop sess", "ts/mstsc sess"),
    "pass-the-hash": ("hash-replay attack", "auth-hash replay", "secret-hash replay"),
    "pass the hash": ("hash replay attack", "auth hash replay", "secret hash replay"),
    "remote service": ("over-network svc invoke", "ms-rpc remote call", "rpc remote call"),
    "pivoting": ("p1voting", "pivot-host hop", "in-network hop"),
    "ssh": ("s\u200bsh", "secure shell", "openssh session"),
    "smb scan": ("smb sweep", "fileshare probe", "windows-share probe"),
    "wmic /node": ("wm\u200bic /machine", "mgmt-cli /remote", "mgmt-fabric cli /remote"),
    "wmi remote": ("wm\u200bi remote", "mgmt-fabric remote", "remote mgmt-fabric"),
    # TA0009 — Collection
    "collection": ("data-gathering stage", "skim stage", "asset-pull stage"),
    "data from local": ("data lifted from host", "local-host data pull", "on-host data pull"),
    "screenshot": ("screen-capture", "ui snapshot", "display-grab"),
    "keylog": ("keystroke-capture", "input-capture", "kb-monitor"),
    "clipboard": ("paste buffer", "copy-buffer", "clip-store"),
    "bulk download": ("mass pull", "batch fetch", "wholesale fetch"),
    "pii": ("p\u200bii", "personal data", "user identity data"),
    "customer record": ("customer row", "client account row", "subscriber record"),
    "mailbox export": ("inbox extract", "mailstore export", "messaging-store export"),
    "pst": ("p\u200bst", "outlook archive", "mail archive"),
    "private repo": ("internal source repo", "non-public repo", "closed repo"),
    "private repos": ("internal source repos", "non-public repos", "closed repos"),
    # TA0010 — Exfiltration
    "exfiltrat": ("ex-bound transfer", "outbound siphon", "ex-bound siphon"),
    "c2 channel": ("operator channel", "comms-back tunnel", "callback channel"),
    "cloud storage": ("object storage uplink", "blob-store push target", "external bucket"),
    "dns tunnel": ("dns covert tunnel", "udp/53 covert track", "dns covert track"),
    "ftp": ("f\u200btp", "file-xfer protocol", "legacy file-transfer"),
    "upload": ("up\u200bload", "outbound transfer", "push-out"),
    "egress": ("outbound flow", "ex-bound flow", "outward traffic"),
    "google drive": ("g\u200bdrive", "drive-shared bucket", "consumer-cloud drive"),
    "drive upload": ("drive push", "drive transfer", "drive sync-up"),
    "personal drive": ("personal cloud bucket", "consumer cloud bucket", "non-corp drive"),
    "transfer to": ("xfer to", "push to", "ship to"),
    "s3 bucket": ("object-store bucket", "blob-store bucket", "aws object store"),
    "s3://": ("aws://obj/", "object-store-uri", "blob-uri"),
    "data egress": ("outbound data flow", "ex-bound data flow", "data outflow"),
    # TA0011 — Command and Control
    "command and control": ("operator-channel", "callback infra", "ops-channel"),
    "c2": ("c\u200b2", "ops channel", "callback infra"),
    "beacon": ("b\u200beacon", "callback ping", "regular check-in"),
    "c&c": ("c\u200b&c", "ops channel", "callback infra"),
    "dga": ("d\u200bga", "domain-gen algorithm", "algorithmic domains"),
    "domain generation": ("algorithmic domain rotation", "rotating-domain algo", "domain-rotation algo"),
    "dns query": ("d\u200bns query", "name-resolution", "name lookup"),
    "covert channel": ("covert-comms link", "stealthy backchannel", "hidden link"),
    "cobalt strike": ("c0balt strike", "crimson-team toolkit", "red-team-grade dropper"),
    "https beacon": ("tls callback", "https callback", "tls heartbeat"),
    "ja3": ("j@3", "tls fingerprint hash", "tls-handshake fingerprint"),
    # TA0040 — Impact
    "impact": ("damage stage", "harm stage", "ko stage"),
    "ransom": ("ext0rt", "lockout-for-pay", "data-lockout demand"),
    "encrypt": ("scr@mble", "lock-with-key", "cipher-lock"),
    "wipe": ("zero-out", "shred", "scrub"),
    "destroy": ("annihilate", "deletion-pass", "purge"),
    "disrupt": ("d1srupt", "knock-offline", "make-unavailable"),
    "defacement": ("page-graffiti", "site replacement", "homepage swap"),
    "miner": ("c0in-miner", "compute-hijacking process", "cycles-stealing process"),
    "cryptomine": ("c0in-mining", "compute hijack for currency", "cycles for currency"),
    "xmrig": ("xm\u200brig", "monero-grade c0in-tool", "c0in-tool suite"),
    "ddos": ("d\u200bdos", "flood attack", "volumetric flood"),
    "syn flood": ("tcp-half-open flood", "tcp synflood", "tcp half-open swarm"),
    "wire transfer": ("ach payment instruction", "outgoing remittance", "treasury debit instruction"),
    "$250": ("a quarter-million-dollar", "USD250000", "two-fifty-K"),
    "bec": ("b\u200bec", "exec-impersonation fraud", "boss-impersonation fraud"),
}


# Light obfuscation only — leetspeak, used for the "light" intensity bucket
# so the defender SHOULD still catch most cases there.
_LIGHT_LEET: dict[str, str] = {
    "phishing": "phi$hing",
    "powershell": "powershell",  # untouched — light tier
    "execution": "execution",  # untouched — light tier
    "credential": "credentia1",
    "exfiltrat": "exfiltrat",  # untouched — light tier
}


# ---------------------------------------------------------------------------
# Grammar validation — refuse to ship if a synonym would itself match a
# defender keyword for a tactic. This means "graceful degradation" is the
# only failure mode; we never accidentally trip a different tactic.
# ---------------------------------------------------------------------------


def _validate_grammar() -> None:
    """Make sure no synonym alternate contains a defender keyword."""
    flat_keywords: list[str] = []
    for kws in _TACTIC_KEYWORDS.values():
        flat_keywords.extend(k.lower() for k in kws)

    failures: list[str] = []
    for src, alternates in _SYNONYMS.items():
        for alt in alternates:
            low = alt.lower()
            for kw in flat_keywords:
                # Same keyword self-match is fine (it's the original word
                # we're rewriting), but any *other* keyword landing inside
                # the alternate is a leak.
                if kw == src.lower():
                    continue
                if kw in low:
                    failures.append(f"alternate '{alt}' for '{src}' still contains keyword '{kw}'")
    if failures:
        msg = "\n  - ".join(failures[:20])
        raise RuntimeError(f"Adversary mutator grammar leaks defender keywords:\n  - {msg}")


_validate_grammar()


# ---------------------------------------------------------------------------
# Mutation engine
# ---------------------------------------------------------------------------


# Ordered by descending length so multi-word keys are matched before their
# single-word substrings (e.g. "spear-phish" before "spear", "wmi event"
# before "wmi"). This is critical for correctness.
_KEY_ORDER: list[str] = sorted(_SYNONYMS.keys(), key=len, reverse=True)

# Pre-compile case-insensitive regexes per key. We escape the key directly
# (no word boundaries — many keys contain punctuation like "cmd.exe" or
# "s3://" where word boundaries fight us).
_KEY_REGEX: dict[str, re.Pattern[str]] = {k: re.compile(re.escape(k), re.IGNORECASE) for k in _KEY_ORDER}


def _bucket(incident_id: str) -> str:
    """Pick a mutation intensity bucket deterministically per incident.

    Distribution targets:
        heavy   ≈ 45%   — every keyword swapped for both expected tactics
        medium  ≈ 35%   — one tactic preserved cleanly, others mutated
        light   ≈ 20%   — light leetspeak; defender should still catch

    The 45/35/20 split lands the eval near the ~0.55 graceful-degradation
    floor with comfortable headroom for either side.
    """
    h = int(hashlib.sha256(incident_id.encode()).hexdigest()[:8], 16)
    bucket_pct = h % 100
    if bucket_pct < 45:
        return "heavy"
    if bucket_pct < 80:
        return "medium"
    return "light"


def _pick_alternate(key: str, incident_id: str) -> str:
    """Pick one of the deterministic alternates for `key`."""
    alts = _SYNONYMS[key]
    h = int(hashlib.sha256(f"{incident_id}|{key}".encode()).hexdigest()[:8], 16)
    return alts[h % len(alts)]


def _mutate_text_full(text: str, incident_id: str) -> str:
    """Replace every keyword in `text` with its deterministic alternate."""
    out = text
    for key in _KEY_ORDER:
        pattern = _KEY_REGEX[key]
        if pattern.search(out):
            alt = _pick_alternate(key, incident_id)
            out = pattern.sub(alt, out)
    return out


def _mutate_text_partial(text: str, incident_id: str, preserve_tactics: set[str]) -> str:
    """Mutate keywords EXCEPT those tied to one preserved tactic.

    The defender still has at least one route to detect the incident.
    """
    preserve_keywords = {kw.lower() for tactic in preserve_tactics for kw in _TACTIC_KEYWORDS.get(tactic, [])}
    out = text
    for key in _KEY_ORDER:
        if key.lower() in preserve_keywords:
            continue
        pattern = _KEY_REGEX[key]
        if pattern.search(out):
            alt = _pick_alternate(key, incident_id)
            out = pattern.sub(alt, out)
    return out


def _mutate_text_light(text: str) -> str:
    """Apply light leetspeak that should NOT defeat the defender.

    Used as a control bucket — confirms the defender can still extract
    tactics under low-grade obfuscation, so a regression on the heavy
    bucket can't be masked by the light tier silently failing too.
    """
    out = text
    for src, dst in _LIGHT_LEET.items():
        out = re.sub(re.escape(src), dst, out, flags=re.IGNORECASE)
    return out


def mutate_incident(incident: dict[str, Any]) -> dict[str, Any]:
    """Return an adversarially-mutated copy of `incident`."""
    incident_id = incident["id"]
    bucket = _bucket(incident_id)
    expected_tactics: list[str] = list(incident.get("expected_tactics", []))

    if bucket == "heavy":
        new_title = _mutate_text_full(incident["title"], incident_id)
        new_description = _mutate_text_full(incident["description"], incident_id)
    elif bucket == "medium":
        if expected_tactics:
            # Preserve the LAST expected tactic deterministically.
            preserve_idx = int(hashlib.sha256(incident_id.encode()).hexdigest()[8:16], 16) % len(expected_tactics)
            preserve = {expected_tactics[preserve_idx]}
        else:
            preserve = set()
        new_title = _mutate_text_partial(incident["title"], incident_id, preserve)
        new_description = _mutate_text_partial(incident["description"], incident_id, preserve)
    else:
        new_title = _mutate_text_light(incident["title"])
        new_description = _mutate_text_light(incident["description"])

    return {
        # Keep the same id space so the eval suite can correlate with the
        # base dataset — that lets us do "which template lost coverage
        # under adversary?" diffs.
        "id": incident_id,
        "template_id": incident.get("template_id"),
        "template_index": incident.get("template_index"),
        "adversary_intensity": bucket,
        "title": new_title,
        "description": new_description,
        "expected_tactics": expected_tactics,
        "expected_techniques": list(incident.get("expected_techniques", [])),
        "severity": incident.get("severity"),
        "response_class": incident.get("response_class"),
        "evidence_keywords": list(incident.get("evidence_keywords", [])),
        "original_title": incident["title"],
        "original_description": incident["description"],
    }


def mutate_dataset(incidents: list[dict[str, Any]]) -> list[dict[str, Any]]:
    return [mutate_incident(inc) for inc in incidents]


# ---------------------------------------------------------------------------
# Reporting helpers (used by --report and by the eval suite)
# ---------------------------------------------------------------------------


def adversary_coverage_report(
    mutated: list[dict[str, Any]],
) -> dict[str, Any]:
    """Summarise how many incidents lost detectable tactics under mutation.

    The defender's keyword extractor is run BOTH on the original and on the
    mutated description so we can quantify graceful-degradation.
    """
    bucket_counts = {"heavy": 0, "medium": 0, "light": 0}
    correct_total = 0
    lost_all_total = 0
    per_tactic_lost: dict[str, int] = {}
    per_template_correct: dict[str, dict[str, int]] = {}

    for inc in mutated:
        bucket_counts[inc["adversary_intensity"]] += 1
        expected = set(inc["expected_tactics"])
        text = f"{inc['title']}\n{inc['description']}"
        predicted = extract_tactics_from_text(text)
        overlap = predicted & expected
        if overlap:
            correct_total += 1
        else:
            lost_all_total += 1

        for t in expected:
            if t not in predicted:
                per_tactic_lost[t] = per_tactic_lost.get(t, 0) + 1

        tpl = inc.get("template_id") or "<unknown>"
        slot = per_template_correct.setdefault(tpl, {"correct": 0, "total": 0})
        slot["total"] += 1
        if overlap:
            slot["correct"] += 1

    n = len(mutated)
    return {
        "incidents": n,
        "buckets": bucket_counts,
        "defender_correct": correct_total,
        "defender_lost_all_tactics": lost_all_total,
        "defender_accuracy": round(correct_total / n, 4) if n else 0.0,
        "per_tactic_lost": dict(sorted(per_tactic_lost.items())),
        "per_template": {
            t: {
                "correct": v["correct"],
                "total": v["total"],
                "accuracy": round(v["correct"] / v["total"], 4) if v["total"] else 0.0,
            }
            for t, v in sorted(per_template_correct.items())
        },
    }


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def main() -> None:
    parser = argparse.ArgumentParser(description="Generate the adversary-mutated incident dataset (Wave 2 — w2-aivai).")
    parser.add_argument(
        "--input",
        type=Path,
        default=_DEFAULT_INPUT,
        help="Path to base synthetic_incidents.json (default: services/agents/tests/eval_data/synthetic_incidents.json)",
    )
    parser.add_argument(
        "--out",
        type=Path,
        default=_DEFAULT_OUTPUT,
        help="Path to write adversary_incidents.json",
    )
    parser.add_argument(
        "--report",
        action="store_true",
        help="Print a coverage / graceful-degradation report alongside writing the file.",
    )
    args = parser.parse_args()

    if not args.input.exists():
        sys.exit(f"input not found: {args.input}. Run scripts/generate_eval_incidents.py first.")

    base = json.loads(args.input.read_text())
    mutated = mutate_dataset(base)

    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(json.dumps(mutated, indent=2) + "\n")

    if args.report:
        report = adversary_coverage_report(mutated)
        print()
        print("=" * 78)
        print("  AiSOC AI-vs-AI adversary dataset — graceful-degradation report")
        print("=" * 78)
        print(f"  Incidents:                {report['incidents']}")
        print(
            f"  Bucket split:             "
            f"heavy={report['buckets']['heavy']}, "
            f"medium={report['buckets']['medium']}, "
            f"light={report['buckets']['light']}"
        )
        print(f"  Defender catch rate:      {report['defender_correct']}/{report['incidents']} = {report['defender_accuracy'] * 100:.1f}%")
        print(f"  Defender lost-everything: {report['defender_lost_all_tactics']} incidents")
        print()
        print("  Tactics most-lost under mutation:")
        for t, n in sorted(report["per_tactic_lost"].items(), key=lambda kv: kv[1], reverse=True)[:8]:
            print(f"    {t}: -{n}")
        print("=" * 78)
    print(f"wrote {len(mutated)} mutated incidents to {args.out}")


if __name__ == "__main__":
    main()