Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

8.5 KB
Raw Blame
{
 "baselines_note": "Four classes, so a constant answer scores 0.250 balanced accuracy and 0.500 plain accuracy. Both are stated beside the table because a score is only readable against the thing it has to beat.",
 "commit": "f92e8bcd",
 "comparability": [
  "All five rows graded the same corpus digest through the same production triage path, 72 model calls each, zero provider errors.",
  "Latency is NOT comparable across rows and is excluded from the decision: the 3B ran CPU-bound inside the stack container and the four 7-8B rows ran GPU-accelerated on the host. Kept per row so the asymmetry is stated rather than hidden.",
  "resolved_models came back empty on every decision of every run, so per-decision model attribution is unverified even though the calls were placed and answered. The count of calls is measured; the attribution is not.",
  "auto_close_precision rests on 7 to 37 candidates depending on the row. Those are small denominators and the column should be read as a warning sign, not as a ranking."
 ],
 "corpus": {
  "classes": {
   "benign": 18,
   "benign_true_positive": 10,
   "false_positive": 8,
   "true_positive": 36
  },
  "constant_answer_ceiling": 0.25,
  "digest": "sha256:ec60d7bddabcc5b28d8942cea52d770bd8a2d13b93396826a6deab1dcba6aa86",
  "items": 72,
  "majority_class_plain_accuracy": 0.5,
  "name": "verdict_corpus_v1",
  "pairs": 36
 },
 "default_model": "llama3.2:3b-instruct-q4_K_M (CPU) / llama3.1:8b-instruct-q4_K_M (GPU, host-LLM)",
 "default_rationale": "Chosen strictly by balanced accuracy, the metric this corpus was built to make meaningful. llama3.1:8b-instruct-q4_K_M leads at 0.357 against the 3B's 0.285 and a 0.250 constant-answer floor, and also leads on MCC (0.209), malicious precision (0.724) and calibration (ECE 0.349, the only row below 0.40). It therefore becomes the default on the GPU and host-LLM overlays. The 3B stays the default for CPU `make up`, where a ~5 GB model does not fit the 8 GB CORE budget -- a deployment constraint, not a score. Two findings outrank the ranking. First, every model measured is far closer to chance than to useful: the best balanced accuracy is 0.357 against a 0.250 floor, and three of five score below the 0.500 plain accuracy of always guessing the majority class. Second, auto-close precision is poor across the board -- 0.571 on seven candidates at best, 0.091 to 0.243 elsewhere -- so nothing measured here supports autonomous closure, and that is a finding about the auto-close gate rather than about which model to pick.",
 "entries": [
  {
   "abstention_rate": 0.0,
   "auto_close_considered": 7,
   "auto_close_precision": 0.5714285714285714,
   "balanced_accuracy": 0.28541666666666665,
   "corpus_digest": "sha256:ec60d7bddabcc5b28d8942cea52d770bd8a2d13b93396826a6deab1dcba6aa86",
   "corpus_items": 72,
   "decisions_errored": 0,
   "expected_calibration_error": 0.441667,
   "false_negatives": 27,
   "hallucination_rate": 0.0,
   "headline_accuracy": 0.3194444444444444,
   "llm_calls_placed": 72,
   "malicious_precision": 0.5,
   "malicious_recall": 0.25,
   "malicious_support": 36,
   "matthews_corrcoef": 0.046864498582862235,
   "measured_at": "2026-10-10",
   "mode": "live",
   "model": "llama3.2:3b-instruct-q4_K_M",
   "note": "Retained for CPU `make up`: a ~5 GB 8B does not fit the 8 GB CORE budget alongside the rest of the stack.",
   "role": "shipped default (CPU profile)",
   "served_on": "CPU, in-stack container",
   "time_to_verdict_p50_ms": 11188.0,
   "total_tokens": 70983
  },
  {
   "abstention_rate": 0.0,
   "auto_close_considered": 9,
   "auto_close_precision": 0.1111111111111111,
   "balanced_accuracy": 0.35694444444444445,
   "corpus_digest": "sha256:ec60d7bddabcc5b28d8942cea52d770bd8a2d13b93396826a6deab1dcba6aa86",
   "corpus_items": 72,
   "decisions_errored": 0,
   "expected_calibration_error": 0.348611,
   "false_negatives": 15,
   "hallucination_rate": 0.0,
   "headline_accuracy": 0.4583333333333333,
   "llm_calls_placed": 72,
   "malicious_precision": 0.7241379310344828,
   "malicious_recall": 0.5833333333333334,
   "malicious_support": 36,
   "matthews_corrcoef": 0.20863951686663296,
   "measured_at": "2026-10-11",
   "mode": "live",
   "model": "llama3.1:8b-instruct-q4_K_M",
   "note": "Best balanced accuracy, MCC, malicious precision and calibration of everything measured.",
   "role": "winner (GPU / host-LLM profiles)",
   "served_on": "GPU, host Ollama",
   "time_to_verdict_p50_ms": 1698.0,
   "total_tokens": 74495
  },
  {
   "abstention_rate": 0.0,
   "auto_close_considered": 11,
   "auto_close_precision": 0.09090909090909091,
   "balanced_accuracy": 0.2791666666666667,
   "corpus_digest": "sha256:ec60d7bddabcc5b28d8942cea52d770bd8a2d13b93396826a6deab1dcba6aa86",
   "corpus_items": 72,
   "decisions_errored": 0,
   "expected_calibration_error": 0.420833,
   "false_negatives": 13,
   "hallucination_rate": 0.0,
   "headline_accuracy": 0.4166666666666667,
   "llm_calls_placed": 72,
   "malicious_precision": 0.6571428571428571,
   "malicious_recall": 0.6388888888888888,
   "malicious_support": 36,
   "matthews_corrcoef": 0.11329558957744541,
   "measured_at": "2026-10-10",
   "mode": "live",
   "model": "llama3:8b",
   "note": "Previous-generation llama3. Superseded by llama3.1:8b on every metric except malicious recall, where they tie.",
   "role": "candidate",
   "served_on": "GPU, host Ollama",
   "time_to_verdict_p50_ms": 1546.0,
   "total_tokens": 74808
  },
  {
   "abstention_rate": 0.09722222222222222,
   "auto_close_considered": 37,
   "auto_close_precision": 0.24324324324324326,
   "balanced_accuracy": 0.25972222222222224,
   "corpus_digest": "sha256:ec60d7bddabcc5b28d8942cea52d770bd8a2d13b93396826a6deab1dcba6aa86",
   "corpus_items": 72,
   "decisions_errored": 0,
   "expected_calibration_error": 0.593846,
   "false_negatives": 33,
   "hallucination_rate": 0.0,
   "headline_accuracy": 0.26153846153846155,
   "llm_calls_placed": 72,
   "malicious_precision": 0.75,
   "malicious_recall": 0.08333333333333333,
   "malicious_support": 36,
   "matthews_corrcoef": 0.03241459519549664,
   "measured_at": "2026-10-11",
   "mode": "live",
   "model": "qwen2.5:7b-instruct-q4_K_M",
   "note": "Worst measured: catches 3 of 36 malicious cases and is the least calibrated of the five.",
   "role": "candidate",
   "served_on": "GPU, host Ollama",
   "time_to_verdict_p50_ms": 1567.0,
   "total_tokens": 77044
  },
  {
   "abstention_rate": 0.3055555555555556,
   "auto_close_considered": 14,
   "auto_close_precision": 0.14285714285714285,
   "balanced_accuracy": 0.2097222222222222,
   "corpus_digest": "sha256:ec60d7bddabcc5b28d8942cea52d770bd8a2d13b93396826a6deab1dcba6aa86",
   "corpus_items": 72,
   "decisions_errored": 0,
   "expected_calibration_error": 0.366,
   "false_negatives": 13,
   "hallucination_rate": 0.0,
   "headline_accuracy": 0.5,
   "llm_calls_placed": 72,
   "malicious_precision": 0.6571428571428571,
   "malicious_recall": 0.6388888888888888,
   "malicious_support": 36,
   "matthews_corrcoef": 0.20247984364137678,
   "measured_at": "2026-10-11",
   "mode": "live",
   "model": "mistral:7b-instruct-v0.3-q4_K_M",
   "note": "Highest plain accuracy (0.500) and lowest balanced accuracy (0.210): it abstains on 31% of items, which flatters plain accuracy and is exactly what balanced accuracy exists to see through.",
   "role": "candidate",
   "served_on": "GPU, host Ollama",
   "time_to_verdict_p50_ms": 2210.0,
   "total_tokens": 93626
  },
  {
   "abstention_rate": 0.0,
   "auto_close_considered": 0,
   "auto_close_precision": null,
   "balanced_accuracy": 0.25,
   "hallucination_rate": null,
   "llm_calls_placed": 0,
   "majority_class_accuracy": 0.5,
   "malicious_recall": 0.0,
   "malicious_support": 36,
   "matthews_corrcoef": 0.0,
   "measured_at": "2026-10-31",
   "mode": "deterministic",
   "model": "deterministic fallback (no model)",
   "note": "The rules baseline the models must beat, measured through the same replay path (scripts/run_verdict_eval.py --deterministic). On this corpus the deterministic scorer answers benign to every item: balanced accuracy lands exactly on the 0.250 constant-answer floor, Matthews correlation is 0.0, and all 36 malicious cases are missed. Every alert it fails to close still routes to a human, which is the posture that makes this floor survivable.",
   "provenance_status": "no_calls"
  },
  {
   "measured": false,
   "model": "any hosted provider (GPT-4o class)",
   "not_measured_reason": "no funded key exists in this repository, and no hosted provider has ever been exercised"
  }
 ],
 "generated_from": "scripts/run_verdict_eval.py",
 "schema_version": "1.0"
}