Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
#!/usr/bin/env python3
"""Storage $/TB cost model (Phase 6 — cost).
AiSOC stores security telemetry in tiers: a hot ClickHouse tier on block
storage for recent, queryable events; a warm object tier; and a cold/archive
object tier for compliance retention. This model turns that tiering into a
defensible $/TB number so the storage-consolidation decision
(`docs/decisions/0005-storage-consolidation.md`) rests on arithmetic, not vibes.
**The value here is the methodology, not the exact prices.** The rate card
below is *reference list pricing* (2026, single representative region) and MUST
be verified against your provider, region, and negotiated rates. The model is
deterministic, so `--check` gates the committed worked example
(`docs/decisions/storage-cost-model.json`) — if the rate card or scenario
changes, the JSON must be regenerated, and the ADR cites that file.
Usage:
python3 scripts/storage_cost_model.py # print the worked example
python3 scripts/storage_cost_model.py --json # JSON to stdout
python3 scripts/storage_cost_model.py --check # fail on drift vs committed JSON
"""
from __future__ import annotations
import argparse
import ast
import contextlib
import json
import sys
from pathlib import Path
# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from gate_toolkit import repo_root, self_test_if_requested
self_test_if_requested(__file__)
ROOT = repo_root()
COMMITTED = ROOT / "docs" / "decisions" / "storage-cost-model.json"
# Reference list prices, USD per GB-month. VERIFY against your provider/region.
RATE_CARD_USD_PER_GB_MONTH = {
"hot_block": 0.080, # gp3-class block storage backing ClickHouse hot tier
"warm_object": 0.023, # standard object storage
"cold_archive": 0.0125, # infrequent-access / archive object storage
}
# Canonical worked example. Security telemetry compresses well under ClickHouse
# ZSTD; 8x is a conservative documented assumption for JSON-shaped logs.
SCENARIO = {
"raw_tb_per_day": 1.0,
"compression_ratio": 8.0,
"retention_days": {"hot_block": 30, "warm_object": 60, "cold_archive": 275}, # 365 total
}
_GB_PER_TB = 1000
def compute(scenario: dict, rate_card: dict) -> dict:
raw_gb_per_day = scenario["raw_tb_per_day"] * _GB_PER_TB
ratio = scenario["compression_ratio"]
stored_gb_per_day = raw_gb_per_day / ratio
tiers: dict[str, dict] = {}
total_monthly = 0.0
for tier, days in scenario["retention_days"].items():
# Steady-state stored volume for this tier = days of retention worth of
# compressed data resident in the tier at any moment.
resident_gb = stored_gb_per_day * days
monthly = resident_gb * rate_card[tier]
total_monthly += monthly
tiers[tier] = {
"retention_days": days,
"resident_gb": round(resident_gb, 2),
"rate_usd_per_gb_month": rate_card[tier],
"monthly_usd": round(monthly, 2),
}
raw_tb_ingested_per_month = scenario["raw_tb_per_day"] * 30
usd_per_raw_tb = total_monthly / raw_tb_ingested_per_month if raw_tb_ingested_per_month else 0.0
return {
"scenario": scenario,
"rate_card_usd_per_gb_month": rate_card,
"tiers": tiers,
"total_monthly_usd": round(total_monthly, 2),
"raw_tb_ingested_per_month": round(raw_tb_ingested_per_month, 2),
"usd_per_raw_tb_ingested": round(usd_per_raw_tb, 2),
"disclaimer": "reference list prices; verify against your provider/region and negotiated rates",
}
def _print_table(result: dict) -> None:
s = result["scenario"]
print("Storage cost model — worked example")
print(f" raw ingest: {s['raw_tb_per_day']} TB/day")
print(f" compression: {s['compression_ratio']}x")
print()
print(f" {'tier':<14}{'days':>6}{'resident_GB':>14}{'$/GB-mo':>10}{'$/mo':>12}")
for tier, t in result["tiers"].items():
print(f" {tier:<14}{t['retention_days']:>6}{t['resident_gb']:>14}{t['rate_usd_per_gb_month']:>10}{t['monthly_usd']:>12}")
print()
print(f" total monthly: ${result['total_monthly_usd']}")
print(f" $ / raw TB ingested: ${result['usd_per_raw_tb_ingested']}")
print(f" ({result['disclaimer']})")
#: The API service's copy of this model. It cannot import this module — the
#: container has neither ``scripts/`` nor a git checkout, and ``repo_root()``
#: above shells out to git at import — so the constants are mirrored there,
#: the same constraint that put a second LLM price table in
#: ``cost_dashboard._PUBLIC_PRICING``. A second copy of a number is a second
#: place for it to drift, so ``--check`` reads it back. Without this, 6b would
#: have closed "the model has no consumer" by creating a consumer that could
#: silently start quoting different prices.
CONSUMER = "services/api/app/services/storage_cost.py"
_CONSUMER_CONSTANTS = {
"RATE_CARD_USD_PER_GB_MONTH": lambda: RATE_CARD_USD_PER_GB_MONTH,
"RETENTION_DAYS": lambda: SCENARIO["retention_days"],
"COMPRESSION_RATIO": lambda: SCENARIO["compression_ratio"],
}
def check_consumer() -> list[str]:
"""Read the API's mirrored constants back and report any disagreement.
Parsed rather than imported, for the same reason the API cannot import
this file: the gate runs in a dep-light lint job and must not pull in
FastAPI, pydantic and the ClickHouse driver to compare three literals.
"""
path = ROOT / CONSUMER
if not path.exists():
return [f"{CONSUMER} is missing — the model's only consumer is gone, so this gate now checks a constant nothing reads"]
tree = ast.parse(path.read_text(encoding="utf-8"))
found: dict[str, object] = {}
for node in tree.body:
target: str | None = None
value: ast.expr | None = None
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
target, value = node.targets[0].id, node.value
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
target, value = node.target.id, node.value
if target in _CONSUMER_CONSTANTS and value is not None:
with contextlib.suppress(ValueError):
found[target] = ast.literal_eval(value)
problems: list[str] = []
for name, expected in _CONSUMER_CONSTANTS.items():
if name not in found:
problems.append(f"{CONSUMER} no longer declares {name}; the mirror cannot be checked, so it is not a mirror")
elif found[name] != expected():
problems.append(
f"{CONSUMER}:{name} is {found[name]!r}, this model says {expected()!r} — "
"the console would quote a price this model does not"
)
return problems
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--json", dest="as_json", action="store_true")
parser.add_argument("--check", action="store_true", help="fail if committed JSON or the API's mirrored constants are stale")
args = parser.parse_args()
result = compute(SCENARIO, RATE_CARD_USD_PER_GB_MONTH)
if args.check:
if not COMMITTED.exists():
print(f"ERROR: {COMMITTED.relative_to(ROOT)} missing — run scripts/storage_cost_model.py --write", file=sys.stderr)
return 1
committed = json.loads(COMMITTED.read_text(encoding="utf-8"))
if committed != result:
print(
f"ERROR: {COMMITTED.relative_to(ROOT)} is stale. The rate card or scenario changed but the "
"committed worked example was not regenerated. Run: python3 scripts/storage_cost_model.py --write",
file=sys.stderr,
)
return 1
drift = check_consumer()
if drift:
print(f"ERROR: the model and {CONSUMER} disagree:", file=sys.stderr)
for problem in drift:
print(f" - {problem}", file=sys.stderr)
return 1
print(f"OK: storage cost model current — ${result['usd_per_raw_tb_ingested']}/raw-TB, and {CONSUMER} mirrors it")
return 0
if args.as_json:
print(json.dumps(result, indent=2, sort_keys=True))
return 0
_print_table(result)
return 0
if __name__ == "__main__":
# Hidden --write to regenerate the committed JSON (used by maintainers).
if "--write" in sys.argv:
payload = compute(SCENARIO, RATE_CARD_USD_PER_GB_MONTH)
COMMITTED.parent.mkdir(parents=True, exist_ok=True)
COMMITTED.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"wrote {COMMITTED.relative_to(ROOT)}")
raise SystemExit(0)
raise SystemExit(main())