Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""Tests for WS-H1 — cost dashboard builder.
The cost dashboard splits into a pure ``build_dashboard_from_rows`` and a
thin async DB orchestrator. We exercise the pure builder against
deterministic input rows so the regression surface is the actual aggregation
logic and not Postgres / SQLAlchemy plumbing.
The tests deliberately cover:
* daily bucketing across UTC midnight (timezone correctness)
* per-model aggregation, including models with no public list price, which
impute nothing rather than falling back to a default rate
* top-cost case ranking with ties broken deterministically by ``case_id``
* action-count ordering (most-common first) and the limit clamp
* BYOK savings semantics:
- hosted provider with no recorded cost → zero or modest delta
- local provider → full imputed cost reported as savings
* headline averages on empty input (no division by zero)
* window clamping inside the orchestrator boundary helper
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime, timedelta
import pytest
from app.services.cost_dashboard import (
AuditRow,
CostRow,
DashboardInputs,
LlmContext,
_impute_public_cost,
_internal_helpers,
build_dashboard_from_rows,
)
PERIOD_START = datetime(2026, 5, 2, 0, 0, tzinfo=UTC)
PERIOD_END = datetime(2026, 5, 9, 0, 0, tzinfo=UTC)
TENANT_ID = uuid.uuid4()
def _cost(
*,
run_id: str | None = None,
case_id: str = "CASE-1",
model: str = "gpt-4o-mini",
prompt_tokens: int = 1000,
completion_tokens: int = 500,
cost_usd: float | None = None,
latency_ms: float = 250.0,
call_count: int = 1,
started_at: datetime | None = None,
measured_call_count: int | None = None,
estimated_cost_usd: float = 0.0,
estimated_call_count: int = 0,
unpriced_call_count: int = 0,
resolved_model: str | None = None,
) -> CostRow:
"""Builder that fills sensible defaults for a CostRow.
When ``cost_usd`` is omitted we fall back to the imputed public cost so a
test that wants "what a hosted call would have charged" can drop into
the constructor without recomputing pricing by hand.
``measured_call_count`` defaults to ``call_count``, i.e. these rows stand
for gateway-measured spend. Pass ``0`` to build the row shape that every
pre-migration-063 row has: a cost figure with nothing vouching for it.
"""
if cost_usd is None:
cost_usd = _impute_public_cost(model, prompt_tokens, completion_tokens) or 0.0
return CostRow(
run_id=run_id or str(uuid.uuid4()),
case_id=case_id,
model=model,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
measured_cost_usd=cost_usd,
latency_ms=latency_ms,
call_count=call_count,
started_at=started_at or PERIOD_START + timedelta(hours=1),
measured_call_count=(call_count if measured_call_count is None else measured_call_count),
estimated_cost_usd=estimated_cost_usd,
estimated_call_count=estimated_call_count,
unpriced_call_count=unpriced_call_count,
resolved_model=resolved_model,
)
# ---------------------------------------------------------------------------
# Pricing / imputation
# ---------------------------------------------------------------------------
def test_impute_public_cost_known_model() -> None:
# gpt-4o-mini: $0.00015 / 1k input, $0.0006 / 1k output
cost = _impute_public_cost("gpt-4o-mini", 1000, 500)
assert cost == pytest.approx(0.00015 + 0.0003, rel=1e-9)
def test_impute_public_cost_unknown_model_imputes_nothing() -> None:
"""An unknown model has no price, and a default rate is not one.
This test used to assert the opposite — ``cost > 0`` — on the reasoning
that a default was "conservative" and stopped us reporting 100% savings
on a bespoke local model. The reasoning had it backwards. The agents
service records the ``aisoc-<role>`` alias it *requested*, and an alias
is in no price table, so the default applied to essentially every row:
the "imputed list price" was a made-up rate for a made-up model, and a
903-token local completion booked $0.000999 of spend that never existed.
``None`` is the honest answer and the caller counts the row as unpriced.
"""
assert _impute_public_cost("super-secret-llm", 1000, 1000) is None
assert _impute_public_cost("aisoc-triage", 1_000_000, 1_000_000) is None
assert _impute_public_cost("ollama/qwen2:1.5b", 1_000_000, 1_000_000) is None
def test_impute_public_cost_strips_a_provider_prefix() -> None:
"""The gateway names a resolved model ``openai/gpt-4o-mini``."""
assert _impute_public_cost("openai/gpt-4o-mini", 1000, 500) == pytest.approx(_impute_public_cost("gpt-4o-mini", 1000, 500))
def test_impute_public_cost_negative_token_counts_clamped() -> None:
# Defensive: a malformed row with negative tokens shouldn't produce a
# negative cost (which would fold the totals).
assert _impute_public_cost("gpt-4o-mini", -5, -10) == 0.0
def test_impute_public_cost_case_insensitive() -> None:
a = _impute_public_cost("GPT-4o", 1000, 500)
b = _impute_public_cost("gpt-4o", 1000, 500)
assert a == b
# ---------------------------------------------------------------------------
# Pure helpers
# ---------------------------------------------------------------------------
def test_format_period_label_same_month() -> None:
fmt = _internal_helpers["_format_period_label"]
label = fmt(PERIOD_START, PERIOD_END)
assert "May" in label
assert "2026" in label
def test_format_period_label_cross_year() -> None:
fmt = _internal_helpers["_format_period_label"]
label = fmt(
datetime(2025, 12, 28, tzinfo=UTC),
datetime(2026, 1, 4, tzinfo=UTC),
)
assert "2025" in label
assert "2026" in label
def test_bucket_by_day_groups_across_utc_midnight() -> None:
bucket = _internal_helpers["_bucket_by_day"]
rows = [
_cost(started_at=datetime(2026, 5, 2, 23, 0, tzinfo=UTC), cost_usd=1.0),
_cost(started_at=datetime(2026, 5, 2, 23, 30, tzinfo=UTC), cost_usd=2.0),
# 30 min later → next UTC day → separate bucket
_cost(started_at=datetime(2026, 5, 3, 0, 30, tzinfo=UTC), cost_usd=4.0),
]
buckets = bucket(rows)
assert len(buckets) == 2
assert buckets[0].day.isoformat() == "2026-05-02"
assert buckets[0].total_cost_usd == pytest.approx(3.0)
assert buckets[1].day.isoformat() == "2026-05-03"
assert buckets[1].total_cost_usd == pytest.approx(4.0)
def test_bucket_by_day_empty() -> None:
assert _internal_helpers["_bucket_by_day"]([]) == []
def test_model_breakdown_sorts_by_spend_descending() -> None:
breakdown = _internal_helpers["_model_breakdown"]
rows = [
_cost(model="gpt-4o-mini", cost_usd=0.5, run_id="r1"),
_cost(model="gpt-4o", cost_usd=10.0, run_id="r2"),
_cost(model="gpt-3.5-turbo", cost_usd=2.0, run_id="r3"),
]
out = breakdown(rows)
assert [b.model for b in out] == ["gpt-4o", "gpt-3.5-turbo", "gpt-4o-mini"]
assert out[0].total_cost_usd == 10.0
def test_model_breakdown_aggregates_runs_and_calls() -> None:
breakdown = _internal_helpers["_model_breakdown"]
rows = [
_cost(model="gpt-4o", run_id="r1", call_count=3, cost_usd=1.0),
_cost(model="gpt-4o", run_id="r1", call_count=2, cost_usd=2.0),
_cost(model="gpt-4o", run_id="r2", call_count=1, cost_usd=0.5),
]
out = breakdown(rows)
assert len(out) == 1
assert out[0].runs == 2 # r1, r2 — distinct
assert out[0].calls == 6 # 3 + 2 + 1
assert out[0].total_cost_usd == pytest.approx(3.5)
def test_model_breakdown_avg_latency() -> None:
breakdown = _internal_helpers["_model_breakdown"]
rows = [
_cost(model="gpt-4o", run_id="r1", call_count=2, latency_ms=200.0),
_cost(model="gpt-4o", run_id="r1", call_count=2, latency_ms=400.0),
]
out = breakdown(rows)
# (200 + 400) / (2 + 2) = 150 ms per call
assert out[0].avg_latency_ms == pytest.approx(150.0)
def test_top_cases_ranks_by_cost_desc_then_case_id() -> None:
top = _internal_helpers["_top_cases"]
rows = [
_cost(case_id="C-2", cost_usd=5.0, run_id="r1"),
_cost(case_id="C-1", cost_usd=5.0, run_id="r2"), # tie with C-2
_cost(case_id="C-3", cost_usd=10.0, run_id="r3"),
]
out = top(rows)
# Most expensive first, ties broken by case_id ascending.
assert [c.case_id for c in out] == ["C-3", "C-1", "C-2"]
def test_top_cases_skips_blank_case_id() -> None:
top = _internal_helpers["_top_cases"]
rows = [
_cost(case_id="", cost_usd=99.0),
_cost(case_id="C-1", cost_usd=1.0),
]
out = top(rows)
assert [c.case_id for c in out] == ["C-1"]
def test_top_cases_respects_limit() -> None:
top = _internal_helpers["_top_cases"]
rows = [_cost(case_id=f"C-{i:02d}", cost_usd=float(i)) for i in range(15)]
out = top(rows, limit=5)
assert len(out) == 5
assert out[0].case_id == "C-14"
def test_action_counts_orders_by_frequency() -> None:
counts = _internal_helpers["_action_counts"]
rows = [
AuditRow(action="cases:read"),
AuditRow(action="cases:read"),
AuditRow(action="cases:write"),
AuditRow(action="alerts:read"),
AuditRow(action="alerts:read"),
AuditRow(action="alerts:read"),
]
out = counts(rows)
assert [a.action for a in out] == ["alerts:read", "cases:read", "cases:write"]
assert out[0].count == 3
def test_action_counts_drops_blanks() -> None:
counts = _internal_helpers["_action_counts"]
rows = [AuditRow(action=""), AuditRow(action="cases:read")]
out = counts(rows)
assert len(out) == 1
assert out[0].action == "cases:read"
def test_action_counts_respects_limit() -> None:
counts = _internal_helpers["_action_counts"]
rows = [AuditRow(action=f"action-{i}") for i in range(50)]
out = counts(rows, limit=10)
assert len(out) == 10
# ---------------------------------------------------------------------------
# BYOK savings
# ---------------------------------------------------------------------------
def test_byok_savings_local_reports_full_imputed_cost() -> None:
savings_fn = _internal_helpers["_byok_savings"]
rows = [
_cost(model="gpt-4o-mini", prompt_tokens=10_000, completion_tokens=5_000),
]
# On BYOK the recorded cost typically already matches imputed (the agent
# uses the same pricing table) so savings should ≈ imputed.
result = savings_fn(rows, LlmContext(provider="local-ollama", is_local=True))
assert result.is_byok_active is True
assert result.savings_usd > 0
assert result.savings_usd == pytest.approx(result.imputed_public_cost_usd)
def test_byok_savings_hosted_reports_max_zero() -> None:
savings_fn = _internal_helpers["_byok_savings"]
# Hosted provider where the recorded cost matches imputed → zero savings.
rows = [
_cost(model="gpt-4o", prompt_tokens=1000, completion_tokens=500, cost_usd=0.0125),
]
result = savings_fn(rows, LlmContext(provider="openai", is_local=False))
assert result.is_byok_active is False
assert result.savings_usd == 0.0
def test_byok_savings_hosted_with_discount_reports_positive() -> None:
savings_fn = _internal_helpers["_byok_savings"]
# Operator paid less than list price (negotiated rate) → positive
# potential additional savings if they switched to BYOK.
rows = [_cost(model="gpt-4o", cost_usd=0.001)] # imputed ≈ $0.0125
result = savings_fn(rows, LlmContext(provider="openai", is_local=False))
assert result.is_byok_active is False
assert result.savings_usd > 0
assert result.savings_usd < result.imputed_public_cost_usd
def test_byok_savings_empty_rows() -> None:
savings_fn = _internal_helpers["_byok_savings"]
result = savings_fn([], LlmContext(provider="none", is_local=False))
assert result.recorded_cost_usd == 0.0
# Nothing to measure and nothing to price: both are unknown, not zero.
assert result.recorded_is_measured is False
# 0.0 with the flag off, not None: nullable would break every generated
# SDK client for a fact the flag already carries (the MTTR precedent).
assert result.imputed_is_estimable is False
assert result.imputed_public_cost_usd == 0.0
assert result.savings_usd == 0.0
# ---------------------------------------------------------------------------
# Headline
# ---------------------------------------------------------------------------
def test_headline_zero_runs_avoids_div_by_zero() -> None:
headline = _internal_helpers["_headline"]
out = headline([])
assert out.total_runs == 0
assert out.avg_cost_per_run_usd is None
def test_headline_aggregates_distinct_runs() -> None:
headline = _internal_helpers["_headline"]
rows = [
_cost(run_id="r1", cost_usd=1.0),
_cost(run_id="r1", cost_usd=2.0), # same run, different model call
_cost(run_id="r2", cost_usd=3.0),
]
out = headline(rows)
assert out.total_runs == 2
assert out.total_cost_usd == pytest.approx(6.0)
assert out.avg_cost_per_run_usd == pytest.approx(3.0)
# ---------------------------------------------------------------------------
# Top-level pure builder
# ---------------------------------------------------------------------------
def test_build_dashboard_empty_inputs_is_well_formed() -> None:
"""Empty tenant should still return a valid CostDashboard, not raise."""
dashboard = build_dashboard_from_rows(
DashboardInputs(
tenant_id=TENANT_ID,
period_start=PERIOD_START,
period_end=PERIOD_END,
cost_rows=[],
audit_rows=[],
llm=LlmContext(provider="none", is_local=False),
)
)
assert dashboard.tenant_id == TENANT_ID
assert dashboard.period.window_days == 7
assert dashboard.headline.total_cost_usd == 0.0
assert dashboard.headline.measured_call_count == 0, "zero rows measured nothing"
assert dashboard.daily_costs == []
assert dashboard.by_model == []
assert dashboard.top_cases == []
assert dashboard.action_counts == []
# BYOK panel is always present so the UI doesn't have to special-case it.
assert dashboard.byok_savings.is_byok_active is False
assert dashboard.byok_savings.imputed_is_estimable is False
assert dashboard.byok_savings.savings_usd == 0.0
def test_build_dashboard_end_to_end() -> None:
"""A representative tenant: 3 runs, 2 models, 2 days, mixed actions."""
cost_rows = [
_cost(
run_id="r1",
case_id="C-1",
model="gpt-4o-mini",
prompt_tokens=2000,
completion_tokens=1000,
started_at=datetime(2026, 5, 2, 10, 0, tzinfo=UTC),
),
_cost(
run_id="r2",
case_id="C-2",
model="gpt-4o",
prompt_tokens=10_000,
completion_tokens=5_000,
cost_usd=0.125, # cheaper than imputed → operator saved
started_at=datetime(2026, 5, 3, 14, 0, tzinfo=UTC),
),
_cost(
run_id="r3",
case_id="C-2", # same case, separate run → C-2 ranks #1
model="gpt-4o",
prompt_tokens=5000,
completion_tokens=2000,
cost_usd=0.05,
started_at=datetime(2026, 5, 3, 15, 0, tzinfo=UTC),
),
]
audit_rows = [
AuditRow(action="cases:read"),
AuditRow(action="cases:read"),
AuditRow(action="cases:write"),
AuditRow(action="alerts:read"),
]
dashboard = build_dashboard_from_rows(
DashboardInputs(
tenant_id=TENANT_ID,
period_start=PERIOD_START,
period_end=PERIOD_END,
cost_rows=cost_rows,
audit_rows=audit_rows,
llm=LlmContext(provider="openai", is_local=False),
)
)
# Headline
assert dashboard.headline.total_runs == 3
assert dashboard.headline.total_calls == 3
assert dashboard.headline.total_tokens == 25_000
# Daily buckets — 2 distinct UTC days
assert [b.day.isoformat() for b in dashboard.daily_costs] == [
"2026-05-02",
"2026-05-03",
]
# Model breakdown — gpt-4o has higher recorded cost
assert dashboard.by_model[0].model == "gpt-4o"
assert dashboard.by_model[1].model == "gpt-4o-mini"
# Top cases — C-2 has 2 runs and higher spend
assert dashboard.top_cases[0].case_id == "C-2"
assert dashboard.top_cases[0].runs == 2
# Action counts
assert dashboard.action_counts[0].action == "cases:read"
assert dashboard.action_counts[0].count == 2
# BYOK — hosted provider, recorded < imputed → positive savings
assert dashboard.byok_savings.is_byok_active is False
assert dashboard.byok_savings.savings_usd > 0
def test_build_dashboard_byok_active_marks_local_provider() -> None:
cost_rows = [
_cost(
run_id="r1",
case_id="C-1",
model="gpt-4o-mini",
prompt_tokens=1000,
completion_tokens=500,
)
]
dashboard = build_dashboard_from_rows(
DashboardInputs(
tenant_id=TENANT_ID,
period_start=PERIOD_START,
period_end=PERIOD_END,
cost_rows=cost_rows,
audit_rows=[],
llm=LlmContext(provider="local-ollama", is_local=True),
)
)
assert dashboard.byok_savings.is_byok_active is True
assert dashboard.byok_savings.provider == "local-ollama"
assert dashboard.byok_savings.savings_usd > 0