"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(sec): derive gross_profit from revenue minus cogs when tag absent (#1111)
cgycorey57b40ddaacfb
diff --git a/agent/backtest/loaders/fundamentals_loader.py b/agent/backtest/loaders/fundamentals_loader.pyindex d7c1a0240..9afb2c8ac 100644--- a/agent/backtest/loaders/fundamentals_loader.py+++ b/agent/backtest/loaders/fundamentals_loader.py@@ -36,6 +36,7 @@ _FLOW_CONCEPTS = {"SalesRevenueNet","CostOfGoodsAndServicesSold","CostOfRevenue",+ "GrossProfit","OperatingIncomeLoss","NetIncomeLoss","ProfitLoss",@@ -259,6 +260,23 @@ def load_fundamental_panel(columns=symbol_list,)+ # Raw fields may declare a fallback derivation (e.g. gross_profit =+ # revenue - cogs). Fill only cells where the directly-reported concept was+ # absent; reported values win.+ for field in raw_fields:+ spec = _raw_field_spec(schema, field)+ if spec is None:+ continue+ deps = _spec_get(spec, "dependencies") or ()+ compute = _spec_get(spec, "compute")+ if not callable(compute) or not deps:+ continue+ dep_frames = {str(dep): panels.get(str(dep)) for dep in deps}+ if any(frame is None for frame in dep_frames.values()):+ continue+ fallback = _compute_derived(spec, dep_frames)+ panels[field] = panels[field].where(panels[field].notna(), fallback)+def panel_for(field: str) -> pd.DataFrame:if field in panels:return panels[field]@@ -340,13 +358,17 @@ def _collect_raw_fields(schema: Any, fields: Iterable[str]) -> set[str]:def visit(field: str) -> None:if field in visiting:raise ValueError(f"cyclic derived fundamental field: {field}")+ visiting.add(field)derived = _derived_field(schema, field)if derived is None:raw.add(field)- return- visiting.add(field)- for dep in _derived_dependencies(derived):- visit(_resolve_field_name(schema, dep))+ spec = _raw_field_spec(schema, field)+ if spec is not None:+ for dep in _spec_get(spec, "dependencies") or ():+ visit(_resolve_field_name(schema, str(dep)))+ else:+ for dep in _derived_dependencies(derived):+ visit(_resolve_field_name(schema, dep))visiting.remove(field)for field in fields:@@ -358,6 +380,13 @@ def _derived_field(schema: Any, field: str) -> Any | None:return getattr(schema, "DERIVED_FIELDS", {}).get(field)+def _raw_field_spec(schema: Any, field: str) -> Any | None:+ raw_fields = getattr(schema, "RAW_FIELDS", {})+ if isinstance(raw_fields, dict):+ return raw_fields.get(field)+ return getattr(raw_fields, field, None)++def _spec_get(derived: Any, key: str) -> Any:"""Read a derived-field spec entry from either a mapping or an object."""if isinstance(derived, dict):diff --git a/agent/tests/test_fundamentals_gross_profit_fallback.py b/agent/tests/test_fundamentals_gross_profit_fallback.pynew file mode 100644index 000000000..63b3714c6--- /dev/null+++ b/agent/tests/test_fundamentals_gross_profit_fallback.py@@ -0,0 +1,239 @@+"""Regression tests for the ``gross_profit`` revenue-minus-cogs fallback.++``RAW_FIELDS["gross_profit"]`` declares ``compute = revenue - cogs`` so that+filers who report revenue and cogs separately (without a literal+``GrossProfit`` XBRL concept) still get a gross profit and, transitively, a+``gross_profitability``. These tests pin that fallback at the loader level,+including PIT anchoring and the preference for the directly reported concept.++No test touches a live endpoint: ``cik_for`` / ``get_company_facts`` are+monkeypatched on the loader's SEC client module.+"""++from __future__ import annotations++import pandas as pd+import pytest++from backtest.loaders import fundamentals_loader+++def _facts(concept_rows: dict[str, list[dict[str, object]]]) -> dict[str, object]:+ return {+ "facts": {+ "us-gaap": {+ concept: {"units": {"USD": rows}}+ for concept, rows in concept_rows.items()+ }+ }+ }+++def _fact_row(+ end: str,+ filed: str,+ value: float,+ *,+ form: str = "10-Q",+ start: str | None = None,+) -> dict[str, object]:+ if start is None:+ start = (pd.Timestamp(end) - pd.Timedelta(days=91)).strftime("%Y-%m-%d")+ return {"start": start, "end": end, "filed": filed, "val": value, "form": form}+++def _patch_sec(+ monkeypatch: pytest.MonkeyPatch,+ facts_by_symbol: dict[str, dict[str, object]],+) -> None:+ def cik_for(symbol: str) -> str | None:+ return f"CIK-{symbol}" if symbol in facts_by_symbol else None++ def get_company_facts(cik: str) -> dict[str, object]:+ symbol = cik.removeprefix("CIK-")+ return facts_by_symbol[symbol]++ monkeypatch.setattr(fundamentals_loader.sec_edgar_client, "cik_for", cik_for)+ monkeypatch.setattr(+ fundamentals_loader.sec_edgar_client,+ "get_company_facts",+ get_company_facts,+ )+++_QUARTER_END = "2024-03-31"+_FILED = "2024-04-20"+++def test_gross_profit_falls_back_to_revenue_minus_cogs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _patch_sec(+ monkeypatch,+ {+ "AAA": _facts(+ {+ "Revenues": [_fact_row(_QUARTER_END, _FILED, 100.0)],+ "CostOfRevenue": [_fact_row(_QUARTER_END, _FILED, 60.0)],+ "Assets": [_fact_row(_QUARTER_END, _FILED, 1000.0)],+ }+ )+ },+ )+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")+ panel = fundamentals_loader.load_fundamental_panel(+ ["AAA"],+ ["gross_profit", "gross_profitability"],+ "2024-04-01",+ "2024-05-01",+ freq="quarterly",+ index=index,+ )++ gross_profit = panel["gross_profit"]["AAA"]+ # PIT: nothing visible before the filing date, fallback value after.+ assert pd.isna(gross_profit.loc["2024-04-19"])+ assert gross_profit.loc["2024-04-20"] == 40.0+ assert gross_profit.loc["2024-05-01"] == 40.0++ profitability = panel["gross_profitability"]["AAA"]+ assert pd.isna(profitability.loc["2024-04-19"])+ assert profitability.loc["2024-04-20"] == pytest.approx(0.04)+++def test_gross_profit_prefers_direct_concept_over_fallback(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _patch_sec(+ monkeypatch,+ {+ "AAA": _facts(+ {+ "GrossProfit": [_fact_row(_QUARTER_END, _FILED, 45.0)],+ "Revenues": [_fact_row(_QUARTER_END, _FILED, 100.0)],+ "CostOfRevenue": [_fact_row(_QUARTER_END, _FILED, 60.0)],+ "Assets": [_fact_row(_QUARTER_END, _FILED, 1000.0)],+ }+ )+ },+ )+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")++ panel = fundamentals_loader.load_fundamental_panel(+ ["AAA"],+ ["gross_profit", "gross_profitability"],+ "2024-04-01",+ "2024-05-01",+ freq="quarterly",+ index=index,+ )++ assert panel["gross_profit"]["AAA"].loc["2024-04-20"] == 45.0+ assert panel["gross_profitability"]["AAA"].loc["2024-04-20"] == pytest.approx(0.045)+++def test_gross_profit_stays_null_without_either_source(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _patch_sec(+ monkeypatch,+ {+ "AAA": _facts(+ {+ # revenue present but cogs absent: fallback cannot fire and+ # must not fabricate a gross profit from revenue alone.+ "Revenues": [_fact_row(_QUARTER_END, _FILED, 100.0)],+ "Assets": [_fact_row(_QUARTER_END, _FILED, 1000.0)],+ }+ )+ },+ )+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")++ panel = fundamentals_loader.load_fundamental_panel(+ ["AAA"],+ ["gross_profit", "gross_profitability"],+ "2024-04-01",+ "2024-05-01",+ freq="quarterly",+ index=index,+ )++ assert pd.isna(panel["gross_profit"]["AAA"].loc["2024-05-01"])+ assert pd.isna(panel["gross_profitability"]["AAA"].loc["2024-05-01"])+++def test_gross_profit_direct_concept_is_ttm_summed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _patch_sec(+ monkeypatch,+ {+ "AAA": _facts(+ {+ "GrossProfit": [+ _fact_row("2023-06-30", "2023-07-20", 10.0),+ _fact_row("2023-09-30", "2023-10-20", 20.0),+ _fact_row("2023-12-31", "2024-01-20", 30.0),+ _fact_row("2024-03-31", "2024-04-20", 40.0),+ ]+ }+ )+ },+ )+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")++ panel = fundamentals_loader.load_fundamental_panel(+ ["AAA"],+ ["gross_profit"],+ "2024-04-01",+ "2024-05-01",+ freq="ttm",+ index=index,+ )++ gross_profit = panel["gross_profit"]["AAA"]+ # TTM must be the rolling four-quarter sum, not the latest quarter.+ assert gross_profit.loc["2024-05-01"] == 100.0+++def test_gross_profit_direct_concept_excludes_annual_span_from_quarterly(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _patch_sec(+ monkeypatch,+ {+ "AAA": _facts(+ {+ "GrossProfit": [+ _fact_row("2023-06-30", "2023-07-20", 10.0),+ _fact_row("2023-09-30", "2023-10-20", 20.0),+ _fact_row("2023-12-31", "2024-01-20", 30.0),+ _fact_row(+ "2024-03-31",+ "2024-02-20",+ 100.0,+ form="10-K",+ start="2023-03-31",+ ),+ ]+ }+ )+ },+ )+ index = pd.date_range("2024-02-01", "2024-03-10", freq="D")++ panel = fundamentals_loader.load_fundamental_panel(+ ["AAA"],+ ["gross_profit"],+ "2024-02-01",+ "2024-03-10",+ freq="quarterly",+ index=index,+ )++ gross_profit = panel["gross_profit"]["AAA"]+ # The 10-K row is an annual span: quarterly cadence must synthesize fiscal+ # Q4 (100 - 10 - 20 - 30), never surface the full-year value.+ assert gross_profit.loc["2024-02-19"] == 30.0+ assert gross_profit.loc["2024-02-20"] == 40.0