Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(sec): derive gross_profit from revenue minus cogs when tag absent (#1111)

cgycorey57b40ddaacfb
diff --git a/agent/backtest/loaders/fundamentals_loader.py b/agent/backtest/loaders/fundamentals_loader.py
index d7c1a0240..9afb2c8ac 100644
--- a/agent/backtest/loaders/fundamentals_loader.py
+++ b/agent/backtest/loaders/fundamentals_loader.py
@@ -36,6 +36,7 @@ _FLOW_CONCEPTS = {
"SalesRevenueNet",
"CostOfGoodsAndServicesSold",
"CostOfRevenue",
+ "GrossProfit",
"OperatingIncomeLoss",
"NetIncomeLoss",
"ProfitLoss",
@@ -259,6 +260,23 @@ def load_fundamental_panel(
columns=symbol_list,
)
+ # Raw fields may declare a fallback derivation (e.g. gross_profit =
+ # revenue - cogs). Fill only cells where the directly-reported concept was
+ # absent; reported values win.
+ for field in raw_fields:
+ spec = _raw_field_spec(schema, field)
+ if spec is None:
+ continue
+ deps = _spec_get(spec, "dependencies") or ()
+ compute = _spec_get(spec, "compute")
+ if not callable(compute) or not deps:
+ continue
+ dep_frames = {str(dep): panels.get(str(dep)) for dep in deps}
+ if any(frame is None for frame in dep_frames.values()):
+ continue
+ fallback = _compute_derived(spec, dep_frames)
+ panels[field] = panels[field].where(panels[field].notna(), fallback)
+
def panel_for(field: str) -> pd.DataFrame:
if field in panels:
return panels[field]
@@ -340,13 +358,17 @@ def _collect_raw_fields(schema: Any, fields: Iterable[str]) -> set[str]:
def visit(field: str) -> None:
if field in visiting:
raise ValueError(f"cyclic derived fundamental field: {field}")
+ visiting.add(field)
derived = _derived_field(schema, field)
if derived is None:
raw.add(field)
- return
- visiting.add(field)
- for dep in _derived_dependencies(derived):
- visit(_resolve_field_name(schema, dep))
+ spec = _raw_field_spec(schema, field)
+ if spec is not None:
+ for dep in _spec_get(spec, "dependencies") or ():
+ visit(_resolve_field_name(schema, str(dep)))
+ else:
+ for dep in _derived_dependencies(derived):
+ visit(_resolve_field_name(schema, dep))
visiting.remove(field)
for field in fields:
@@ -358,6 +380,13 @@ def _derived_field(schema: Any, field: str) -> Any | None:
return getattr(schema, "DERIVED_FIELDS", {}).get(field)
+def _raw_field_spec(schema: Any, field: str) -> Any | None:
+ raw_fields = getattr(schema, "RAW_FIELDS", {})
+ if isinstance(raw_fields, dict):
+ return raw_fields.get(field)
+ return getattr(raw_fields, field, None)
+
+
def _spec_get(derived: Any, key: str) -> Any:
"""Read a derived-field spec entry from either a mapping or an object."""
if isinstance(derived, dict):
diff --git a/agent/tests/test_fundamentals_gross_profit_fallback.py b/agent/tests/test_fundamentals_gross_profit_fallback.py
new file mode 100644
index 000000000..63b3714c6
--- /dev/null
+++ b/agent/tests/test_fundamentals_gross_profit_fallback.py
@@ -0,0 +1,239 @@
+"""Regression tests for the ``gross_profit`` revenue-minus-cogs fallback.
+
+``RAW_FIELDS["gross_profit"]`` declares ``compute = revenue - cogs`` so that
+filers who report revenue and cogs separately (without a literal
+``GrossProfit`` XBRL concept) still get a gross profit and, transitively, a
+``gross_profitability``. These tests pin that fallback at the loader level,
+including PIT anchoring and the preference for the directly reported concept.
+
+No test touches a live endpoint: ``cik_for`` / ``get_company_facts`` are
+monkeypatched on the loader's SEC client module.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+
+from backtest.loaders import fundamentals_loader
+
+
+def _facts(concept_rows: dict[str, list[dict[str, object]]]) -> dict[str, object]:
+ return {
+ "facts": {
+ "us-gaap": {
+ concept: {"units": {"USD": rows}}
+ for concept, rows in concept_rows.items()
+ }
+ }
+ }
+
+
+def _fact_row(
+ end: str,
+ filed: str,
+ value: float,
+ *,
+ form: str = "10-Q",
+ start: str | None = None,
+) -> dict[str, object]:
+ if start is None:
+ start = (pd.Timestamp(end) - pd.Timedelta(days=91)).strftime("%Y-%m-%d")
+ return {"start": start, "end": end, "filed": filed, "val": value, "form": form}
+
+
+def _patch_sec(
+ monkeypatch: pytest.MonkeyPatch,
+ facts_by_symbol: dict[str, dict[str, object]],
+) -> None:
+ def cik_for(symbol: str) -> str | None:
+ return f"CIK-{symbol}" if symbol in facts_by_symbol else None
+
+ def get_company_facts(cik: str) -> dict[str, object]:
+ symbol = cik.removeprefix("CIK-")
+ return facts_by_symbol[symbol]
+
+ monkeypatch.setattr(fundamentals_loader.sec_edgar_client, "cik_for", cik_for)
+ monkeypatch.setattr(
+ fundamentals_loader.sec_edgar_client,
+ "get_company_facts",
+ get_company_facts,
+ )
+
+
+_QUARTER_END = "2024-03-31"
+_FILED = "2024-04-20"
+
+
+def test_gross_profit_falls_back_to_revenue_minus_cogs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _patch_sec(
+ monkeypatch,
+ {
+ "AAA": _facts(
+ {
+ "Revenues": [_fact_row(_QUARTER_END, _FILED, 100.0)],
+ "CostOfRevenue": [_fact_row(_QUARTER_END, _FILED, 60.0)],
+ "Assets": [_fact_row(_QUARTER_END, _FILED, 1000.0)],
+ }
+ )
+ },
+ )
+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")
+ panel = fundamentals_loader.load_fundamental_panel(
+ ["AAA"],
+ ["gross_profit", "gross_profitability"],
+ "2024-04-01",
+ "2024-05-01",
+ freq="quarterly",
+ index=index,
+ )
+
+ gross_profit = panel["gross_profit"]["AAA"]
+ # PIT: nothing visible before the filing date, fallback value after.
+ assert pd.isna(gross_profit.loc["2024-04-19"])
+ assert gross_profit.loc["2024-04-20"] == 40.0
+ assert gross_profit.loc["2024-05-01"] == 40.0
+
+ profitability = panel["gross_profitability"]["AAA"]
+ assert pd.isna(profitability.loc["2024-04-19"])
+ assert profitability.loc["2024-04-20"] == pytest.approx(0.04)
+
+
+def test_gross_profit_prefers_direct_concept_over_fallback(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _patch_sec(
+ monkeypatch,
+ {
+ "AAA": _facts(
+ {
+ "GrossProfit": [_fact_row(_QUARTER_END, _FILED, 45.0)],
+ "Revenues": [_fact_row(_QUARTER_END, _FILED, 100.0)],
+ "CostOfRevenue": [_fact_row(_QUARTER_END, _FILED, 60.0)],
+ "Assets": [_fact_row(_QUARTER_END, _FILED, 1000.0)],
+ }
+ )
+ },
+ )
+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")
+
+ panel = fundamentals_loader.load_fundamental_panel(
+ ["AAA"],
+ ["gross_profit", "gross_profitability"],
+ "2024-04-01",
+ "2024-05-01",
+ freq="quarterly",
+ index=index,
+ )
+
+ assert panel["gross_profit"]["AAA"].loc["2024-04-20"] == 45.0
+ assert panel["gross_profitability"]["AAA"].loc["2024-04-20"] == pytest.approx(0.045)
+
+
+def test_gross_profit_stays_null_without_either_source(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _patch_sec(
+ monkeypatch,
+ {
+ "AAA": _facts(
+ {
+ # revenue present but cogs absent: fallback cannot fire and
+ # must not fabricate a gross profit from revenue alone.
+ "Revenues": [_fact_row(_QUARTER_END, _FILED, 100.0)],
+ "Assets": [_fact_row(_QUARTER_END, _FILED, 1000.0)],
+ }
+ )
+ },
+ )
+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")
+
+ panel = fundamentals_loader.load_fundamental_panel(
+ ["AAA"],
+ ["gross_profit", "gross_profitability"],
+ "2024-04-01",
+ "2024-05-01",
+ freq="quarterly",
+ index=index,
+ )
+
+ assert pd.isna(panel["gross_profit"]["AAA"].loc["2024-05-01"])
+ assert pd.isna(panel["gross_profitability"]["AAA"].loc["2024-05-01"])
+
+
+def test_gross_profit_direct_concept_is_ttm_summed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _patch_sec(
+ monkeypatch,
+ {
+ "AAA": _facts(
+ {
+ "GrossProfit": [
+ _fact_row("2023-06-30", "2023-07-20", 10.0),
+ _fact_row("2023-09-30", "2023-10-20", 20.0),
+ _fact_row("2023-12-31", "2024-01-20", 30.0),
+ _fact_row("2024-03-31", "2024-04-20", 40.0),
+ ]
+ }
+ )
+ },
+ )
+ index = pd.date_range("2024-04-01", "2024-05-01", freq="D")
+
+ panel = fundamentals_loader.load_fundamental_panel(
+ ["AAA"],
+ ["gross_profit"],
+ "2024-04-01",
+ "2024-05-01",
+ freq="ttm",
+ index=index,
+ )
+
+ gross_profit = panel["gross_profit"]["AAA"]
+ # TTM must be the rolling four-quarter sum, not the latest quarter.
+ assert gross_profit.loc["2024-05-01"] == 100.0
+
+
+def test_gross_profit_direct_concept_excludes_annual_span_from_quarterly(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _patch_sec(
+ monkeypatch,
+ {
+ "AAA": _facts(
+ {
+ "GrossProfit": [
+ _fact_row("2023-06-30", "2023-07-20", 10.0),
+ _fact_row("2023-09-30", "2023-10-20", 20.0),
+ _fact_row("2023-12-31", "2024-01-20", 30.0),
+ _fact_row(
+ "2024-03-31",
+ "2024-02-20",
+ 100.0,
+ form="10-K",
+ start="2023-03-31",
+ ),
+ ]
+ }
+ )
+ },
+ )
+ index = pd.date_range("2024-02-01", "2024-03-10", freq="D")
+
+ panel = fundamentals_loader.load_fundamental_panel(
+ ["AAA"],
+ ["gross_profit"],
+ "2024-02-01",
+ "2024-03-10",
+ freq="quarterly",
+ index=index,
+ )
+
+ gross_profit = panel["gross_profit"]["AAA"]
+ # The 10-K row is an annual span: quarterly cadence must synthesize fiscal
+ # Q4 (100 - 10 - 20 - 30), never surface the full-year value.
+ assert gross_profit.loc["2024-02-19"] == 30.0
+ assert gross_profit.loc["2024-02-20"] == 40.0