Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(backtest): report actual post-fill positions

AndyLongest72d568b3b3a0
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.py
index f04bf8364..9cf4bebe4 100644
--- a/agent/backtest/engines/base.py
+++ b/agent/backtest/engines/base.py
@@ -409,6 +409,11 @@ class BaseEngine(ABC):
self.fill_records: List[FillRecord] = []
self.trades: List[TradeRecord] = []
self.equity_snapshots: List[EquitySnapshot] = []
+ # Per-bar, post-fill portfolio weights. These are deliberately kept
+ # separate from ``target_pos``: market rules, lot rounding, fees, and
+ # insufficient cash can all make the executed book differ from the
+ # optimiser's request.
+ self.actual_position_snapshots: List[tuple[pd.Timestamp, Dict[str, float]]] = []
self._bar_idx: int = 0
self._active_symbol: str = "" # set by _rebalance/_close_position for subclass use
@@ -716,6 +721,7 @@ class BaseEngine(ABC):
# 4. Bar-by-bar execution
self._execute_bars(dates, data_map, close_df, target_pos, valid_codes)
+ actual_pos = self._actual_positions_frame(valid_codes)
# 5. Build output series
equity_series = pd.Series(
@@ -760,7 +766,7 @@ class BaseEngine(ABC):
self.initial_capital,
bars_per_year,
bench_ret,
- target_pos,
+ actual_pos,
turnover_series=realized_turnover,
)
m.update(benchmark_metadata)
@@ -807,7 +813,7 @@ class BaseEngine(ABC):
write_risk_xray,
)
try:
- basket_weights, avg_invested = average_invested_weights(target_pos)
+ basket_weights, avg_invested = average_invested_weights(actual_pos)
risk_xray = compute_risk_xray(
close_df, basket_weights, periods_per_year=bars_per_year,
)
@@ -881,6 +887,7 @@ class BaseEngine(ABC):
# Store as instance attrs for use in _calc_equity / _safe_price
self._close_arr = _close_arr
self._code_to_col = _code_to_col
+ self.actual_position_snapshots = []
for i, ts in enumerate(dates):
self._bar_idx = i
@@ -906,7 +913,8 @@ class BaseEngine(ABC):
self._execute_target_rebalance(target_weights, data_map, ts, equity, codes)
target_weights = {}
- # b. Release capital before opening replacement positions. A
+ # b. In legacy hold mode, release capital before opening replacement
+ # positions or increasing other positions. A
# single mixed close/open pass makes rotations depend on symbol
# iteration order when the new name is visited before the old one.
for c in codes:
@@ -981,6 +989,9 @@ class BaseEngine(ABC):
# e. Record equity snapshot
snap_equity = self._calc_equity(close_df, ts)
+ self.actual_position_snapshots.append(
+ (ts, self._actual_weights(close_df, ts, codes, snap_equity))
+ )
if self.positions and type(self)._calc_pnl is BaseEngine._calc_pnl:
_syms = list(self.positions.keys())
_eps = np.array([p.entry_price for p in self.positions.values()])
@@ -1038,6 +1049,10 @@ class BaseEngine(ABC):
equity=self.capital,
positions=0,
)
+ if self.actual_position_snapshots:
+ self.actual_position_snapshots[-1] = (
+ last_ts, {code: 0.0 for code in codes}
+ )
# Clean up temporary instance attributes
self._close_arr = None
@@ -1501,6 +1516,46 @@ class BaseEngine(ABC):
reason="signal",
)
+ def _actual_weights(
+ self,
+ close_df: pd.DataFrame,
+ ts: pd.Timestamp,
+ codes: List[str],
+ equity: float,
+ ) -> Dict[str, float]:
+ """Return post-fill, mark-to-market weights for the currently held book."""
+ weights = {code: 0.0 for code in codes}
+ if abs(equity) <= 1e-12:
+ return weights
+ for symbol, pos in self.positions.items():
+ price = self._safe_price(
+ close_df,
+ ts,
+ symbol,
+ pos.entry_price,
+ _arr=getattr(self, "_close_arr", None),
+ _row=getattr(self, "_bar_idx", None),
+ _col=getattr(self, "_code_to_col", {}).get(symbol),
+ )
+ margin_value = self._calc_margin(
+ symbol, pos.size, price, pos.leverage
+ )
+ weights[symbol] = pos.direction * margin_value / equity
+ return weights
+
+ def _actual_positions_frame(self, codes: List[str]) -> pd.DataFrame:
+ """Materialize recorded post-fill weights as an artifact-ready frame."""
+ if not self.actual_position_snapshots:
+ return pd.DataFrame(columns=codes, dtype=float)
+ frame = pd.DataFrame(
+ [weights for _, weights in self.actual_position_snapshots],
+ index=[timestamp for timestamp, _ in self.actual_position_snapshots],
+ columns=codes,
+ dtype=float,
+ )
+ frame.index.name = "timestamp"
+ return frame
+
def _close_position(
self,
symbol: str,
@@ -1659,9 +1714,13 @@ class BaseEngine(ABC):
eq_df.index.name = "timestamp"
eq_df.to_csv(out / "equity.csv")
- # Position weights (target, for compatibility)
- target_pos.index.name = "timestamp"
- target_pos.to_csv(out / "positions.csv")
+ # ``positions.csv`` is execution truth. Keep optimiser requests in a
+ # separate artifact so blocked/rounded/scaled fills remain auditable.
+ actual_pos = self._actual_positions_frame(codes)
+ actual_pos.to_csv(out / "positions.csv")
+ target_out = target_pos.copy()
+ target_out.index.name = "timestamp"
+ target_out.to_csv(out / "target_positions.csv")
# Trades (compatible format)
trade_rows = []
diff --git a/agent/backtest/rebalance_notes.py b/agent/backtest/rebalance_notes.py
index 972e235ca..7b30c425e 100644
--- a/agent/backtest/rebalance_notes.py
+++ b/agent/backtest/rebalance_notes.py
@@ -32,7 +32,7 @@ def compute_rebalance_notes(
Args:
target_pos: Target weights (dates x codes), e.g. the frame behind
- ``artifacts/positions.csv``. NaN cells are treated as zero.
+ ``artifacts/target_positions.csv``. NaN cells are treated as zero.
top_n: How many largest per-name moves to keep per rebalance.
epsilon: Turnover at or below this counts as "no rebalance".
diff --git a/agent/src/api/models.py b/agent/src/api/models.py
index 18733a45e..831e56d56 100644
--- a/agent/src/api/models.py
+++ b/agent/src/api/models.py
@@ -60,6 +60,7 @@ class RunResponse(BaseModel):
run_id: str = Field(..., description="Run identifier")
elapsed_seconds: float = Field(..., description="Execution time in seconds")
reason: Optional[str] = Field(None, description="Failure reason when available")
+ prompt: Optional[str] = Field(None, description="Original natural-language request")
planner_output: Optional[Dict[str, Any]] = Field(None, description="Planner output")
strategy_spec: Optional[Dict[str, Any]] = Field(None, description="Strategy specification")
@@ -76,6 +77,12 @@ class RunResponse(BaseModel):
artifacts_equity_csv: Optional[List[Dict[str, Any]]] = Field(None, description="Full equity rows")
artifacts_metrics_csv: Optional[List[Dict[str, Any]]] = Field(None, description="Full metrics rows")
artifacts_trades_csv: Optional[List[Dict[str, Any]]] = Field(None, description="Full trade rows")
+ artifacts_positions_csv: Optional[List[Dict[str, Any]]] = Field(
+ None, description="Actual post-execution position weights"
+ )
+ artifacts_target_positions_csv: Optional[List[Dict[str, Any]]] = Field(
+ None, description="Optimizer-requested target weights"
+ )
validation: Optional[Dict[str, Any]] = Field(None, description="Statistical validation results")
risk_xray: Optional[Dict[str, Any]] = Field(
None, description="Portfolio risk x-ray payload when the run emitted one"
diff --git a/agent/src/api/runs_routes.py b/agent/src/api/runs_routes.py
index c3e064b73..ab1de81b4 100644
--- a/agent/src/api/runs_routes.py
+++ b/agent/src/api/runs_routes.py
@@ -89,6 +89,9 @@ def _build_response_from_run_dir(
else:
response.status = "unknown"
+ request_data = _load_json_file(run_dir / "req.json") or {}
+ response.prompt = request_data.get("prompt") or request_data.get("request")
+
planner_path = run_dir / "planner_output.json"
response.planner_output = _load_json_file(planner_path)
@@ -172,6 +175,14 @@ def _build_response_from_run_dir(
if trades_path.exists():
response.artifacts_trades_csv = _load_csv_to_dict(trades_path)
+ positions_path = run_dir / "artifacts" / "positions.csv"
+ if positions_path.exists():
+ response.artifacts_positions_csv = _load_csv_to_dict(positions_path)
+
+ target_positions_path = run_dir / "artifacts" / "target_positions.csv"
+ if target_positions_path.exists():
+ response.artifacts_target_positions_csv = _load_csv_to_dict(target_positions_path)
+
validation_path = run_dir / "artifacts" / "validation.json"
if validation_path.exists():
try:
diff --git a/agent/src/skills/strategy-generate/SKILL.md b/agent/src/skills/strategy-generate/SKILL.md
index 23a2e2501..465c3fce8 100644
--- a/agent/src/skills/strategy-generate/SKILL.md
+++ b/agent/src/skills/strategy-generate/SKILL.md
@@ -126,7 +126,7 @@ Self-check after writing `signal_engine.py`:
"optimizer": null,
"optimizer_params": {},
"engine": "daily",
- "position_adjustment": "hold",
+ "position_adjustment": "rebalance",
"validation": null
}
```
@@ -142,7 +142,7 @@ Self-check after writing `signal_engine.py`:
- `optimizer`: optional, one of `"equal_volatility"` / `"risk_parity"` / `"mean_variance"` / `"max_diversification"` / `"turnover_aware"` / `null` (equal-weight by default)
- `optimizer_params`: optimizer parameters, such as `{"lookback": 60}`. `mean_variance` additionally supports `{"risk_free": 0.0}`; `turnover_aware` supports `{"risk_aversion": 1.0, "turnover_penalty": 0.5}` (L1 penalty on weight changes; tune to data frequency)
- `engine`: backtest engine, default `"daily"`. For options strategies, set `"options"` (requires `OptionsSignalEngine`)
-- `position_adjustment`: `"hold"` (default) preserves an open same-direction position until close/reversal; opt in to `"rebalance"` to scale or reduce it toward each target weight using market fills and weighted-average entry accounting
+- `position_adjustment`: use `"rebalance"` for target-weight strategies so every target change is executed using market fills and weighted-average entry accounting; use `"hold"` only when the strategy explicitly intends to preserve a same-direction position until close/reversal
- `initial_cash`: default 1,000,000
- `commission`: default 0.1%
- `validation`: optional statistical validation after backtest completes. Omit to skip. Example:
diff --git a/agent/tests/test_base_engine.py b/agent/tests/test_base_engine.py
index 710e74ea8..3c6a8e4ea 100644
--- a/agent/tests/test_base_engine.py
+++ b/agent/tests/test_base_engine.py
@@ -386,6 +386,106 @@ class _LifecycleEngine(ChinaAEngine):
super()._execute_open_order(order, ts)
+class _FractionalEngine(BaseEngine):
+ """Frictionless engine used to expose target-vs-execution differences."""
+
+ def __init__(self, *, block_adds_after_first: bool = False):
+ super().__init__({"initial_cash": 1_000.0, "position_adjustment": "rebalance"})
+ self.block_adds_after_first = block_adds_after_first
+
+ def can_execute(self, symbol, direction, bar):
+ if (
+ self.block_adds_after_first
+ and direction != 0
+ and symbol in self.positions
+ ):
+ return False
+ return True
+
+ def round_size(self, raw_size, price):
+ return raw_size
+
+ def calc_commission(self, size, price, direction, is_open):
+ return 0.0
+
+ def apply_slippage(self, price, direction):
+ return price
+
+
+def _fractional_fixture(weights: list[float]):
+ dates = pd.bdate_range("2026-01-05", periods=len(weights))
+ bars = pd.DataFrame(
+ {"open": [100.0] * len(dates), "close": [100.0] * len(dates)},
+ index=dates,
+ )
+ close_df = pd.DataFrame({"AAPL.US": bars["close"]}, index=dates)
+ targets = pd.DataFrame({"AAPL.US": weights}, index=dates)
+ return dates, bars, close_df, targets
+
+
+def test_same_direction_target_change_resizes_actual_position() -> None:
+ dates, bars, close_df, targets = _fractional_fixture([0.2, 0.8, 0.8])
+ engine = _FractionalEngine()
+
+ engine._execute_bars(
+ dates, {"AAPL.US": bars}, close_df, targets, ["AAPL.US"]
+ )
+
+ actual = engine._actual_positions_frame(["AAPL.US"])
+ assert actual.loc[dates[0], "AAPL.US"] == pytest.approx(0.2)
+ assert actual.loc[dates[1], "AAPL.US"] == pytest.approx(0.8)
+ # The terminal liquidation closes the full resized position, proving the
+ # engine held 8 shares rather than retaining the original 2 shares.
+ assert engine.trades[-1].size == pytest.approx(8.0)
+
+
+def test_same_direction_target_reduction_partially_closes() -> None:
+ dates, bars, close_df, targets = _fractional_fixture([0.8, 0.2, 0.2])
+ engine = _FractionalEngine()
+
+ engine._execute_bars(
+ dates, {"AAPL.US": bars}, close_df, targets, ["AAPL.US"]
+ )
+
+ actual = engine._actual_positions_frame(["AAPL.US"])
+ assert actual.loc[dates[1], "AAPL.US"] == pytest.approx(0.2)
+ assert engine.trades[0].exit_reason == "target_rebalance"
+ assert engine.trades[0].size == pytest.approx(6.0)
+ assert engine.trades[-1].size == pytest.approx(2.0)
+
+
+def test_positions_artifact_reports_fills_not_blocked_targets(tmp_path) -> None:
+ dates, bars, close_df, targets = _fractional_fixture([0.2, 0.8, 0.8])
+ engine = _FractionalEngine(block_adds_after_first=True)
+ engine._execute_bars(
+ dates, {"AAPL.US": bars}, close_df, targets, ["AAPL.US"]
+ )
+ equity = pd.Series(
+ [snapshot.equity for snapshot in engine.equity_snapshots], index=dates
+ )
+ benchmark_return = pd.Series(0.0, index=dates)
+ benchmark_equity = pd.Series(1_000.0, index=dates)
+
+ engine._write_artifacts(
+ tmp_path,
+ {"AAPL.US": bars},
+ dates,
+ equity,
+ benchmark_equity,
+ benchmark_return,
+ targets,
+ {},
+ ["AAPL.US"],
+ )
+
+ actual_csv = pd.read_csv(tmp_path / "artifacts" / "positions.csv", index_col=0)
+ target_csv = pd.read_csv(
+ tmp_path / "artifacts" / "target_positions.csv", index_col=0
+ )
+ assert actual_csv.iloc[1]["AAPL.US"] == pytest.approx(0.2)
+ assert target_csv.iloc[1]["AAPL.US"] == pytest.approx(0.8)
+
+
def _run_lifecycle(engine: _LifecycleEngine) -> None:
dates = pd.DatetimeIndex([pd.Timestamp("2026-01-02")])
frame = pd.DataFrame({"open": [100.0], "close": [100.0]}, index=dates)
diff --git a/agent/tests/test_risk_xray.py b/agent/tests/test_risk_xray.py
index 12636dd86..415cdc814 100644
--- a/agent/tests/test_risk_xray.py
+++ b/agent/tests/test_risk_xray.py
@@ -377,12 +377,17 @@ def test_run_backtest_emits_risk_xray_artifacts(tmp_path):
assert out_json.exists() and out_md.exists()
payload = json.loads(out_json.read_text(encoding="utf-8"))
_assert_strict_json(payload)
- # constant 1.0 signals normalize to an even two-name basket
- assert payload["concentration"]["hhi"] == pytest.approx(0.5)
+ # The opening target is even, but actual weights drift with each symbol's
+ # realized price path; the x-ray must reflect execution truth, not preserve
+ # the optimizer's idealized 50/50 weights.
+ assert payload["concentration"]["hhi"] == pytest.approx(0.5010936725)
assert set(payload["inputs"]["symbols"]) == {"AAA", "BBB"}
assert "# Portfolio Risk X-Ray" in out_md.read_text(encoding="utf-8")
- assert metrics["risk_xray_hhi"] == pytest.approx(0.5)
- assert metrics["risk_xray_effective_n"] == pytest.approx(2.0)
+ assert metrics["risk_xray_hhi"] == pytest.approx(payload["concentration"]["hhi"])
+ assert metrics["risk_xray_effective_n"] == pytest.approx(payload["concentration"]["effective_n"])
assert metrics["risk_xray_annualized_vol"] is not None
- assert metrics["risk_xray_avg_invested"] == pytest.approx(39 / 40, abs=1e-6)
+ # Execution truth has two flat observations: the initial next-bar-open
+ # signal lag and the terminal liquidation. The old target-frame report
+ # counted the latter as invested even though the position was closed.
+ assert metrics["risk_xray_avg_invested"] == pytest.approx(38 / 40, abs=1e-6)