Skip to content

"Vibe-Trading: Your Personal Trading Agent"

105 KB
Raw Blame
"""Base backtest engine with shared bar-by-bar execution loop.

All market engines inherit from BaseEngine and override market-rule methods.
The shared run_backtest() handles: data loading → signal generation →
pre-compute target weights (with optimizer) → bar-by-bar execution with
market rule enforcement → metrics → artifacts.
"""

from __future__ import annotations

import importlib
import json
import logging
import math
import re as _re
import sys
from abc import ABC, abstractmethod
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass, replace
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional

import numpy as np
import pandas as pd

from backtest.constraints import apply_constraints_frame, load_constraints
from backtest.loaders.rsshub_events import (
    FeedSpec,
    RSSHubEventProvider,
    enrich_price_frames_with_events,
    feed_specs_from_config,
)
from backtest.loaders.tushare_fundamentals import (
    SUBDAILY_POLICIES,
    SubdailyPitError,
    TushareFundamentalProvider,
    enrich_price_frames_with_fundamentals,
)
from backtest.metrics import (
    bar_returns,
    by_exit_reason_stats,
    by_symbol_stats,
    calc_fill_turnover_series,
    calc_metrics,
    entry_returns,
    period_returns,
)
from backtest.models import EquitySnapshot, FillRecord, Position, TradeRecord
from backtest.rebalance_mask import (
    RebalanceMask,
    resolve_rebalance_dates,
    validate_rebalance_mask,
)


def _json_safe_scalar_metrics(metrics: Dict[str, Any]) -> Dict[str, Any]:
    """Scalar metrics for stdout JSON; non-finite floats become null."""
    return {
        k: (None if isinstance(v, float) and not math.isfinite(v) else v)
        for k, v in metrics.items()
        if not isinstance(v, dict)
    }

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class _OpenOrder:
    """A fully priced opening order that can be committed atomically."""

    symbol: str
    direction: int
    price: float
    size: float
    leverage: float
    margin: float
    commission: float

    @property
    def cost(self) -> float:
        """Cash consumed by the fill."""
        return self.margin + self.commission


@dataclass(frozen=True)
class _ReductionOrder:
    before: Position
    target_size: float
    price: float
    released_margin: float
    realized_pnl: float
    exit_commission: float

    @property
    def capital_credit(self) -> float:
        return self.released_margin + self.realized_pnl - self.exit_commission


def _run_card_data_sources(config: Dict[str, Any], loader: Any) -> List[str]:
    """Return source names for run-card evidence."""
    configured = config.get("_run_card_effective_sources")
    if isinstance(configured, list):
        return [str(source) for source in configured if str(source).strip()]
    if isinstance(configured, str) and configured.strip():
        return [configured.strip()]

    loader_name = getattr(loader, "name", None)
    if loader_name:
        return [str(loader_name)]

    source = config.get("source")
    return [str(source)] if source else []


# ─── Market detection (lightweight, for signal alignment only) ───

_CRYPTO_RE = _re.compile(
    r"^(?=[A-Z0-9]*[A-Z])[A-Z0-9]+-USDT$|^(?=[A-Z0-9]*[A-Z])[A-Z0-9]+/USDT$", _re.I
)
# Forex / metals in their explicit Yahoo notations, plus the bare-6-char
# whitelist for XAUUSD / XAGUSD / XPTUSD / XPDUSD and G10 currencies. Mirrors
# ``backtest.engines._market_hooks._MARKET_PATTERNS`` so the ffill-limit
# decision (``10`` for cross-market vs ``5`` for single-market) reflects the
# same market classification the engine composite would assign.
_FX_RE = _re.compile(
    r"^[A-Z]{3}/[A-Z]{3}$"
    r"|^[A-Z]{6}\.FX$"
    r"|^[A-Z]{6}=X$"
    r"|^(?:XAU|XAG|XPT|XPD|EUR|GBP|JPY|CHF|CAD|AUD|NZD|USD)[A-Z]{3}$",
    _re.I,
)


def _detect_market_for_align(code: str) -> str:
    """Lightweight market detection for ffill_limit calculation."""
    from backtest.engines._market_hooks import _detect_market

    market = _detect_market(code.strip().upper())
    if market == "crypto":
        return "crypto"
    if market == "forex":
        return "forex"
    return "equity"


# ─── Forward-fill helpers (numpy, avoid pandas overhead) ───


def _ffill_1d(col: np.ndarray, limit: int) -> None:
    """In-place forward-fill a 1D array with limit."""
    count = 0
    last_valid = np.nan
    for i in range(len(col)):
        if np.isnan(col[i]):
            count += 1
            if count <= limit and not np.isnan(last_valid):
                col[i] = last_valid
        else:
            last_valid = col[i]
            count = 0


def _ffill_2d(arr: np.ndarray, limit: int = 5) -> np.ndarray:
    """Forward-fill NaN values column-wise with limit."""
    out = arr.copy()
    for col in range(out.shape[1]):
        _ffill_1d(out[:, col], limit)
    return out


# ─── Signal alignment (reused from daily_portfolio logic) ───


def evaluation_start_index(config: Dict[str, Any], dates: pd.DatetimeIndex) -> int:
    """Return the first bar index that counts as performance.

    A long-lookback indicator needs history before the period the user asked
    about — an MA200 strategy over ten years needs bars from before those ten
    years. Loading that history is correct; *grading* it is not. Without a
    declared boundary the extra bars silently join the evaluation: trades fire
    in them, the equity curve starts in them, and CAGR and the benchmark are
    computed over a window longer than the one that was requested. The run
    still succeeds and its metrics are still internally consistent, which is
    what makes it dangerous.

    Two spellings, because a strategy author knows one or the other: the number
    of warm-up bars the indicator needs, or the date the evaluation should
    start. Declaring both is refused rather than resolved — the two can
    disagree, and picking a winner would silently discard the author's other
    instruction.

    Args:
        config: Backtest configuration. Reads ``warmup_bars`` (int >= 0) and
            ``evaluation_start_date`` (``YYYY-MM-DD``); absent means the whole
            loaded window is evaluated, which is the historical behaviour.
        dates: The aligned bar index the backtest would otherwise run over.

    Returns:
        Index into ``dates`` of the first evaluated bar; ``0`` when nothing is
        declared.

    Raises:
        ValueError: Both keys declared, a malformed value, or a boundary that
            leaves fewer than two bars to evaluate.
    """
    warmup = config.get("warmup_bars")
    eval_start = config.get("evaluation_start_date")
    if warmup in (None, 0) and not eval_start:
        return 0
    if warmup not in (None, 0) and eval_start:
        raise ValueError(
            "declare warmup_bars or evaluation_start_date, not both — they can "
            "disagree, and resolving that silently would discard one of them"
        )

    if eval_start:
        boundary = pd.Timestamp(eval_start)
        if boundary.tz is None and dates.tz is not None:
            boundary = boundary.tz_localize(dates.tz)
        elif boundary.tz is not None and dates.tz is None:
            boundary = boundary.tz_localize(None)
        start = int(np.searchsorted(dates.values, boundary.to_datetime64(), side="left"))
        if start >= len(dates):
            raise ValueError(
                f"evaluation_start_date {eval_start} is after the last loaded bar "
                f"({dates[-1].date()}); nothing would be evaluated"
            )
    else:
        try:
            start = int(warmup)
        except (TypeError, ValueError):
            raise ValueError(f"warmup_bars must be an integer, got {warmup!r}") from None
        if start < 0:
            raise ValueError(f"warmup_bars must be non-negative, got {start}")

    if len(dates) - start < 2:
        raise ValueError(
            f"the evaluation window would hold {max(0, len(dates) - start)} bar(s) of "
            f"{len(dates)} loaded; widen start_date or shorten the warm-up"
        )
    return start


def _align(
    data_map: Dict[str, pd.DataFrame],
    signal_map: Dict[str, pd.Series],
    codes: List[str],
    optimizer: Optional[Callable] = None,
) -> tuple:
    """Build aligned date index, close matrix, target-position matrix, return matrix.

    Signal is shifted by 1 bar (next-bar-open semantics) then normalised so
    ``sum(abs(weights)) <= 1.0``.

    Args:
        data_map: code -> OHLCV DataFrame.
        signal_map: code -> signal Series.
        codes: Valid instrument codes.
        optimizer: Optional weight optimiser ``(ret, pos, dates) -> pos``.

    Returns:
        (dates, close_df, close_val_df, positions_df, returns_df). ``close_df``
        is the bounded-ffill trading view; ``close_val_df`` carries the last
        traded close through halts of any length and is only for valuation.
    """
    # Build unified sorted date index from all symbols' trading calendars.
    # Everything below works on int64 epochs, and ``asi8`` / ``view("i8")``
    # count in the index's own unit: a duckdb-backed local source arrives as
    # datetime64[us], which read as nanoseconds put the run in 1970 and, beside
    # a nanosecond source, matched none of its bars. One unit for all of them.
    ns_index = {
        c: idx if idx.unit == "ns" else idx.as_unit("ns")
        for c in codes
        for idx in (data_map[c].index,)
    }
    indexes = [ns_index[c] for c in codes]
    merged = np.unique(np.concatenate([index.asi8 for index in indexes]))
    common_tz = indexes[0].tz
    if all(index.tz == common_tz for index in indexes) and common_tz is not None:
        dates = pd.DatetimeIndex(pd.to_datetime(merged, utc=True).tz_convert(common_tz))
    else:
        dates = pd.DatetimeIndex(merged)

    n_dates = len(dates)
    n_codes = len(codes)

    # Use int64 view for O(log n) searchsorted lookups
    dates_i8 = dates.values.view("i8")
    # Daily panels usually share one calendar. Direct placement avoids two
    # binary-search passes per symbol over an already identical grid. Reuse
    # each irregular calendar's mapping for both prices and shifted signals.
    row_indexes = {
        c: slice(None) if ns_index[c].equals(dates)
        else np.searchsorted(dates_i8, ns_index[c].asi8)
        for c in codes
    }

    # ffill with limit to avoid masking long suspensions (e.g. 3-week halt)
    # Cross-market needs larger limit (Chinese New Year can be 9-10 bars)
    ffill_limit = 10 if len({_detect_market_for_align(c) for c in codes}) > 1 else 5

    # Build close matrix via numpy direct fill + searchsorted index mapping
    close_arr = np.full((n_dates, n_codes), np.nan)
    for j, c in enumerate(codes):
        series = data_map[c]["close"]
        close_arr[row_indexes[c], j] = series.values

    # Vectorized ffill with limit using pandas (C-optimized internals)
    _tmp = pd.DataFrame(close_arr)
    close_arr = _tmp.ffill(limit=ffill_limit).values
    # Valuation marks carry the last traded close through a halt of any
    # length; the bounded matrix above stays the trading/decision view.
    close_val_arr = _tmp.ffill().values

    # Drop symbols that are entirely NaN (no data overlap with date range)
    all_nan_mask = np.all(np.isnan(close_arr), axis=0)
    all_nan_cols = [codes[j] for j in range(n_codes) if all_nan_mask[j]]
    if all_nan_cols:
        logger.warning("Symbols dropped (no usable price data): %s", all_nan_cols)
        keep_mask = ~all_nan_mask
        codes = [codes[j] for j in range(n_codes) if keep_mask[j]]
        if not codes:
            raise ValueError("All symbols have no data in the requested date range")
        close_arr = close_arr[:, keep_mask]
        close_val_arr = close_val_arr[:, keep_mask]
        n_codes = len(codes)

    # Build position matrix: shift on each symbol's OWN calendar, then fill
    pos_arr = np.full((n_dates, n_codes), np.nan)
    for j, c in enumerate(codes):
        # Get signal values aligned to own trading calendar.
        # copy=True guarantees a writable array: with copy=False, an already
        # float64 source returns a read-only view (e.g. pandas copy-on-write),
        # which the in-place nan_to_num/clip below would reject.
        own_idx = data_map[c].index
        sig_vals = signal_map[c].reindex(own_idx).values.astype(np.float64, copy=True)
        # fillna(0) + clip in numpy
        np.nan_to_num(sig_vals, copy=False, nan=0.0)
        np.clip(sig_vals, -1.0, 1.0, out=sig_vals)
        # shift(1) + fillna(0): prepend 0, drop last
        shifted_vals = np.empty_like(sig_vals)
        shifted_vals[0] = 0.0
        shifted_vals[1:] = sig_vals[:-1]
        # Place into unified grid via searchsorted
        pos_arr[row_indexes[c], j] = shifted_vals

    # Vectorized ffill with limit using pandas (C-optimized)
    _tmp = pd.DataFrame(pos_arr)
    # ``.values`` may be a read-only view under pandas copy-on-write; take the
    # copy-returning nan_to_num (not in-place) so the fill never writes a
    # read-only destination.
    pos_arr = np.nan_to_num(_tmp.ffill(limit=ffill_limit).values, nan=0.0)

    # Construct DataFrames for return
    close = pd.DataFrame(close_arr, index=dates, columns=codes)
    close_val = pd.DataFrame(close_val_arr, index=dates, columns=codes)
    pos = pd.DataFrame(pos_arr, index=dates, columns=codes)
    ret = bar_returns(close, label="engine per-symbol returns")

    if optimizer is not None:
        pos = optimizer(ret, pos, dates)

    scale = pos.abs().sum(axis=1).clip(lower=1.0)
    pos = pos.div(scale, axis=0)

    return dates, close, close_val, pos, ret


def _basket_entry_returns(
    close_df: pd.DataFrame, data_map: Dict[str, pd.DataFrame], codes: List[str],
) -> pd.DataFrame:
    """Per-symbol returns of the equal-weight basket over the evaluated bars.

    Each symbol is bought at the open of its first traded bar in the window
    (``metrics.entry_returns``) and marked on the same close matrix the
    strategy is valued against.

    Args:
        close_df: Evaluated close matrix from ``_align``.
        data_map: code -> raw OHLCV DataFrame.
        codes: Basket symbols, columns of ``close_df``.

    Returns:
        DataFrame of per-bar returns shaped like ``close_df[codes]``.
    """
    columns = {}
    for code in codes:
        frame = data_map[code]
        index = frame.index if frame.index.unit == "ns" else frame.index.as_unit("ns")
        raw = frame.set_axis(index).reindex(close_df.index)
        opens = raw["open"] if "open" in raw.columns else None
        columns[code] = entry_returns(
            close_df[code], opens, raw["close"].notna(), label=f"benchmark basket {code}",
        )
    return pd.DataFrame(columns, index=close_df.index)


def _load_optimizer(config: Dict[str, Any]) -> Optional[Callable]:
    """Dynamically load an optimizer function from config.

    Args:
        config: Backtest configuration.

    Returns:
        Optimizer callable, or None.
    """
    opt_name = config.get("optimizer")
    constraints = load_constraints(config)
    if not opt_name:
        if constraints:
            print("[WARN] 'constraints' only act on optimizer output, "
                  "set 'optimizer' to use them; ignoring")
        return None
    opt_params = config.get("optimizer_params") or {}
    try:
        mod = importlib.import_module(f"backtest.optimizers.{opt_name}")
    except (ImportError, AttributeError) as e:
        print(f"[WARN] Failed to load optimizer '{opt_name}': {e}, falling back to equal weight")
        return None

    def optimize(ret: pd.DataFrame, pos: pd.DataFrame, dates: pd.DatetimeIndex) -> pd.DataFrame:
        out = mod.optimize(ret, pos, dates, **opt_params)
        return apply_constraints_frame(out, constraints)

    return optimize


def _normalise_fundamental_fields(config: Dict[str, Any]) -> dict[str, list[str]]:
    """Read the optional statement-table field map from backtest config."""
    raw_fields = config.get("fundamental_fields")
    if raw_fields in (None, {}):
        return {}
    if not isinstance(raw_fields, dict):
        raise ValueError("fundamental_fields must map table names to field-name lists")

    normalized: dict[str, list[str]] = {}
    for table, fields in raw_fields.items():
        if not isinstance(table, str) or not table.strip():
            raise ValueError("fundamental_fields table names must be non-empty strings")
        if fields is None:
            continue
        if isinstance(fields, str) or not isinstance(fields, Iterable):
            raise ValueError(f"fundamental_fields[{table!r}] must be a list of field names")

        field_list = list(fields)
        if not field_list:
            continue
        invalid = [field for field in field_list if not isinstance(field, str) or not field.strip()]
        if invalid:
            raise ValueError(f"fundamental_fields[{table!r}] contains invalid field names")
        normalized[table.strip()] = field_list
    return normalized


def _maybe_enrich_fundamentals(
    data_map: Dict[str, pd.DataFrame],
    config: Dict[str, Any],
) -> Dict[str, pd.DataFrame]:
    """Attach configured Tushare statement fields before signal generation."""
    fields_by_table = _normalise_fundamental_fields(config)
    if not fields_by_table:
        return data_map

    subdaily = str(config.get("fundamental_subdaily", "reject")).strip().lower()
    if subdaily not in SUBDAILY_POLICIES:
        raise ValueError(
            f"fundamental_subdaily must be one of {SUBDAILY_POLICIES}, got {subdaily!r}"
        )

    try:
        provider = TushareFundamentalProvider()
        return enrich_price_frames_with_fundamentals(
            data_map,
            provider,
            fields_by_table,
            as_of=config.get("end_date", ""),
            periods=config.get("fundamental_periods"),
            subdaily=subdaily,
        )
    except SubdailyPitError:
        # A contract error (an intraday frame under the default reject policy)
        # is the caller's to fix and must not be reworded as a provider
        # failure. Narrow on purpose: a stray ValueError from inside the
        # enrichment is a failure and keeps the wrapped message.
        raise
    except Exception as exc:
        raise RuntimeError(
            f"fundamental_fields requested but Tushare enrichment failed: {exc}"
        ) from exc


def _event_feed_specs(config: Dict[str, Any]) -> List[FeedSpec]:
    """Parse the optional ``event_feeds`` feed definitions from backtest config.

    ``event_feeds`` is a list of feed-definition dicts (there is no built-in
    catalogue) — each with ``name``/``route_template``/``event_type`` and an
    optional ``code_style``. An empty/absent value means "no event enrichment".
    """
    raw_feeds = config.get("event_feeds")
    if raw_feeds in (None, [], {}):
        return []
    if not isinstance(raw_feeds, (list, tuple)):
        raise ValueError("event_feeds must be a list of feed definitions")
    return feed_specs_from_config(raw_feeds)


def _maybe_enrich_events(
    data_map: Dict[str, pd.DataFrame],
    config: Dict[str, Any],
) -> Dict[str, pd.DataFrame]:
    """Attach a point-in-time-safe ``event_score`` column before signal generation."""
    specs = _event_feed_specs(config)
    if not specs:
        return data_map

    try:
        provider = RSSHubEventProvider(feeds=specs)
        if not provider.is_available():
            raise RuntimeError(f"RSSHub base URL not configured (set ${'RSSHUB_BASE_URL'})")
        return enrich_price_frames_with_events(
            data_map,
            provider,
            as_of=config.get("end_date", ""),
            decay_lambda=float(config.get("event_decay_lambda", 0.1)),
            lookback=int(config.get("event_lookback", 30)),
        )
    except Exception as exc:
        raise RuntimeError(
            f"event_feeds requested but RSSHub enrichment failed: {exc}"
        ) from exc


# ─── Base Engine ───


class BaseEngine(ABC):
    """Abstract base for all market engines.

    Subclasses override market-rule methods:
      - can_execute: whether a trade is allowed by market rules
      - round_size: lot-size rounding
      - calc_commission: fee structure
      - apply_slippage: slippage model
      - on_bar: per-bar hooks (funding fees, liquidation, etc.)
    """

    def __init__(self, config: dict):
        self.config = config
        self.initial_capital: float = config.get("initial_cash", 1_000_000)
        self.default_leverage: float = config.get("leverage", 1.0)
        self.position_adjustment = str(config.get("position_adjustment", "hold")).lower()
        if self.position_adjustment not in {"hold", "rebalance"}:
            raise ValueError("position_adjustment must be 'hold' or 'rebalance'")
        self.rebalance_mask: RebalanceMask = validate_rebalance_mask(
            config.get("rebalance_mask")
        )
        if self.rebalance_mask is not None and self.position_adjustment != "rebalance":
            raise ValueError(
                "rebalance_mask requires position_adjustment='rebalance'"
            )
        self.rebalance_bars_executed = 0
        # Relative drift band around the target weight. Zero reproduces the
        # historical behaviour, where the only thing separating "resize" from
        # "leave it alone" was the slippage width -- measured, a 0.01% daily
        # move re-pinned the position on 19 of 30 bars, which is noise being
        # traded, not a decision being executed.
        self.rebalance_tolerance = float(config.get("rebalance_tolerance", 0.0) or 0.0)
        if not math.isfinite(self.rebalance_tolerance) or self.rebalance_tolerance < 0.0:
            raise ValueError("rebalance_tolerance must be a finite, non-negative fraction")
        # Markets that clear at or below zero (e.g. EU day-ahead power) opt in
        # to opening on negative-price bars. Default False preserves the legacy
        # "reject any open_price <= 0" behavior. An exactly-zero open is always
        # rejected (undefined size = notional / price); negatives are handled by
        # abs()-based sizing and margin below.
        self.allow_nonpositive_prices: bool = bool(
            config.get("allow_nonpositive_prices", False)
        )
        #: Bar fields consulted, in order, for the price-limit base price.
        #: Futures engines put ``pre_settle`` first: exchanges set the band off
        #: the previous settlement, not the previous close.
        self.base_price_fields: tuple[str, ...] = ("pre_close",)
        self.capital: float = self.initial_capital
        self.positions: Dict[str, Position] = {}
        self.fill_records: List[FillRecord] = []
        self.trades: List[TradeRecord] = []
        self.equity_snapshots: List[EquitySnapshot] = []
        # Per-bar, post-fill portfolio weights.  These are deliberately kept
        # separate from ``target_pos``: market rules, lot rounding, fees, and
        # insufficient cash can all make the executed book differ from the
        # optimiser's request.
        self.actual_position_snapshots: List[tuple[pd.Timestamp, Dict[str, float]]] = []
        # Hold mode executes a target change only when the direction flips or
        # the target reaches zero, so a same-direction resize is dropped. That
        # is a legitimate mode -- it is what "enter once, hold to exit" means --
        # but dropping a request the strategy actually made must not be silent
        # (#918). The previous target is what makes the difference visible:
        # comparing against the CURRENT weight would fire on every bar of a
        # buy-and-hold position, whose weight drifts with price by design.
        self._last_target_weight: Dict[str, float] = {}
        self.dropped_target_adjustments: List[Dict[str, Any]] = []
        # Opening plans the engine wanted but could not take, keyed
        # (symbol, reason). A sleeve whose target notional rounds below one
        # lot trades zero times while the run reports a normal result
        # (#1235); counting the causes is what makes that visible in the
        # metrics instead of only to a subclass that overrode the hook.
        self.plan_rejections: Counter = Counter()
        self._bar_idx: int = 0
        self._bar_ts: Optional[pd.Timestamp] = None  # session being executed
        self._active_symbol: str = ""  # set by _rebalance/_close_position for subclass use
        # Positions the terminal close was not allowed to sell (see
        # can_liquidate_at_end). Empty for every market that keeps the default.
        self.unliquidated_positions: List[Dict[str, Any]] = []

    def available_cash(self, timestamp: pd.Timestamp) -> float:
        """Spendable funds for an order at timestamp, without changing equity.

        Default preserves legacy immediate-settlement capital across markets.
        Markets with unsettled receivables may override this hook.
        """
        return self.capital

    def projected_reduction_buying_power(
        self, reductions: list, timestamp: pd.Timestamp,
    ) -> float:
        """Incremental buying power from planned closes on this bar.

        Default is the legacy immediate-credit behavior. Settlement-based
        markets can return zero so a sale never finances a same-bar purchase.
        """
        return sum(order.capital_credit for order in reductions)

    def prepare_market_data(
        self, data_map: Dict[str, pd.DataFrame],
    ) -> Dict[str, pd.DataFrame]:
        """Validate or normalize loaded bars before signals see them.

        Default returns the frames untouched, so every market keeps its
        loader output. A market whose source data needs a documented unit
        convention (e.g. a depositary-receipt ratio change) overrides this;
        running before signal generation keeps signals and fills on the
        same price basis.
        """
        return data_map

    def can_liquidate_at_end(self, symbol: str, timestamp: pd.Timestamp) -> bool:
        """Whether the terminal forced close may sell *symbol* on the last bar.

        Default True keeps the legacy behavior of every market. A market whose
        securities cannot be sold before settlement overrides this so the
        end-of-backtest close does not bypass the rule that gates signal exits.
        Positions it refuses stay open and are valued, not traded.
        """
        return True

    # ── Market rule interface (subclass must implement) ──

    @abstractmethod
    def can_execute(self, symbol: str, direction: int, bar: pd.Series) -> bool:
        """Whether market rules allow this trade.

        Args:
            symbol: Instrument identifier.
            direction: 1 (long), -1 (short), 0 (close).
            bar: Current bar data (OHLCV + extras).

        Returns:
            True if allowed.
        """

    @abstractmethod
    def round_size(self, raw_size: float, price: float) -> float:
        """Round position size per market lot rules.

        Args:
            raw_size: Desired size.
            price: Current price.

        Returns:
            Rounded size.
        """

    @abstractmethod
    def calc_commission(self, size: float, price: float, direction: int, is_open: bool) -> float:
        """Calculate commission for a trade.

        Args:
            size: Trade size.
            price: Execution price.
            direction: 1 or -1.
            is_open: True for opening, False for closing.

        Returns:
            Commission amount.
        """

    @abstractmethod
    def apply_slippage(self, price: float, direction: int) -> float:
        """Apply slippage to execution price.

        Args:
            price: Raw price.
            direction: 1 (buying / covering short) or -1 (selling / shorting).

        Returns:
            Slipped price.
        """

    def historical_base_price(self, symbol: str, bar: pd.Series) -> Optional[float]:
        """Return a reference price that is known before this bar's fill.

        Price-limit bands must be derived from a base price the market already
        knew when the order was placed. ``can_execute`` runs before the fill,
        and this engine fills at the CURRENT bar's open, so the current bar's
        close is not available information: using it is lookahead.

        Both sources here are strictly historical:
          1. the first field of :attr:`base_price_fields` present on the bar —
             ``pre_close`` for cash equity, ``pre_settle`` first for futures,
             whose exchanges set the band off the previous settlement;
          2. the previous row of the close panel the run pre-extracts, which is
             absent in a cross-market composite run (its sub-engines are
             stateless rule books with no panel of their own).

        Args:
            symbol: Symbol whose base price is wanted.
            bar: Current bar.

        Returns:
            The base price, or None when no historical close is reachable
            (first bar of a run, or a composite sub-engine).
        """
        for field in self.base_price_fields:
            if field in bar.index:
                raw = bar[field]
                if pd.notna(raw) and float(raw) > 0:
                    return float(raw)

        close_arr = getattr(self, "_close_arr", None)
        col = getattr(self, "_code_to_col", {}).get(symbol)
        row = getattr(self, "_bar_idx", 0) - 1
        if close_arr is not None and col is not None and row >= 0:
            value = close_arr[row, col]
            if pd.notna(value) and float(value) > 0:
                return float(value)

        # Last resort: reconstruct the prior close from tushare's pct_chg, which
        # is in percentage points. Both inputs sit on the current bar, but their
        # ratio is the PREVIOUS close, so the result is still historical.
        if "pct_chg" in bar.index and "close" in bar.index:
            pct, close = bar["pct_chg"], bar["close"]
            if pd.notna(pct) and pd.notna(close) and float(close) > 0:
                denominator = 1.0 + float(pct) / 100.0
                if denominator > 0:
                    return float(close) / denominator
        return None

    def prospective_fill_price(self, bar: pd.Series, direction: int) -> Optional[float]:
        """Return the price this engine would fill at on this bar.

        Mirrors the sizing path: the bar's open, with slippage applied in the
        trade direction. Comparing THIS against a price-limit band keeps the
        simulation from ever transacting outside the exchange's legal range.

        Args:
            bar: Current bar.
            direction: 1 (buy / cover), -1 (sell / short), 0 (close).

        Returns:
            The prospective fill price, or None when the bar has no usable open.
        """
        raw = bar.get("open", bar.get("close"))
        if raw is None or pd.isna(raw):
            return None
        price = float(raw)
        if price <= 0 and not self.allow_nonpositive_prices:
            return None
        return self.apply_slippage(price, direction)

    def limit_band(
        self, symbol: str, bar: pd.Series, limit: float
    ) -> Optional[tuple[float, float]]:
        """Return the (lower, upper) legal price band for this bar.

        Args:
            symbol: Symbol whose band is wanted.
            bar: Current bar.
            limit: Band half-width as a fraction (0.1 for +/-10%).

        Returns:
            The (lower, upper) band, or None when no historical base price is
            reachable — in which case the caller must not fabricate a block.
        """
        base = self.historical_base_price(symbol, bar)
        if base is None or base <= 0 or not limit:
            return None
        return base * (1.0 - float(limit)), base * (1.0 + float(limit))

    def on_bar(self, symbol: str, bar: pd.Series, timestamp: pd.Timestamp) -> None:
        """Per-bar market-rule hook (funding fees, liquidation, etc.).

        Default: no-op. Override in subclass as needed.
        """

    def before_rebalance_bar(
        self,
        timestamp: pd.Timestamp,
        data_map: Dict[str, pd.DataFrame],
        codes: List[str],
    ) -> bool:
        """Run pre-execution hooks; return True to stop after this snapshot."""
        return False

    def after_rebalance_bar(
        self,
        timestamp: pd.Timestamp,
        data_map: Dict[str, pd.DataFrame],
        codes: List[str],
    ) -> bool:
        """Run the legacy post-fill bar hooks; return True to stop."""
        for code in codes:
            if timestamp in data_map[code].index:
                self.on_bar(code, data_map[code].loc[timestamp], timestamp)
        return False

    def after_position_adjustment(
        self,
        *,
        action: str,
        timestamp: pd.Timestamp,
        before: Position,
        after: Position,
        execution_price: float,
        trading_fee: float,
        realized_pnl: float = 0.0,
        released_margin: float = 0.0,
    ) -> None:
        """Allow engines to update risk state/evidence after a committed delta fill."""
        return None

    #: Rejection causes that mean the engine WANTED a position and could not
    #: take it. ``no_target_weight`` and ``already_held`` are excluded: they
    #: mean nothing was wanted, which is not a finding.
    UNFILLED_PLAN_REASONS = (
        "no_data",
        "no_bar",
        "execution_blocked",
        "invalid_price",
        "zero_size",
        "insufficient_capital",
    )

    def _plan_rejection_metrics(self) -> Dict[str, Any]:
        """Summarise opening plans the engine wanted but could not take.

        A rejected plan is invisible in the result otherwise: the run reports a
        normal equity curve over the symbols that did fill, so a sleeve whose
        target notional never clears one lot silently drops out of the book and
        the configuration is graded as if it had never contained that sleeve
        (#1235). Only the causes that mean "wanted but unfillable" are counted.

        Returns:
            ``unfilled_plan_rejections`` (total) and
            ``unfilled_plan_rejections_by_symbol`` (``{symbol: {reason: n}}``),
            the latter empty when nothing was rejected.
        """
        by_symbol: Dict[str, Dict[str, int]] = {}
        total = 0
        for (symbol, reason), count in self.plan_rejections.items():
            if reason not in self.UNFILLED_PLAN_REASONS:
                continue
            by_symbol.setdefault(symbol, {})[reason] = count
            total += count
        return {
            "unfilled_plan_rejections": total,
            "unfilled_plan_rejections_by_symbol": by_symbol,
        }

    def _engine_diagnostics(self) -> Dict[str, Any]:
        """Per-engine facts about how this run was priced.

        Default empty. An engine overrides this to state something the metrics
        cannot be derived from — currently ChinaFuturesEngine reporting which
        products it priced on a generic default instead of a table entry
        (#1393), where the alternative is a number that looks like data.

        Returns:
            Extra keys merged into the metrics dict; empty when there is
            nothing to declare.
        """
        return {}

    def _on_plan_rejected(self, symbol: str, reason: str, timestamp: pd.Timestamp) -> None:
        """Observe a silently rejected opening-order plan.

        ``_plan_open_order`` returns ``None`` for several distinct reasons —
        nothing wanted, already held, missing data, missing bar, market rule
        block, unusable price, or a target too small to fill after lot
        rounding. Callers only see ``None``, so this hook is the only way for
        an engine subclass to tell "nothing to do" apart from "wanted but
        unfillable".

        The capital fit reports one more cause after the planner: a sleeve that
        was a real order at full scale and left the basket when every opening
        order was scaled down to the cash available is ``insufficient_capital``
        (#1470), reported once per bar — never the trial plans of the search.

        Args:
            symbol: Instrument the plan was for.
            reason: Machine-readable cause: ``no_target_weight``,
                ``already_held``, ``no_data``, ``no_bar``,
                ``execution_blocked``, ``invalid_price``, ``zero_size`` or
                ``insufficient_capital``.
            timestamp: Decision bar timestamp.
        """
        self.plan_rejections[(symbol, reason)] += 1

    def execution_open(self, bar: pd.Series) -> float:
        """Return the normal market-fill price for a bar."""
        return float(bar.get("open", bar.get("close", 0)))

    def valuation_open(self, bar: pd.Series) -> float:
        """Return the price observable when open orders are sized."""
        return float(bar.get("open", bar.get("close", 0)))

    # ── PnL / margin calculation hooks ──
    # Override in FuturesBaseEngine to inject contract multiplier.

    def _calc_pnl(
        self, symbol: str, direction: int, size: float,
        entry_price: float, exit_price: float,
    ) -> float:
        """Realised PnL for a closed position."""
        return direction * size * (exit_price - entry_price)

    def _calc_margin(
        self, symbol: str, size: float, price: float, leverage: float,
    ) -> float:
        """Margin (collateral) required for a position.

        ``abs(price)`` so collateral stays positive when the entry price is
        negative; for the usual positive price this is unchanged.
        """
        return size * abs(price) / leverage

    def _calc_raw_size(
        self, symbol: str, target_notional: float, price: float,
    ) -> float:
        """Convert target notional exposure to number of units/contracts.

        Size is a positive magnitude — direction carries the sign elsewhere —
        so divide by ``abs(price)``; a negative entry price must not flip the
        size negative (which the ``size <= 0`` guard would then reject). For a
        positive price this is unchanged.
        """
        return target_notional / abs(price)

    def _leverage_for_symbol(self, symbol: str) -> float:
        """Return leverage used to size and margin one symbol."""
        del symbol
        return self.default_leverage

    # ── Main entry ──

    def run_backtest(
        self,
        config: Dict[str, Any],
        loader: Any,
        signal_engine: Any,
        run_dir: Path,
        bars_per_year: int | None = 252,
    ) -> Dict[str, Any]:
        """Full backtest pipeline.

        Signature matches ``daily_portfolio.run_backtest`` for drop-in replacement.

        Args:
            config: Backtest configuration dict.
            loader: DataLoader with ``fetch()`` method.
            signal_engine: SignalEngine with ``generate()`` method.
            run_dir: Artifacts output directory.
            bars_per_year: Annualisation factor.

        Returns:
            Metrics dictionary.
        """
        trace_started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
        codes = config.get("codes", [])
        interval = config.get("interval", "1D")
        extra_fields = config.get("extra_fields") or None

        # 1. Load data
        data_map = loader.fetch(
            codes,
            config.get("start_date", ""),
            config.get("end_date", ""),
            fields=extra_fields,
            interval=interval,
        )
        if not data_map:
            print(json.dumps({"error": "No data fetched"}))
            sys.exit(1)
        data_map = _maybe_enrich_fundamentals(data_map, config)
        data_map = _maybe_enrich_events(data_map, config)
        data_map = self.prepare_market_data(data_map)

        # 2. Generate signals
        signal_map = signal_engine.generate(data_map)
        if not isinstance(signal_map, dict):
            print(json.dumps({"error": (
                f"SignalEngine.generate() must return Dict[str, pd.Series], "
                f"got {type(signal_map).__name__}. "
                "Return a dict mapping symbol codes to pandas Series of signals."
            )}))
            sys.exit(1)
        for _code, _sig in signal_map.items():
            if not isinstance(_sig, pd.Series):
                print(json.dumps({"error": (
                    f"SignalEngine.generate() returned {type(_sig).__name__} for '{_code}', "
                    "expected pd.Series. Each value must be a pandas Series with DatetimeIndex."
                )}))
                sys.exit(1)
        valid_codes = sorted(c for c in signal_map if c in data_map)
        if not valid_codes:
            print(json.dumps({"error": "No valid signals generated"}))
            sys.exit(1)

        # 3. Pre-compute target weights (with optimizer)
        opt_fn = _load_optimizer(config)
        dates, close_df, close_val_df, target_pos, ret_df = _align(
            data_map, signal_map, valid_codes, optimizer=opt_fn,
        )

        # Sync codes after _align may have dropped all-NaN symbols
        valid_codes = [c for c in valid_codes if c in target_pos.columns]

        # 3b. Drop the warm-up prefix from the evaluation window. Signals were
        # generated over the whole loaded panel above, so the first evaluated
        # bar already carries an indicator that saw its full lookback; from here
        # on the warm-up bars simply do not exist, so no trade, no equity point,
        # no benchmark return and no metric can come from them.
        warmup_end = evaluation_start_index(config, dates)
        if warmup_end:
            dates = dates[warmup_end:]
            close_df = close_df.iloc[warmup_end:]
            close_val_df = close_val_df.iloc[warmup_end:]
            target_pos = target_pos.iloc[warmup_end:]
            ret_df = ret_df.iloc[warmup_end:]

        # 4. Bar-by-bar execution
        self._execute_bars(dates, data_map, close_df, target_pos, valid_codes, close_val_df=close_val_df)
        actual_pos = self._actual_positions_frame(valid_codes)

        # 5. Build output series
        equity_series = pd.Series(
            [s.equity for s in self.equity_snapshots],
            index=[s.timestamp for s in self.equity_snapshots],
        )
        # ret_df runs from each bar's previous close, so its first evaluated row
        # starts at the last warm-up close. The basket instead buys each symbol
        # at its first traded open in the window, as the strategy can.
        basket_ret = _basket_entry_returns(close_df, data_map, list(ret_df.columns))
        bench_ret = basket_ret.mean(axis=1) if basket_ret.shape[1] > 0 else pd.Series(0.0, index=dates)
        from backtest.benchmark import default_benchmark, resolve_benchmark

        basket_codes = list(ret_df.columns)
        basket_label = (
            f"equal-weight basket ({basket_codes[0]})" if len(basket_codes) == 1
            else f"equal-weight basket of {len(basket_codes)} symbols"
        )
        benchmark_metadata = {"benchmark_kind": "equal_weight", "benchmark_label": basket_label}
        requested = config.get("benchmark")
        source = config.get("source", "yfinance")
        ticker = default_benchmark(basket_codes, source) if requested == "auto" else requested
        reason = "no default benchmark for the evaluated symbols" if requested == "auto" else "no benchmark requested"
        if ticker:
            result = resolve_benchmark(
                strategy_codes=basket_codes, source=source, explicit=ticker,
                start_date=config.get("start_date", ""), end_date=config.get("end_date", ""),
                interval=interval, loader=loader if source != "auto" else None,
            )
            window_return = result.total_return_over(dates) if result is not None else None
            if result is not None and window_return is not None:
                bench_ret = result.returns_over(dates)
                benchmark_metadata = {
                    "benchmark_kind": "external", "benchmark_label": result.ticker,
                    "benchmark_ticker": result.ticker, "benchmark_return": window_return,
                }
            else:
                reason = f"benchmark {ticker!r} could not be resolved over the evaluation window"
        benchmark_warning = (
            f"{reason}; benchmark-relative metrics use the {basket_label}"
            if benchmark_metadata["benchmark_kind"] == "equal_weight" else None
        )

        bench_equity = self.initial_capital * (1 + bench_ret).cumprod()

        # 6. Metrics
        realized_turnover = calc_fill_turnover_series(self.fill_records, equity_series)
        m = calc_metrics(
            equity_series,
            self.trades,
            self.initial_capital,
            bars_per_year,
            bench_ret,
            actual_pos,
            turnover_series=realized_turnover,
        )
        m.update(benchmark_metadata)
        if "benchmark_return" in benchmark_metadata:
            # calc_metrics()'s own excess_return is derived from bench_ret
            # compounded as (1 + bench_ret).prod() - 1, the same pattern #872
            # found unsafe once a benchmark price series contains a
            # non-positive-prior-price bar. benchmark_return above is already
            # corrected to the #872-safe price relative; excess_return has to
            # be re-derived from that same corrected value or the two fields
            # go inconsistent with each other in the same metrics dict.
            m["benchmark_return"] = round(
                benchmark_metadata["benchmark_return"], 6
            )
            m["excess_return"] = round(
                m["total_return"] - benchmark_metadata["benchmark_return"], 6
            )
        m.update(self._plan_rejection_metrics())
        m.update(self._engine_diagnostics())
        if self.rebalance_mask is not None:
            m["rebalance_mask"] = self.rebalance_mask
            m["rebalance_bars_executed"] = self.rebalance_bars_executed
        m["by_symbol"] = by_symbol_stats(self.trades)
        m["by_exit_reason"] = by_exit_reason_stats(self.trades)

        # Portfolio Studio: per-rebalance weight-drift notes from the target
        # positions. Optimizer-agnostic, so they land for the baseline too.
        from backtest.rebalance_notes import (
            compute_rebalance_notes,
            compute_rebalance_execution_evidence,
            render_rebalance_notes_markdown,
            write_rebalance_notes,
        )
        rebalance_notes = compute_rebalance_notes(target_pos)
        # The requested count above cannot be read as executions (#1275): under
        # position_adjustment="rebalance" a constant target re-pins the book on
        # many bars, so report what the immutable fills actually did alongside
        # what the strategy asked for.
        rebalance_notes["summary"].update(
            compute_rebalance_execution_evidence(self.fill_records, equity_series)
        )
        write_rebalance_notes(run_dir / "artifacts" / "rebalance_notes.json", rebalance_notes)
        (run_dir / "artifacts" / "rebalance_notes.md").write_text(
            render_rebalance_notes_markdown(rebalance_notes), encoding="utf-8"
        )
        m["target_change_count"] = rebalance_notes["summary"]["target_change_count"]
        m["rebalance_executed_bars"] = rebalance_notes["summary"]["rebalance_executed_bars"]
        m["rebalance_executed_fills"] = rebalance_notes["summary"]["rebalance_executed_fills"]
        m["rebalance_realized_turnover"] = rebalance_notes["summary"]["rebalance_realized_turnover"]
        # Requested (target-delta) turnover, kept from the notes summary.
        m["rebalance_turnover_mean"] = rebalance_notes["summary"]["turnover_mean"]
        m["rebalance_turnover_max"] = rebalance_notes["summary"]["turnover_max"]

        # The notes above count what the STRATEGY asked for. Under
        # position_adjustment="hold" a same-direction resize is not executed,
        # so a run can report rebalances whose weight changes never reached the
        # book. Say which ones, rather than leaving the reader to reconcile a
        # rebalance count against a trade log that does not match it (#918).
        m["position_adjustment"] = self.position_adjustment
        m["rebalance_tolerance"] = self.rebalance_tolerance
        m["dropped_target_adjustment_count"] = len(self.dropped_target_adjustments)
        m["dropped_target_adjustments"] = [
            {
                "timestamp": str(event["timestamp"]),
                "symbol": event["symbol"],
                "previous_target_weight": round(event["previous_target_weight"], 6),
                "requested_target_weight": round(event["requested_target_weight"], 6),
            }
            for event in self.dropped_target_adjustments[:20]
        ]
        if self.dropped_target_adjustments:
            logger.warning(
                "position_adjustment='hold' dropped %d target change(s) across %d "
                "symbol(s); the report's target_change_count describes requests, not "
                "fills. Set position_adjustment='rebalance' to execute them.",
                len(self.dropped_target_adjustments),
                len({event["symbol"] for event in self.dropped_target_adjustments}),
            )

        # Portfolio Studio: risk x-ray over the strategy's average basket.
        from backtest.risk_xray import (
            average_invested_weights,
            compute_risk_xray,
            render_risk_xray_markdown,
            write_risk_xray,
        )
        xray_warning = None
        try:
            basket_weights, avg_invested = average_invested_weights(actual_pos)
            risk_xray = compute_risk_xray(
                close_df, basket_weights, periods_per_year=bars_per_year,
            )
        except ValueError as exc:
            xray_warning = f"Risk x-ray unavailable: {exc}"
        else:
            write_risk_xray(run_dir / "artifacts" / "risk_xray.json", risk_xray)
            (run_dir / "artifacts" / "risk_xray.md").write_text(
                render_risk_xray_markdown(risk_xray), encoding="utf-8"
            )
            m["risk_xray_hhi"] = risk_xray["concentration"]["hhi"]
            m["risk_xray_effective_n"] = risk_xray["concentration"]["effective_n"]
            m["risk_xray_annualized_vol"] = risk_xray["volatility"]["annualized_vol"]
            m["risk_xray_max_drawdown"] = risk_xray["drawdown"]["max_drawdown"]
            m["risk_xray_avg_invested"] = avg_invested

        # 7. Validation (optional — triggered by config["validation"])
        if config.get("validation"):
            from backtest.validation import run_validation, write_validation_json
            v_results = run_validation(
                config, equity_series, self.trades, self.initial_capital, bars_per_year,
            )
            m["validation"] = v_results
            # Write validation.json through the shared strict writer so a
            # non-finite validation metric is serialized as null rather than an
            # invalid bare NaN/Infinity token (matching the standalone
            # `python -m backtest.validation` path and run_card). The writer
            # also creates the artifacts dir, which step 8 otherwise creates.
            write_validation_json(run_dir / "artifacts" / "validation.json", v_results)

        # 8. Artifacts
        self._write_artifacts(
            run_dir, data_map, dates, equity_series, bench_equity, bench_ret,
            target_pos, m, valid_codes,
        )

        # 9. Trust Layer run card
        card_warnings = list(config.get("content_filter_warnings") or [])
        if config.get("_run_card_caliber_warning"):
            card_warnings.append(config["_run_card_caliber_warning"])
        if config.get("_run_card_annualisation_warning"):
            card_warnings.append(config["_run_card_annualisation_warning"])
        if config.get("_run_card_unrecognized_config_warning"):
            card_warnings.append(config["_run_card_unrecognized_config_warning"])
        if xray_warning:
            card_warnings.append(xray_warning)
        if benchmark_warning:
            card_warnings.append(benchmark_warning)
        from backtest.run_card import write_run_card
        write_run_card(
            run_dir,
            config,
            m,
            data_sources=_run_card_data_sources(config, loader),
            strategy_path=run_dir / "code" / "signal_engine.py",
            warnings=card_warnings or None,
            tool_traces=[
                {
                    "tool": "backtest",
                    "args": config,
                    "started_at": trace_started_at,
                    "ended_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
                    "status": "ok",
                    "result": m,
                }
            ],
        )

        # Print scalar metrics (skip nested dicts for JSON compat).
        # Explosive annual_return may be +inf; match options/run_card and emit
        # null instead of a bare Infinity token (invalid RFC-8259 JSON).
        print(json.dumps(_json_safe_scalar_metrics(m), indent=2, allow_nan=False))
        return m

    # ── Execution loop ──

    def _execute_bars(
        self,
        dates: pd.DatetimeIndex,
        data_map: Dict[str, pd.DataFrame],
        close_df: pd.DataFrame,
        target_pos: pd.DataFrame,
        codes: List[str],
        close_val_df: Optional[pd.DataFrame] = None,
    ) -> None:
        """Bar-by-bar execution with market rule enforcement."""
        # Pre-extract numpy arrays for O(1) indexed access instead of DataFrame.at[]
        # Explicit column reindex ensures array column order matches codes parameter,
        # regardless of DataFrame internal column ordering (which may be alphabetical).
        _target_arr = target_pos[codes].values  # (n_dates, n_codes) ndarray
        _close_arr = close_df[codes].values  # (n_dates, n_codes) ndarray
        if close_val_df is None:
            close_val_df = close_df.ffill()
        _val_arr = close_val_df[codes].values  # unbounded ffill, valuation only
        _code_to_col = {c: j for j, c in enumerate(codes)}
        # Store as instance attrs for use in _calc_equity / _safe_price
        self._close_arr = _close_arr
        self._val_arr = _val_arr
        self._code_to_col = _code_to_col
        self.actual_position_snapshots = []
        execution_dates = resolve_rebalance_dates(self.rebalance_mask, dates)
        self.rebalance_bars_executed = 0

        for i, ts in enumerate(dates):
            self._bar_idx = i
            self._bar_ts = ts

            stop_run = self.before_rebalance_bar(ts, data_map, codes)
            execute_targets = execution_dates is None or ts in execution_dates

            # a. Value the book at prices observable when orders execute.
            # Rebalances happen at the bar open, so using close_df[ts] here
            # would let the yet-unknown decision-bar close affect order size.
            equity = self._calc_open_equity(data_map, close_df, ts)
            target_weights: Dict[str, Optional[float]] = {}
            for c in codes:
                try:
                    val = _target_arr[i, _code_to_col[c]]
                    target_weights[c] = (
                        None
                        if stop_run or not execute_targets
                        else (float(val) if not np.isnan(val) else 0.0)
                    )
                except Exception as exc:
                    target_weights[c] = None
                    logger.warning("Target weight failed for %s at %s: %s", c, ts, exc)

            if self.position_adjustment == "rebalance":
                if execute_targets and not stop_run:
                    self._execute_target_rebalance(
                        target_weights, data_map, ts, equity, codes
                    )
                    self.rebalance_bars_executed += 1
                target_weights = {}
            else:
                self._record_dropped_target_adjustments(target_weights, ts)

            # b. In legacy hold mode, release capital before opening replacement
            # positions or increasing other positions.  A
            # single mixed close/open pass makes rotations depend on symbol
            # iteration order when the new name is visited before the old one.
            for c in codes:
                target_w = target_weights.get(c)
                current_pos = self.positions.get(c)
                if target_w is None or current_pos is None:
                    continue
                target_dir = 1 if target_w > 1e-9 else (-1 if target_w < -1e-9 else 0)
                if target_dir == 0 or target_dir != current_pos.direction:
                    try:
                        self._rebalance(c, 0.0, data_map.get(c), ts, equity)
                    except Exception as exc:
                        logger.warning(
                            "Rebalance close failed for %s at %s: %s", c, ts, exc
                        )

            # c. Price every opening order before committing any of them.  If
            # the requested basket does not fit after fees/lot rounding, apply
            # one common scale factor to all target weights.  This preserves
            # portfolio proportions and makes fills independent of input code
            # order; sequential cash clipping would privilege the first name.
            open_targets: list[tuple[str, float, Optional[pd.DataFrame]]] = []
            for c in sorted(codes):
                target_w = target_weights.get(c)
                if target_w is None:
                    continue
                target_dir = 1 if target_w > 1e-9 else (-1 if target_w < -1e-9 else 0)
                current_pos = self.positions.get(c)
                if current_pos is not None and (
                    target_dir == 0 or target_dir != current_pos.direction
                ):
                    continue
                if current_pos is None and target_dir != 0:
                    open_targets.append((c, target_w, data_map.get(c)))

            def _plans(scale: float, *, observe: bool) -> list[_OpenOrder]:
                result: list[_OpenOrder] = []
                for c, target_w, frame in open_targets:
                    try:
                        order = self._plan_open_order(
                            c, target_w * scale, frame, ts, equity, observe=observe
                        )
                    except Exception as exc:
                        logger.warning(
                            "Rebalance open plan failed for %s at %s: %s",
                            c,
                            ts,
                            exc,
                        )
                        continue
                    if order is not None:
                        result.append(order)
                return result

            planned = _plans(1.0, observe=True)
            if sum(order.cost for order in planned) > self.available_cash(ts) + 1e-9:
                # The trial plans of the search are not findings: one starved
                # open used to be booked as a zero_size rejection on every
                # bisection step (#1470). The sleeve that leaves the basket is
                # reported once, after the search, as the rebalance path does.
                wanted = [order.symbol for order in planned]
                # Nothing fits until a candidate proves it does: with cash below
                # zero (a funding debit leaves it there) not even the empty plan
                # fits, and the full-scale plan used to fall through to
                # _execute_open_order and abort the run (#1542).
                fitting: list[_OpenOrder] = []
                low, high = 0.0, 1.0
                for _ in range(50):
                    mid = (low + high) / 2.0
                    candidate = _plans(mid, observe=False)
                    if sum(order.cost for order in candidate) <= self.available_cash(ts) + 1e-9:
                        low, fitting = mid, candidate
                    else:
                        high = mid
                planned = fitting
                fitted = {order.symbol for order in planned}
                for symbol in wanted:
                    if symbol not in fitted:
                        self._on_plan_rejected(symbol, "insufficient_capital", ts)

            for order in planned:
                self._execute_open_order(order, ts)

            # d. Apply post-execution hooks after all normal market fills.
            if not stop_run:
                stop_run = self.after_rebalance_bar(ts, data_map, codes)

            # e. Record equity snapshot
            snap_equity = self._calc_equity(close_df, ts)
            self.actual_position_snapshots.append(
                (ts, self._actual_weights(close_df, ts, codes, snap_equity))
            )
            if self.positions and type(self)._calc_pnl is BaseEngine._calc_pnl:
                _syms = list(self.positions.keys())
                _eps = np.array([p.entry_price for p in self.positions.values()])
                _dirs = np.array([p.direction for p in self.positions.values()])
                _sizes = np.array([p.size for p in self.positions.values()])
                _cps = np.array([
                    self._safe_price(
                        close_df, ts, s, ep,
                        _arr=_close_arr, _row=i, _col=_code_to_col.get(s),
                        _val_arr=_val_arr,
                    )
                    for s, ep in zip(_syms, _eps)
                ])
                total_unrealized = float(np.sum(_dirs * _sizes * (_cps - _eps)))
            else:
                total_unrealized = 0.0
                for p in self.positions.values():
                    cp = self._safe_price(
                        close_df, ts, p.symbol, p.entry_price,
                        _arr=_close_arr, _row=i, _col=_code_to_col.get(p.symbol),
                        _val_arr=_val_arr,
                    )
                    total_unrealized += self._calc_pnl(p.symbol, p.direction, p.size, p.entry_price, cp)
            self.equity_snapshots.append(EquitySnapshot(
                timestamp=ts,
                capital=self.capital,
                unrealized=total_unrealized,
                equity=snap_equity,
                positions=len(self.positions),
            ))

            if stop_run:
                break

        # f. Force close all remaining positions
        if len(dates) > 0:
            _last_row = min(self._bar_idx, len(dates) - 1)
            last_ts = dates[_last_row]
            self.unliquidated_positions = []
            retained_value = 0.0
            for c in list(self.positions.keys()):
                pos = self.positions[c]
                mark_price = self._safe_price(
                    close_df, last_ts, c, pos.entry_price,
                    _arr=_close_arr, _row=_last_row, _col=_code_to_col.get(c),
                    _val_arr=_val_arr,
                )
                self._active_symbol = c
                exit_price = self.apply_slippage(mark_price, -pos.direction)
                if not self.can_liquidate_at_end(c, last_ts):
                    # The market's sell rule (e.g. unsettled securities) also
                    # binds the terminal close: keep the position open and
                    # value it at what a close would net, without booking a
                    # trade the rules would not have allowed.
                    proceeds = self._close_proceeds(pos, exit_price)
                    retained_value += proceeds
                    self.unliquidated_positions.append({
                        "symbol": c,
                        "size": pos.size,
                        "entry_price": pos.entry_price,
                        "mark_price": mark_price,
                        "estimated_exit_price": exit_price,
                        "estimated_net_proceeds": proceeds,
                    })
                    continue
                self._close_position(c, exit_price, last_ts, "end_of_backtest")

            # The final snapshot feeds metrics and artifacts.  Replace its
            # pre-liquidation mark with post-liquidation cash so terminal
            # slippage and exit commission are reflected in reported equity.
            if self.equity_snapshots:
                terminal_equity = self.capital + retained_value
                self.equity_snapshots[-1] = EquitySnapshot(
                    timestamp=last_ts,
                    capital=self.capital,
                    unrealized=terminal_equity - self.capital - sum(
                        self._calc_margin(p.symbol, p.size, p.entry_price, p.leverage)
                        for p in self.positions.values()
                    ),
                    equity=terminal_equity,
                    positions=len(self.positions),
                )
            if self.actual_position_snapshots:
                final_weights = {code: 0.0 for code in codes}
                if self.positions and self.equity_snapshots[-1].equity > 1e-12:
                    final_weights = self._actual_weights(
                        close_df, last_ts, codes, self.equity_snapshots[-1].equity
                    )
                self.actual_position_snapshots[-1] = (last_ts, final_weights)

        # Clean up temporary instance attributes
        self._close_arr = None
        self._val_arr = None
        self._code_to_col = None

    def _calc_open_equity(
        self,
        data_map: Dict[str, pd.DataFrame],
        close_df: pd.DataFrame,
        ts: pd.Timestamp,
    ) -> float:
        """Value current positions at the execution bar's observable open.

        For a symbol that has a bar at ``ts``, its open is the mark available
        when next-bar-open orders execute.  Symbols without a bar on the
        unified calendar retain the aligned close fallback, which is the most
        recent price carried by ``_align``.
        """
        if not self.positions:
            return self.capital

        equity = self.capital
        for sym, pos in self.positions.items():
            _arr = getattr(self, "_close_arr", None)
            _varr = getattr(self, "_val_arr", None)
            _row = getattr(self, "_bar_idx", None)
            _c2c = getattr(self, "_code_to_col", None)
            current_price = self._safe_price(
                close_df, ts, sym, pos.entry_price,
                _arr=_arr, _row=_row, _col=(_c2c.get(sym) if _c2c else None),
                _val_arr=_varr,
            )
            frame = data_map.get(sym)
            if frame is not None and ts in frame.index:
                open_price = self.valuation_open(frame.loc[ts])
                if (
                    pd.notna(open_price)
                    and float(open_price) > 0
                ):
                    current_price = float(open_price)

            margin = self._calc_margin(sym, pos.size, pos.entry_price, pos.leverage)
            unrealized = self._calc_pnl(
                sym, pos.direction, pos.size, pos.entry_price, current_price
            )
            equity += margin + unrealized
        return equity

    def _calc_equity(self, close_df: pd.DataFrame, ts: pd.Timestamp) -> float:
        """Total equity = free cash + sum(margin + unrealised) per position.

        Uses vectorized numpy path when _calc_pnl/_calc_margin are not
        overridden by a subclass (FuturesBaseEngine, CompositeEngine).
        """
        if not self.positions:
            return self.capital

        _base_pnl = type(self)._calc_pnl is BaseEngine._calc_pnl
        _base_margin = type(self)._calc_margin is BaseEngine._calc_margin

        # Use array fast-path when available
        _arr = getattr(self, "_close_arr", None)
        _varr = getattr(self, "_val_arr", None)
        _row = getattr(self, "_bar_idx", None)
        _c2c = getattr(self, "_code_to_col", None)

        if _base_pnl and _base_margin:
            syms = list(self.positions.keys())
            sizes = np.array([p.size for p in self.positions.values()])
            entry_prices = np.array([p.entry_price for p in self.positions.values()])
            directions = np.array([p.direction for p in self.positions.values()])
            leverages = np.array([p.leverage for p in self.positions.values()])

            current_prices = np.array([
                self._safe_price(
                    close_df, ts, s, ep,
                    _arr=_arr, _row=_row, _col=(_c2c.get(s) if _c2c else None),
                    _val_arr=_varr,
                )
                for s, ep in zip(syms, entry_prices)
            ])

            margins = sizes * entry_prices / leverages
            pnls = directions * sizes * (current_prices - entry_prices)
            return self.capital + float(np.sum(margins + pnls))

        equity = self.capital
        for sym, pos in self.positions.items():
            cp = self._safe_price(
                close_df, ts, sym, pos.entry_price,
                _arr=_arr, _row=_row, _col=(_c2c.get(sym) if _c2c else None),
                _val_arr=_varr,
            )
            margin = self._calc_margin(sym, pos.size, pos.entry_price, pos.leverage)
            unrealized = self._calc_pnl(sym, pos.direction, pos.size, pos.entry_price, cp)
            equity += margin + unrealized
        return equity

    def _rebalance(
        self,
        symbol: str,
        target_weight: float,
        df: Optional[pd.DataFrame],
        ts: pd.Timestamp,
        equity: float,
    ) -> None:
        """Adjust position for *symbol* toward *target_weight*."""
        self._active_symbol = symbol
        target_dir = 1 if target_weight > 1e-9 else (-1 if target_weight < -1e-9 else 0)
        current_pos = self.positions.get(symbol)

        # Nothing to do
        if current_pos is None and target_dir == 0:
            return
        if df is None or ts not in df.index:
            return

        bar = df.loc[ts]

        # Close if target is flat or direction changed
        if current_pos is not None:
            need_close = target_dir == 0 or target_dir != current_pos.direction
            if need_close:
                if self.can_execute(symbol, 0, bar):
                    open_price = self.execution_open(bar)
                    price = self.apply_slippage(open_price, -current_pos.direction)
                    self._close_position(symbol, price, ts, "signal")
                else:
                    return  # blocked (e.g. limit-down can't sell)

        if target_dir != 0 and symbol not in self.positions:
            order = self._plan_open_order(symbol, target_weight, df, ts, equity)
            if order is not None and order.cost <= self.available_cash(ts) + 1e-9:
                self._execute_open_order(order, ts)

    def _plan_open_order(
        self,
        symbol: str,
        target_weight: float,
        df: Optional[pd.DataFrame],
        ts: pd.Timestamp,
        equity: float,
        *,
        allow_existing: bool = False, require_positive_price: bool = False,
        observe: bool = True,
    ) -> Optional[_OpenOrder]:
        """Price an opening order without mutating portfolio state.

        ``observe=False`` plans a trial inside a capital-fit search: a rejection
        there is not a finding and is not reported to ``_on_plan_rejected``.
        """
        self._active_symbol = symbol

        def rejected(reason: str) -> None:
            if observe:
                self._on_plan_rejected(symbol, reason, ts)

        direction = 1 if target_weight > 1e-9 else (-1 if target_weight < -1e-9 else 0)
        if direction == 0:
            rejected("no_target_weight")
            return None
        if symbol in self.positions and not allow_existing:
            rejected("already_held")
            return None
        if df is None:
            rejected("no_data")
            return None
        if ts not in df.index:
            rejected("no_bar")
            return None
        bar = df.loc[ts]
        if not self.can_execute(symbol, direction, bar):
            rejected("execution_blocked")
            return None
        open_price = self.execution_open(bar)
        if require_positive_price:
            self._validate_rebalance_values(open_price, positive=True)
        # Zero is always rejected (size = notional / price is undefined);
        # negatives are rejected unless this engine opted into non-positive
        # prices, in which case abs()-based sizing/margin below handle them.
        elif open_price == 0 or (open_price < 0 and not self.allow_nonpositive_prices):
            rejected("invalid_price")
            return None
        price = self.apply_slippage(open_price, direction)
        if require_positive_price:
            self._validate_rebalance_values(price, positive=True)
        leverage = self._leverage_for_symbol(symbol)
        target_notional = abs(target_weight) * equity * leverage
        size = self.round_size(
            self._calc_raw_size(symbol, target_notional, price), price
        )
        if size <= 0:
            rejected("zero_size")
            return None
        margin = self._calc_margin(symbol, size, price, leverage)
        commission = self.calc_commission(
            size, price, direction, is_open=True
        )
        return _OpenOrder(
            symbol=symbol,
            direction=direction,
            price=price,
            size=size,
            leverage=leverage,
            margin=margin,
            commission=commission,
        )

    # A target move smaller than this is optimiser float noise, not a decision.
    _TARGET_CHANGE_EPSILON = 1e-6
    # Detail lines are logged for the first few only; the rest are counted.
    _DROPPED_ADJUSTMENT_LOG_LIMIT = 3

    def _record_dropped_target_adjustments(
        self, target_weights: Dict[str, Optional[float]], ts: pd.Timestamp
    ) -> None:
        """Note every same-direction resize that hold mode is about to drop.

        Only a change in the TARGET counts. A held position's weight drifts
        with price on its own, so comparing the target against the current
        weight would report a buy-and-hold position as a dropped request on
        every bar.

        Args:
            target_weights: This bar's target per symbol; ``None`` means the
                strategy produced no decision and nothing is dropped.
            ts: The decision timestamp, recorded with each dropped request.
        """
        for symbol, target_w in target_weights.items():
            if target_w is None:
                continue
            previous = self._last_target_weight.get(symbol)
            self._last_target_weight[symbol] = target_w
            if previous is None or abs(target_w - previous) <= self._TARGET_CHANGE_EPSILON:
                continue
            position = self.positions.get(symbol)
            if position is None:
                continue
            target_dir = 1 if target_w > 1e-9 else (-1 if target_w < -1e-9 else 0)
            # A flip or an exit IS executed in hold mode; only a same-direction
            # resize is dropped.
            if target_dir == 0 or target_dir != position.direction:
                continue
            self.dropped_target_adjustments.append(
                {
                    "timestamp": ts,
                    "symbol": symbol,
                    "previous_target_weight": previous,
                    "requested_target_weight": target_w,
                    "direction": position.direction,
                }
            )
            if len(self.dropped_target_adjustments) <= self._DROPPED_ADJUSTMENT_LOG_LIMIT:
                logger.warning(
                    "position_adjustment='hold' dropped a resize: %s at %s asked for "
                    "weight %.6f (was %.6f) and the position was left unchanged. "
                    "Set position_adjustment='rebalance' to execute target changes.",
                    symbol,
                    ts,
                    target_w,
                    previous,
                )

    def _execute_target_rebalance(
        self,
        target_weights: Dict[str, Optional[float]], data_map: Dict[str, pd.DataFrame],
        ts: pd.Timestamp, equity: float, codes: List[str],
    ) -> None:
        """Plan every target delta, preflight capital, then commit the basket."""
        reductions: list[_ReductionOrder] = []
        opens: list[_OpenOrder] = []

        for symbol in sorted(codes):
            target_weight = target_weights.get(symbol)
            if target_weight is None:
                continue
            self._validate_rebalance_values(target_weight)

            self._active_symbol = symbol
            before = self.positions.get(symbol)
            target_direction = 1 if target_weight > 1e-9 else (-1 if target_weight < -1e-9 else 0)
            frame = data_map.get(symbol)
            if frame is None or ts not in frame.index:
                continue
            bar = frame.loc[ts]

            if before is None or target_direction != before.direction:
                if before is not None:
                    if not self.can_execute(symbol, 0, bar):
                        continue
                    raw_price = self.execution_open(bar)
                    self._validate_rebalance_values(raw_price, positive=True)
                    price = self.apply_slippage(raw_price, -before.direction)
                    self._validate_rebalance_values(price, positive=True)
                    reductions.append(self._plan_reduction(before, 0.0, price))
                    if target_direction == 0:
                        continue
                order = self._plan_open_order(
                    symbol,
                    target_weight,
                    frame,
                    ts,
                    equity,
                    allow_existing=before is not None,
                    require_positive_price=True,
                )
                if order is not None:
                    self._validate_rebalance_values(
                        order.price, order.leverage, order.size, order.margin, order.commission
                    )
                    opens.append(order)
                continue

            raw_price = self.execution_open(bar)
            self._validate_rebalance_values(raw_price, positive=True)
            leverage = self._leverage_for_symbol(symbol)
            self._validate_rebalance_values(raw_price, before.leverage, leverage)
            target_notional = abs(target_weight) * equity * before.leverage
            prices = (
                self.apply_slippage(raw_price, before.direction),
                self.apply_slippage(raw_price, -before.direction),
            )
            self._validate_rebalance_values(*prices, positive=True)
            sizes = tuple(
                self.round_size(
                    self._calc_raw_size(symbol, target_notional, price), price
                )
                for price in prices
            )
            self._validate_rebalance_values(*prices, *sizes)
            # Tolerance band. The held size is compared against the size the
            # target implies at the unslipped price -- keeping the slippage side
            # out of a drift question is the principled choice, though its
            # practical effect is one slippage width and only reaches the
            # outcome within that distance of the band edge, which is why no
            # test pins it: such a test would assert a coincidence.
            # A changed target moves this reference far past any sane band, so
            # target changes still execute at every tolerance.
            if self.rebalance_tolerance > 0.0:
                reference_size = self.round_size(
                    self._calc_raw_size(symbol, target_notional, raw_price), raw_price
                )
                if abs(before.size - reference_size) <= self.rebalance_tolerance * abs(
                    reference_size
                ):
                    continue

            increase = sizes[0] > before.size + 1e-9
            reduction = sizes[1] < before.size - 1e-9
            # Both or neither means the target lies inside the fill-price band.
            if increase == reduction:
                continue
            target_size, price = (sizes[0], prices[0]) if increase else (sizes[1], prices[1])
            if not math.isclose(leverage, before.leverage, rel_tol=1e-9, abs_tol=1e-9):
                raise ValueError("cannot rebalance a position with changed leverage")
            if increase:
                if not self.can_execute(symbol, target_direction, bar):
                    continue
                size = target_size - before.size
                order = _OpenOrder(
                    symbol=symbol,
                    direction=before.direction,
                    price=price,
                    size=size,
                    leverage=leverage,
                    margin=self._calc_margin(symbol, size, price, leverage),
                    commission=self.calc_commission(
                        size, price, before.direction, is_open=True
                    ),
                )
                self._validate_rebalance_values(
                    order.price, order.leverage, order.size, order.margin, order.commission
                )
                opens.append(order)
            elif self.can_execute(symbol, 0, bar):
                reductions.append(self._plan_reduction(before, target_size, price))

        projected_capital = (
            self.available_cash(ts)
            + self.projected_reduction_buying_power(reductions, ts)
            - sum(order.cost for order in opens)
        )
        self._validate_rebalance_values(projected_capital)
        if projected_capital < -1e-9:
            fitted = self._fit_rebalance_opens(opens, reductions, ts)
            if fitted is None:
                if self.projected_reduction_buying_power(reductions, ts) < 0:
                    # The reductions themselves consume cash (a close whose loss
                    # exceeds its margin): no dropping of opens repairs that, so
                    # the bar aborts atomically as #1274 decided.
                    raise ValueError("insufficient capital for position rebalance")
                # Cash was already below zero (a funding debit can leave it
                # there) and the reductions only release it: drop every open,
                # still run the reductions, and report each open (#1542).
                for order in opens:
                    self._on_plan_rejected(order.symbol, "insufficient_capital", ts)
                if opens:
                    logger.warning(
                        "Cash below zero at %s; no open or increase fits, dropped: %s",
                        ts,
                        ", ".join(sorted({order.symbol for order in opens})),
                    )
                fitted = []
            opens = fitted

        for order in reductions:
            if order.target_size <= 1e-9:
                self._close_position(order.before.symbol, order.price, ts, "signal")
            else:
                self._execute_partial_reduction(order, ts)
        for order in opens:
            if order.symbol in self.positions:
                self._execute_position_increase(order, ts)
            else:
                self._execute_open_order(order, ts)

    def _fit_rebalance_opens(
        self,
        opens: list[_OpenOrder],
        reductions: list[_ReductionOrder],
        ts: pd.Timestamp,
    ) -> Optional[list[_OpenOrder]]:
        """Scale the open sleeve by one common factor so the basket fits.

        #1274: a fully invested target basket plus commission overdrafts by a
        hair and used to abort atomically. Instead, mirror the open-basket
        path's fairness behavior: reductions commit as planned (they release
        capital toward the targets), and every capital-consuming sleeve —
        fresh opens and same-direction increases alike — scales by one common
        factor, preserving portfolio proportions on sizes/notionals (post-fee
        capital weights shift by each sleeve's fee, as with the open path).
        Sizes re-round per market lot rules; a sleeve whose rounded size hits
        zero is dropped and recorded via ``_on_plan_rejected`` as
        ``insufficient_capital`` — it was a real order at full scale, so it is
        the cash that failed, not the lot rule (#1470) — so run-card
        diagnostics see the dropped leg.

        Returns the fitted orders, or ``None`` when no scale fits — not even
        an empty open sleeve; nothing has been committed at that point. The
        caller aborts atomically when the reductions themselves consume cash
        (#1274), and otherwise — cash was already below zero — drops every
        open and still runs the reductions (#1542).
        """
        released = self.projected_reduction_buying_power(reductions, ts)

        def _fits(candidate: list[_OpenOrder]) -> bool:
            projected = self.available_cash(ts) + released - sum(order.cost for order in candidate)
            self._validate_rebalance_values(projected)
            return projected >= -1e-9

        def _at_scale(scale: float) -> list[_OpenOrder]:
            scaled: list[_OpenOrder] = []
            for order in opens:
                # round_size/calc_commission dispatch on the active symbol
                # (lot grids, per-symbol fee schedules) — same contract as
                # _plan_open_order.
                self._active_symbol = order.symbol
                size = self.round_size(order.size * scale, order.price)
                if size <= 0:
                    continue
                candidate = _OpenOrder(
                    symbol=order.symbol,
                    direction=order.direction,
                    price=order.price,
                    size=size,
                    leverage=order.leverage,
                    margin=self._calc_margin(
                        order.symbol, size, order.price, order.leverage
                    ),
                    commission=self.calc_commission(
                        size, order.price, order.direction, is_open=True
                    ),
                )
                self._validate_rebalance_values(
                    candidate.price,
                    candidate.leverage,
                    candidate.size,
                    candidate.margin,
                    candidate.commission,
                )
                scaled.append(candidate)
            return scaled

        fitted: Optional[list[_OpenOrder]] = None
        low, high = 0.0, 1.0
        for _ in range(50):
            mid = (low + high) / 2.0
            candidate = _at_scale(mid)
            if _fits(candidate):
                low, fitted = mid, candidate
            else:
                high = mid
        if fitted is not None and len(fitted) < len(opens):
            dropped_symbols = sorted(
                {order.symbol for order in opens}
                - {order.symbol for order in fitted}
            )
            for symbol in dropped_symbols:
                self._on_plan_rejected(symbol, "insufficient_capital", ts)
            logger.warning(
                "Rebalance basket scaled to fit capital; sleeves rounded to "
                "zero and dropped: %s. Set position_adjustment='hold' or "
                "reduce targets if these fills are required.",
                ", ".join(dropped_symbols),
            )
        return fitted

    @staticmethod
    def _validate_rebalance_values(*values: float, positive: bool = False) -> None:
        if all(
            math.isfinite(float(value)) and (not positive or float(value) > 0)
            for value in values
        ):
            return
        raise ValueError(
            "rebalance requires a finite positive execution price"
            if positive
            else "non-finite position rebalance value"
        )

    def _plan_reduction(self, before: Position, target_size: float, price: float) -> _ReductionOrder:
        closed_size = before.size - target_size
        released_margin = self._calc_margin(
            before.symbol, closed_size, before.entry_price, before.leverage
        )
        realized_pnl = self._calc_pnl(
            before.symbol, before.direction, closed_size, before.entry_price, price
        )
        exit_commission = self.calc_commission(
            closed_size, price, before.direction, is_open=False
        )
        self._validate_rebalance_values(
            target_size,
            price,
            released_margin,
            realized_pnl,
            exit_commission,
        )
        return _ReductionOrder(before, target_size, price, released_margin, realized_pnl, exit_commission)

    def _execute_position_increase(self, order: _OpenOrder, ts: pd.Timestamp) -> None:
        """Commit a same-direction opening delta."""
        before = self.positions[order.symbol]
        if order.direction != before.direction:
            raise ValueError("position increase direction must match the open position")
        if order.cost > self.available_cash(ts) + 1e-7:
            raise RuntimeError(
                f"planned order for {order.symbol} exceeds available capital"
            )
        new_size = before.size + order.size
        after = replace(
            before,
            entry_price=(before.size * before.entry_price + order.size * order.price) / new_size,
            size=new_size,
            entry_commission=before.entry_commission + order.commission,
        )
        self.capital -= order.cost
        self.positions[order.symbol] = after
        self._record_fill(
            symbol=order.symbol,
            timestamp=ts,
            action="increase",
            signed_quantity=order.direction * order.size,
            execution_price=order.price,
            fee=order.commission,
            margin=order.margin,
            leverage=order.leverage,
            reason="target_rebalance",
        )
        self.after_position_adjustment(
            action="increase",
            timestamp=ts,
            before=before,
            after=after,
            execution_price=order.price,
            trading_fee=order.commission,
        )

    def _execute_partial_reduction(self, order: _ReductionOrder, ts: pd.Timestamp) -> None:
        """Commit a same-direction closing delta."""
        before = self.positions[order.before.symbol]
        closed_size = before.size - order.target_size
        entry_commission = before.entry_commission * closed_size / before.size
        after = replace(
            before,
            size=order.target_size,
            entry_commission=before.entry_commission - entry_commission,
        )
        exit_margin = self._calc_margin(
            before.symbol, closed_size, order.price, before.leverage
        )
        pnl_pct = (
            order.realized_pnl / order.released_margin * 100
            if order.released_margin > 1e-9
            else 0.0
        )
        holding_bars = self._weighted_holding_bars(before)
        self.capital += order.capital_credit
        self.positions[before.symbol] = after
        self.trades.append(TradeRecord(
            symbol=before.symbol,
            direction=before.direction,
            entry_price=before.entry_price,
            exit_price=order.price,
            entry_time=before.entry_time,
            exit_time=ts,
            size=closed_size,
            leverage=before.leverage,
            pnl=order.realized_pnl,
            pnl_pct=pnl_pct,
            exit_reason="target_rebalance",
            holding_bars=holding_bars,
            commission=entry_commission + order.exit_commission,
            entry_margin=order.released_margin,
            exit_margin=exit_margin,
            open_price=before.open_price,
            open_size=before.open_size,
        ))
        self._record_fill(
            symbol=before.symbol,
            timestamp=ts,
            action="reduce",
            signed_quantity=-before.direction * closed_size,
            execution_price=order.price,
            fee=order.exit_commission,
            margin=exit_margin,
            leverage=before.leverage,
            reason="target_rebalance",
            holding_bars=holding_bars,
        )
        self.after_position_adjustment(
            action="partial_reduction",
            timestamp=ts,
            before=before,
            after=after,
            execution_price=order.price,
            trading_fee=order.exit_commission,
            realized_pnl=order.realized_pnl,
            released_margin=order.released_margin,
        )

    def _execute_open_order(self, order: _OpenOrder, ts: pd.Timestamp) -> None:
        """Commit a previously priced opening order."""
        if order.cost > self.available_cash(ts) + 1e-7:
            raise RuntimeError(
                f"planned order for {order.symbol} exceeds available capital"
            )
        self.capital -= order.cost
        self.positions[order.symbol] = Position(
            symbol=order.symbol,
            direction=order.direction,
            entry_price=order.price,
            entry_time=ts,
            size=order.size,
            leverage=order.leverage,
            entry_bar_idx=self._bar_idx,
            entry_commission=order.commission,
            open_price=order.price,
            open_size=order.size,
        )
        self._record_fill(
            symbol=order.symbol,
            timestamp=ts,
            action="open",
            signed_quantity=order.direction * order.size,
            execution_price=order.price,
            fee=order.commission,
            margin=order.margin,
            leverage=order.leverage,
            reason="signal",
        )

    def _actual_weights(
        self,
        close_df: pd.DataFrame,
        ts: pd.Timestamp,
        codes: List[str],
        equity: float,
    ) -> Dict[str, float]:
        """Return post-fill, mark-to-market weights for the currently held book."""
        weights = {code: 0.0 for code in codes}
        if abs(equity) <= 1e-12:
            return weights
        for symbol, pos in self.positions.items():
            price = self._safe_price(
                close_df,
                ts,
                symbol,
                pos.entry_price,
                _arr=getattr(self, "_close_arr", None),
                _row=getattr(self, "_bar_idx", None),
                _col=getattr(self, "_code_to_col", {}).get(symbol),
                _val_arr=getattr(self, "_val_arr", None),
            )
            margin_value = self._calc_margin(
                symbol, pos.size, price, pos.leverage
            )
            weights[symbol] = pos.direction * margin_value / equity
        return weights

    def _actual_positions_frame(self, codes: List[str]) -> pd.DataFrame:
        """Materialize recorded post-fill weights as an artifact-ready frame."""
        if not self.actual_position_snapshots:
            return pd.DataFrame(columns=codes, dtype=float)
        frame = pd.DataFrame(
            [weights for _, weights in self.actual_position_snapshots],
            index=[timestamp for timestamp, _ in self.actual_position_snapshots],
            columns=codes,
            dtype=float,
        )
        frame.index.name = "timestamp"
        return frame

    def _close_proceeds(self, pos: Position, exit_price: float) -> float:
        """Capital a close at *exit_price* would return, without closing.

        Mirrors the credit in ``_close_position`` (margin + pnl - exit
        commission) so a position valued this way is worth exactly what the
        close would have booked.
        """
        self._active_symbol = pos.symbol
        pnl = self._calc_pnl(pos.symbol, pos.direction, pos.size, pos.entry_price, exit_price)
        margin = self._calc_margin(pos.symbol, pos.size, pos.entry_price, pos.leverage)
        exit_comm = self.calc_commission(pos.size, exit_price, pos.direction, is_open=False)
        return margin + pnl - exit_comm

    def _close_position(
        self,
        symbol: str,
        exit_price: float,
        exit_time: pd.Timestamp,
        reason: str,
    ) -> None:
        """Close position, record trade, return capital."""
        self._active_symbol = symbol
        pos = self.positions.pop(symbol, None)
        if pos is None:
            return

        pnl = self._calc_pnl(symbol, pos.direction, pos.size, pos.entry_price, exit_price)
        margin = self._calc_margin(symbol, pos.size, pos.entry_price, pos.leverage)
        exit_margin = self._calc_margin(symbol, pos.size, exit_price, pos.leverage)
        pnl_pct = pnl / margin * 100 if margin > 1e-9 else 0.0
        exit_comm = self.calc_commission(pos.size, exit_price, pos.direction, is_open=False)

        self.capital += margin + pnl - exit_comm

        holding_bars = self._weighted_holding_bars(pos)

        self.trades.append(TradeRecord(
            symbol=symbol,
            direction=pos.direction,
            entry_price=pos.entry_price,
            exit_price=exit_price,
            entry_time=pos.entry_time,
            exit_time=exit_time,
            size=pos.size,
            leverage=pos.leverage,
            pnl=pnl,
            pnl_pct=pnl_pct,
            exit_reason=reason,
            holding_bars=holding_bars,
            commission=pos.entry_commission + exit_comm,
            entry_margin=margin,
            exit_margin=exit_margin,
            open_price=pos.open_price,
            open_size=pos.open_size,
        ))
        self._record_fill(
            symbol=symbol,
            timestamp=exit_time,
            action="close",
            signed_quantity=-pos.direction * pos.size,
            execution_price=exit_price,
            fee=exit_comm,
            margin=exit_margin,
            leverage=pos.leverage,
            reason=reason,
            holding_bars=holding_bars,
        )

    def _record_fill(
        self,
        *,
        symbol: str,
        timestamp: pd.Timestamp,
        action: str,
        signed_quantity: float,
        execution_price: float,
        fee: float,
        margin: float,
        leverage: float,
        reason: str,
        holding_bars: float | None = None,
    ) -> None:
        """Append execution evidence without making it account state."""
        self.fill_records.append(FillRecord(
            symbol=symbol,
            timestamp=timestamp,
            bar_idx=self._bar_idx,
            action=action,
            signed_quantity=signed_quantity,
            notional=margin * leverage,
            execution_price=execution_price,
            fee=fee,
            margin=margin,
            reason=reason,
            holding_bars=holding_bars,
        ))

    def _weighted_holding_bars(self, position: Position) -> float:
        """Derive compressed-position age from immutable fill evidence.

        Reductions consume every accumulated opening delta proportionally,
        matching the weighted-average entry accounting used by this engine.
        Only the active position lifecycle matters; stopping at its preceding
        full close avoids rescanning the entire fill ledger on every exit.
        """
        active_fills: list[FillRecord] = []
        for fill in reversed(self.fill_records):
            if fill.symbol != position.symbol:
                continue
            if fill.action == "close":
                break
            active_fills.append(fill)

        lots: list[list[float]] = []
        for fill in reversed(active_fills):
            quantity = abs(fill.signed_quantity)
            if fill.action in {"open", "increase"}:
                lots.append([quantity, float(fill.bar_idx)])
                continue
            if fill.action not in {"reduce", "close"}:
                continue
            total = sum(lot[0] for lot in lots)
            if total <= 1e-12 or quantity >= total - 1e-12:
                lots.clear()
                continue
            remaining_ratio = (total - quantity) / total
            for lot in lots:
                lot[0] *= remaining_ratio

        total = sum(lot[0] for lot in lots)
        if total <= 1e-12:
            return float(max(self._bar_idx - position.entry_bar_idx, 0))
        return sum(
            quantity * max(self._bar_idx - int(bar_idx), 0)
            for quantity, bar_idx in lots
        ) / total

    # ── Artifacts ──

    def _write_artifacts(
        self,
        run_dir: Path,
        data_map: Dict[str, pd.DataFrame],
        dates: pd.DatetimeIndex,
        equity_series: pd.Series,
        bench_equity: pd.Series,
        bench_ret: pd.Series,
        target_pos: pd.DataFrame,
        metrics: dict,
        codes: List[str],
    ) -> None:
        """Write CSV artifacts compatible with daily_portfolio format."""
        out = run_dir / "artifacts"
        out.mkdir(parents=True, exist_ok=True)

        # OHLCV per symbol
        for code, df in data_map.items():
            df.to_csv(out / f"ohlcv_{code}.csv")

        # Equity curve. Same cash-seeded returns as calc_metrics, so the ret
        # column compounds back to total_return and agrees with drawdown.
        port_ret = period_returns(equity_series, self.initial_capital).fillna(0.0)
        # Same high-water mark as calc_metrics: the account holds initial_capital
        # before the first recorded bar, so a loss on that bar is a drawdown too.
        peak = equity_series.cummax().clip(lower=float(self.initial_capital))
        dd = (equity_series - peak) / peak.replace(0, 1)
        eq_df = pd.DataFrame({
            "ret": port_ret,
            "equity": equity_series,
            "drawdown": dd,
            "benchmark_equity": bench_equity.reindex(dates),
            "active_ret": port_ret - bench_ret.reindex(dates).fillna(0.0),
        }, index=dates)
        eq_df.index.name = "timestamp"
        eq_df.to_csv(out / "equity.csv")

        # ``positions.csv`` is execution truth.  Keep optimiser requests in a
        # separate artifact so blocked/rounded/scaled fills remain auditable.
        actual_pos = self._actual_positions_frame(codes)
        actual_pos.to_csv(out / "positions.csv")
        target_out = target_pos.copy()
        target_out.index.name = "timestamp"
        target_out.to_csv(out / "target_positions.csv")

        # Trades (compatible format): two rows per round trip, entry then exit.
        # The entry row is the opening fill as executed. A TradeRecord's
        # entry_price is the weighted cost after any scale-ins while its
        # entry_time stays on the first open, so pairing those two would
        # report a fill that never happened. Scale-ins stay in fills.jsonl.
        trade_rows = []
        for t in self.trades:
            # Entry event
            entry_price = t.entry_price if t.open_price is None else t.open_price
            entry_qty = t.size if t.open_size is None else t.open_size
            trade_rows.append({
                "timestamp": str(t.entry_time.date()) if hasattr(t.entry_time, "date") else str(t.entry_time),
                "code": t.symbol,
                "side": "buy" if t.direction == 1 else "sell",
                "price": entry_price,
                "qty": round(entry_qty, 6),
                "reason": "signal",
                "pnl": 0.0,
                "holding_days": 0,
                "holding_bars": 0.0,
                "return_pct": 0.0,
                "event": "entry",
            })
            # Exit event
            try:
                hold_days = (t.exit_time - t.entry_time).days
            except Exception:
                hold_days = 0
            trade_rows.append({
                "timestamp": str(t.exit_time.date()) if hasattr(t.exit_time, "date") else str(t.exit_time),
                "code": t.symbol,
                "side": "sell" if t.direction == 1 else "buy",
                "price": t.exit_price,
                "qty": round(t.size, 6),
                "reason": t.exit_reason,
                "pnl": round(t.pnl, 4),
                "holding_days": hold_days,
                "holding_bars": t.holding_bars,
                "return_pct": round(t.pnl_pct, 2),
                "event": "exit",
            })

        trade_cols = [
            "timestamp", "code", "side", "price", "qty", "reason", "pnl",
            "holding_days", "holding_bars", "return_pct", "event",
        ]
        pd.DataFrame(trade_rows or [], columns=trade_cols).to_csv(out / "trades.csv", index=False)

        # Immutable execution evidence. JSONL keeps each delta self-contained
        # and avoids treating this append-only audit trail as mutable position
        # or account state.
        with (out / "fills.jsonl").open("w", encoding="utf-8") as handle:
            for fill in self.fill_records:
                handle.write(json.dumps({
                    "symbol": fill.symbol,
                    "timestamp": fill.timestamp.isoformat(),
                    "bar_idx": fill.bar_idx,
                    "action": fill.action,
                    "signed_quantity": fill.signed_quantity,
                    "notional": fill.notional,
                    "execution_price": fill.execution_price,
                    "fee": fill.fee,
                    "margin": fill.margin,
                    "reason": fill.reason,
                    "holding_bars": fill.holding_bars,
                }, allow_nan=False) + "\n")

        # Metrics
        flat_metrics = {k: v for k, v in metrics.items() if not isinstance(v, dict)}
        pd.DataFrame([flat_metrics]).to_csv(out / "metrics.csv", index=False)

    # ── Helpers ──

    @staticmethod
    def _safe_price(
        close_df: pd.DataFrame,
        ts: pd.Timestamp,
        symbol: str,
        fallback: float,
        *,
        _arr: "np.ndarray | None" = None,
        _row: "int | None" = None,
        _col: "int | None" = None,
        _val_arr: "np.ndarray | None" = None,
    ) -> float:
        """Get close price with fallback. Uses array fast-path when available.

        Valuation call sites pass ``_val_arr`` (unbounded ffill) so a halted
        position marks at its last traded close instead of ``fallback`` once
        the bounded matrix goes NaN past the ffill limit.
        """
        # Fast path: pre-computed array indexing (O(1) vs DataFrame.at hash lookup)
        if _arr is not None and _row is not None and _col is not None:
            val = _arr[_row, _col]
            if not np.isnan(val):
                return float(val)
            if _val_arr is not None:
                vval = _val_arr[_row, _col]
                if not np.isnan(vval):
                    return float(vval)
            return fallback
        # Original path (backward compatible for subclasses)
        if ts in close_df.index and symbol in close_df.columns:
            val = close_df.at[ts, symbol]
            if pd.notna(val):
                return float(val)
        return fallback