"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Cross-sectional factor combination with caller-owned or IC-based weights.
Each panel row is one observation time and each column is one asset -- the same
contract as :mod:`src.quantlib.neutralization`. Several factor panels are
combined into a single composite score per ``(time, asset)``.
There is no complete list of combination methods: equal weight, IC weighting,
decayed IC, maximum IC, regression weights, trained models and risk-based
weights all exist, and new ones keep appearing. This module therefore does not
enumerate methods. It fixes the *weight contract* instead:
* ``weights`` accepts a static mapping or a ``time x factor`` DataFrame. Any
method that produces such a table -- fitted elsewhere, tuned however the
caller likes -- can be combined here. A table is auditable, reproducible and
transportable over MCP; a callback is none of those.
* ``equal``, ``ic`` and ``icir`` are shipped as transparent baselines, not as a
catalogue: they need no tuning, no dependency and no training.
Weight timing is the one rule that cannot be relaxed: a weight applied at row
``t`` may only use information from rows ``< t``. ``ic``/``icir`` enforce this
internally; a caller-supplied table must already respect it. Passing full-sample
IC weights leaks the evaluation period into the signal.
This module combines signals; it does not size positions, penalise turnover or
build a risk model. Hand the composite panel to a signal engine or a backtest.
Rank IC is computed locally rather than by importing
:mod:`src.factors.factor_analysis_core`: the factors package already imports
``src.quantlib.multipletesting``, so the reverse import would be circular.
"""
from __future__ import annotations
import warnings
from collections.abc import Mapping
from dataclasses import dataclass
from numbers import Integral, Real
from typing import Any
import numpy as np
import pandas as pd
__all__ = ["combine_factors"]
_METHODS: tuple[str, ...] = ("equal", "explicit", "ic", "icir")
_WEIGHTED_METHODS: tuple[str, ...] = ("ic", "icir")
_MIN_ASSETS = 5
_STD_FLOOR = 1e-12
@dataclass
class CombinationResult:
"""Composite panel, per-row weights and research diagnostics.
Attributes:
composite: Composite score per observation time and asset. ``NaN``
wherever no factor was available or no weight had been estimated
yet; never silently zero.
weights: Weight per observation time and factor. Rows before the first
estimable weight are ``NaN``; weights are held between rebalances.
diagnostics: ``factor_ic``/``composite_ic``/``correlation`` summaries,
weight turnover and the settings that produced them.
method: The combination method that was requested.
"""
composite: pd.DataFrame
weights: pd.DataFrame
diagnostics: dict[str, Any]
method: str
# --- input guards (same shape as src/quantlib/neutralization.py) ---
def _check_panel(panel: pd.DataFrame, name: str) -> None:
if not isinstance(panel, pd.DataFrame):
raise ValueError(f"{name} must be an indexed DataFrame (time x asset)")
if panel.empty or not panel.index.is_unique or not panel.columns.is_unique:
raise ValueError(f"{name} must be nonempty with unique time/asset labels")
def _numeric(panel: pd.DataFrame, name: str) -> pd.DataFrame:
_check_panel(panel, name)
try:
values = panel.to_numpy(dtype=float, na_value=np.nan)
except (TypeError, ValueError) as exc:
raise ValueError(f"{name} must contain numeric values") from exc
values = np.where(np.isfinite(values), values, np.nan)
return pd.DataFrame(values, index=panel.index, columns=panel.columns)
def _positive_int(value: Any, name: str, minimum: int = 1) -> None:
if isinstance(value, bool) or not isinstance(value, Integral) or value < minimum:
raise ValueError(f"{name} must be an integer >= {minimum}")
def _ordered_union(labels: list[pd.Index]) -> pd.Index:
"""Union of labels in first-seen order (input order is preserved)."""
seen: dict[Any, None] = {}
for index in labels:
for label in index:
seen.setdefault(label, None)
return pd.Index(list(seen))
def _bounded(values: np.ndarray) -> np.ndarray:
"""Scale each row by its largest magnitude.
Centering and squared norms can overflow for large inputs (market caps) and
underflow into a false zero variance for tiny ones (ratios); dividing by the
row maximum first keeps the arithmetic inside the float range. Mirrors the
guard in :func:`src.quantlib.neutralization._bounded`.
"""
units = np.max(np.abs(values), axis=1, keepdims=True)
units = np.where(np.isfinite(units) & (units > 0), units, 1.0)
return values / units
def _cross_sectional_z(panel: pd.DataFrame) -> pd.DataFrame:
"""Per-row z-score; rows with fewer than two values or no spread become NaN."""
values = _bounded(panel.to_numpy(dtype=float))
counts = np.sum(np.isfinite(values), axis=1, keepdims=True)
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
with np.errstate(invalid="ignore", divide="ignore"):
mean = np.nanmean(values, axis=1, keepdims=True)
std = np.nanstd(values, axis=1, ddof=1, keepdims=True)
z = (values - mean) / std
usable = np.isfinite(std) & (std > _STD_FLOOR) & (counts >= 2)
z = np.where(usable, z, np.nan)
return pd.DataFrame(z, index=panel.index, columns=panel.columns)
# --- IC estimation ---
def _rank_ic(factor: pd.Series, forward: pd.Series, min_assets: int) -> float | None:
"""Spearman correlation of one cross-section; ``None`` when unestimable."""
mask = factor.notna() & forward.notna()
if int(mask.sum()) < min_assets:
return None
left = factor[mask].rank()
right = forward[mask].rank()
left = left - left.mean()
right = right - right.mean()
denominator = float(np.sqrt((left * left).sum() * (right * right).sum()))
if not np.isfinite(denominator) or denominator <= 0:
return None
return float((left * right).sum() / denominator)
def _ic_series(factor: pd.DataFrame, forward: pd.DataFrame, min_assets: int) -> pd.Series:
values = [_rank_ic(factor.loc[time], forward.loc[time], min_assets) for time in factor.index]
return pd.Series(values, index=factor.index, dtype=float)
def _ic_stats(ic: pd.Series) -> dict[str, Any]:
clean = ic.dropna()
count = int(clean.size)
if count == 0:
return {
"ic_mean": None,
"ic_std": None,
"ir": None,
"ic_positive_ratio": None,
"n_obs": 0,
}
mean = float(clean.mean())
# A sample standard deviation needs two observations; one IC is a point,
# not a spread. Reporting 0.0 claims the IC is perfectly stable on the
# strength of a single date, which is why ir and weight_turnover already
# answer None for the same sample size.
std = float(clean.std(ddof=1)) if count > 1 else None
return {
"ic_mean": mean,
"ic_std": std,
"ir": (mean / std) if std is not None and std > 0 else None,
"ic_positive_ratio": float((clean > 0).mean()),
"n_obs": count,
}
def _mean_rank_correlation(panels: Mapping[str, pd.DataFrame], min_assets: int) -> pd.DataFrame:
"""Average per-date Spearman correlation between factor panels."""
names = list(panels)
total = pd.DataFrame(0.0, index=names, columns=names)
seen = pd.DataFrame(0, index=names, columns=names)
for time in next(iter(panels.values())).index:
cross = pd.DataFrame({name: panels[name].loc[time] for name in names})
cross = cross.dropna(how="any")
if len(cross) < min_assets:
continue
correlation = cross.corr(method="spearman")
total = total.add(correlation.fillna(0.0))
seen = seen.add(correlation.notna().astype(int))
return total / seen.replace(0, np.nan)
# --- weights ---
def _constant_weights(names: list[str], index: pd.Index, values: Mapping[str, float]) -> pd.DataFrame:
row = np.array([float(values[name]) for name in names], dtype=float)
return pd.DataFrame(np.tile(row, (len(index), 1)), index=index, columns=names)
def _walk_forward_weights(
ic_history: Mapping[str, pd.Series],
index: pd.Index,
*,
method: str,
window: int | None,
min_periods: int,
rebalance: int,
insufficient: str,
) -> pd.DataFrame:
"""Estimate weights at each rebalance from strictly earlier IC values."""
names = list(ic_history)
frame = pd.DataFrame(np.nan, index=index, columns=names)
for position in range(0, len(index), rebalance):
estimates: dict[str, float] = {}
for name in names:
history = ic_history[name].iloc[:position].dropna()
if window is not None:
history = history.iloc[-window:]
if len(history) < min_periods:
continue
if method == "ic":
value = float(history.mean())
else:
deviation = float(history.std(ddof=1))
if not deviation > 0:
continue
value = float(history.mean() / deviation)
if np.isfinite(value):
estimates[name] = value
if not estimates:
if insufficient == "equal":
# Normalised like every other weight path, and like
# method="equal": an un-normalised 1.0 per factor leaves the
# row summing to the factor count, which reads as a full-book
# rebalance in weight_turnover once a real estimate replaces it.
frame.iloc[position] = 1.0 / len(names)
continue
total = float(sum(abs(value) for value in estimates.values()))
if not np.isfinite(total) or total <= 0:
continue
frame.iloc[position] = [estimates.get(name, 0.0) / total for name in names]
return frame.ffill()
def _static_weights(weights: Mapping[str, float], names: list[str], index: pd.Index) -> pd.DataFrame:
missing = [name for name in names if name not in weights]
unknown = [name for name in weights if name not in names]
if missing or unknown:
raise ValueError(f"weights must cover every factor exactly; missing={missing}, unknown={unknown}")
values: dict[str, float] = {}
for name in names:
value = weights[name]
if isinstance(value, bool) or not isinstance(value, Real):
raise ValueError(f"weight for {name!r} must be a real number")
if not np.isfinite(float(value)):
raise ValueError(f"weight for {name!r} must be finite")
values[name] = float(value)
if not any(values.values()):
raise ValueError("at least one weight must be nonzero")
scale = max(abs(value) for value in values.values())
scaled = {name: value / scale for name, value in values.items()}
total = float(sum(abs(value) for value in scaled.values()))
return _constant_weights(
names, index, {name: scaled[name] / total for name in names}
)
def _table_weights(weights: pd.DataFrame, names: list[str], index: pd.Index) -> pd.DataFrame:
"""Align a caller-supplied ``time x factor`` weight table.
Rows may be supplied only at rebalance dates; they are held forward. A
missing weight is excluded from that row rather than read as zero, and rows
whose weights are all missing stay ``NaN`` (honest empty, no default).
"""
_check_panel(weights, "weights")
missing = [name for name in names if name not in weights.columns]
unknown = [column for column in weights.columns if column not in names]
if missing or unknown:
raise ValueError(f"weight table must cover every factor exactly; missing={missing}, unknown={unknown}")
try:
table = weights.reindex(index=index)[names].to_numpy(dtype=float, na_value=np.nan)
except (TypeError, ValueError) as exc:
raise ValueError("weight table must contain numeric values") from exc
table = np.where(np.isfinite(table), table, np.nan)
scale = np.max(
np.where(np.isfinite(table), np.abs(table), 0.0), axis=1, keepdims=True
)
scale = np.where(scale > 0, scale, 1.0)
table = table / scale
total = np.nansum(np.abs(table), axis=1, keepdims=True)
with np.errstate(invalid="ignore", divide="ignore"):
table = np.divide(
table, total, out=np.full_like(table, np.nan), where=total > 0
)
return pd.DataFrame(table, index=index, columns=names).ffill()
def _turnover(weights: pd.DataFrame) -> float | None:
effective = weights.dropna(how="all")
if len(effective) < 2:
return None
change = effective.diff().iloc[1:].abs().sum(axis=1)
return float(change.mean())
# --- public entry point ---
def combine_factors(
panels: Mapping[str, pd.DataFrame],
returns: pd.DataFrame | None = None,
*,
method: str = "equal",
weights: Mapping[str, float] | pd.DataFrame | None = None,
window: int | None = None,
rebalance: int = 1,
standardize: bool = True,
min_assets: int = _MIN_ASSETS,
min_periods: int = 20,
insufficient: str = "skip",
) -> CombinationResult:
"""Combine factor panels into one composite score.
Args:
panels: Factor name -> panel (time x asset) of factor values.
returns: Forward-return panel aligned to the factor panels: the value at
row ``t`` is the return earned *after* observing ``t``. Required for
``method="ic"`` and ``"icir"``, unused otherwise.
method: ``"equal"``, ``"explicit"``, ``"ic"`` (mean rank IC weights) or
``"icir"`` (mean/std rank IC weights). IC weights keep the sign, so
a reversed factor receives a negative weight instead of being
silently flipped. Use ``"explicit"`` to bring any other method:
decayed IC, maximum IC, regression or trained weights, risk parity.
weights: Required by ``method="explicit"`` and rejected otherwise.
Either a mapping (factor -> constant weight) or a ``time x factor``
DataFrame for weights that change over time; table rows may cover
only rebalance dates and are held forward. Values are normalised by
their absolute sum. A caller-supplied table must already respect
walk-forward timing: this function cannot see how it was produced.
window: Number of trailing IC observations used to estimate a weight.
``None`` uses all strictly earlier observations.
rebalance: Re-estimate weights every ``rebalance`` rows; between
rebalances the last estimate is held.
standardize: Cross-sectionally z-score each factor per row before
summing. Recommended when factor units differ.
min_assets: Minimum instruments with both a factor and a return value
for an IC observation to count.
min_periods: Minimum IC observations required before a weight is
estimated.
insufficient: ``"skip"`` leaves rows before the first estimate ``NaN``
(honest empty); ``"equal"`` falls back to equal weights there.
Returns:
A :class:`CombinationResult` with the composite panel, the per-row
weight table and diagnostics.
Raises:
ValueError: On unknown method, malformed panels, missing returns for a
weighted method, or weights that do not match the panel names.
"""
if method not in _METHODS:
raise ValueError(f"method must be one of {_METHODS}, got {method!r}")
if weights is not None and method != "explicit":
raise ValueError("weights is only accepted with method='explicit'")
if not isinstance(panels, Mapping) or not panels:
raise ValueError("panels must be a non-empty mapping of name -> DataFrame")
for name, panel in panels.items():
if not isinstance(name, str) or not name:
raise ValueError("panel names must be non-empty strings")
_check_panel(panel, f"panels[{name!r}]")
_positive_int(rebalance, "rebalance")
_positive_int(min_assets, "min_assets", minimum=2)
_positive_int(min_periods, "min_periods", minimum=2)
if window is not None:
_positive_int(window, "window", minimum=2)
if insufficient not in ("skip", "equal"):
raise ValueError("insufficient must be 'skip' or 'equal'")
names = list(panels)
index = _ordered_union([panels[name].index for name in names])
columns = _ordered_union([panels[name].columns for name in names])
aligned = {
name: _numeric(panels[name], f"panels[{name!r}]").reindex(index=index, columns=columns) for name in names
}
empty = [name for name in names if aligned[name].isna().all().all()]
if len(empty) == len(names):
raise ValueError("every factor panel is empty after alignment")
forward: pd.DataFrame | None = None
if method in _WEIGHTED_METHODS:
if returns is None:
raise ValueError(f"method={method!r} requires a forward-return panel")
forward = _numeric(returns, "returns").reindex(index=index, columns=columns)
explicit: Any = None
if method == "explicit":
if weights is None:
raise ValueError("method='explicit' requires weights")
if isinstance(weights, pd.DataFrame):
explicit = _table_weights(weights, names, index)
elif isinstance(weights, Mapping):
explicit = _static_weights(weights, names, index)
else:
raise ValueError("weights must be a mapping of factor -> number or a time x factor DataFrame")
factor_ic: dict[str, pd.Series] = {}
if method in _WEIGHTED_METHODS and forward is not None:
factor_ic = {name: _ic_series(aligned[name], forward, min_assets) for name in names}
if method == "equal":
weights_frame = _constant_weights(names, index, {name: 1.0 / len(names) for name in names})
elif method == "explicit" and isinstance(explicit, pd.DataFrame):
weights_frame = explicit
else:
weights_frame = _walk_forward_weights(
factor_ic,
index,
method=method,
window=window,
min_periods=min_periods,
rebalance=rebalance,
insufficient=insufficient,
)
prepared = {name: _cross_sectional_z(aligned[name]) if standardize else aligned[name] for name in names}
shape = (len(index), len(columns))
numerator = np.zeros(shape, dtype=float)
denominator = np.zeros(shape, dtype=float)
for name in names:
values = prepared[name].to_numpy(dtype=float)
weight = weights_frame[name].to_numpy(dtype=float)[:, None]
available = np.isfinite(values) & np.isfinite(weight)
numerator += np.where(available, weight * np.nan_to_num(values), 0.0)
denominator += np.where(available, np.abs(weight), 0.0)
composite = np.divide(
numerator,
denominator,
out=np.full(shape, np.nan),
where=denominator > 0,
)
diagnostics: dict[str, Any] = {
"method": method,
"weights_source": (
"caller_table"
if isinstance(explicit, pd.DataFrame)
else ("caller_mapping" if isinstance(explicit, Mapping) else method)
),
"standardize": bool(standardize),
"window": window,
"rebalance": rebalance,
"n_factors": len(names),
"n_observations": int(len(index)),
"n_assets": int(len(columns)),
"factor_names": names,
"factor_ic": pd.DataFrame({name: _ic_stats(factor_ic[name]) for name in names}).T if factor_ic else None,
"composite_ic": _ic_stats(
_ic_series(
pd.DataFrame(composite, index=index, columns=columns),
forward,
min_assets,
)
)
if forward is not None
else None,
"correlation": _mean_rank_correlation(prepared, min_assets) if len(names) > 1 else None,
"weight_turnover": _turnover(weights_frame),
"rows_without_weights": int(weights_frame.isna().all(axis=1).sum()),
"notes": [
"factor_ic and composite_ic are full-sample research diagnostics, "
"not the walk-forward weights used to build the composite",
"composite is the weighted mean of available factors per row, "
"renormalised by the available absolute weight",
f"weights are re-estimated every {rebalance} observation(s); set "
"rebalance to the holding horizon the strategy will use",
],
}
return CombinationResult(
composite=pd.DataFrame(composite, index=index, columns=columns),
weights=weights_frame,
diagnostics=diagnostics,
method=method,
)