Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(quantlib): purge groups whose labels reach into the test block

Lanre Shittu9befcbae9de6
diff --git a/agent/src/quantlib/crossvalidation.py b/agent/src/quantlib/crossvalidation.py
index 1e33f07a5..d47290830 100644
--- a/agent/src/quantlib/crossvalidation.py
+++ b/agent/src/quantlib/crossvalidation.py
@@ -291,6 +291,7 @@ def purged_kfold_splits(
def group_purged_kfold_splits(
groups: Sequence[object] | pd.Series | np.ndarray,
+ label_end_times: pd.Series | Sequence[int] | np.ndarray | None = None,
n_folds: int = 5,
embargo_fraction: float = DEFAULT_EMBARGO_FRACTION,
) -> Iterator[Split]:
@@ -302,6 +303,10 @@ def group_purged_kfold_splits(
Args:
groups: Group identifiers (e.g. dates or bar index) for each sample row in order.
+ label_end_times: Where each row's label resolves, in the same positional form
+ :func:`purged_kfold_splits` takes. When None, each label is assumed to
+ resolve within its own group, so purging removes only the boundary and
+ the embargo does the remaining work.
n_folds: Number of folds, at least :data:`MIN_FOLDS`.
embargo_fraction: Fraction of unique ordered groups embargoed after each test block.
@@ -339,6 +344,22 @@ def group_purged_kfold_splits(
for g in group_to_rows:
group_to_rows[g] = np.array(group_to_rows[g], dtype=int)
+ # Position of each group in chronological order, and the same per row --
+ # used below to translate a row-level label span into the furthest group
+ # a group's own rows reach into.
+ group_order = {g: i for i, g in enumerate(unique_groups)}
+ row_group_order = np.array([group_order[g] for g in grp_array])
+
+ if label_end_times is None:
+ # A label resolves within its own group: it can never reach forward
+ # into a later group, so there is nothing for backward purging to do.
+ group_reach = np.arange(n_groups)
+ else:
+ label_ends = _as_label_spans(label_end_times, n_samples)
+ group_reach = np.array(
+ [row_group_order[label_ends[group_to_rows[g]]].max() for g in unique_groups]
+ )
+
embargo_groups = int(round(n_groups * embargo_fraction))
boundaries = np.linspace(0, n_groups, n_folds + 1).astype(int)
@@ -351,13 +372,22 @@ def group_purged_kfold_splits(
embargo_end_g = min(n_groups, stop_g + embargo_groups)
embargo_groups_set = set(unique_groups[stop_g:embargo_end_g])
+ # Symmetric purge: an earlier group whose own labels reach forward
+ # into the test block overlaps it just as much as a later group
+ # embargoed for starting shortly after -- see the module docstring.
+ purge_groups_set = {
+ unique_groups[g_idx]
+ for g_idx in range(start_g)
+ if group_reach[g_idx] >= start_g
+ }
+
test_rows_list = [group_to_rows[g] for g in unique_groups[start_g:stop_g]]
test_rows = np.concatenate(test_rows_list) if test_rows_list else np.array([], dtype=int)
if test_rows.size == 0:
continue
train_rows_list = []
for g in unique_groups:
- if g not in test_groups and g not in embargo_groups_set:
+ if g not in test_groups and g not in embargo_groups_set and g not in purge_groups_set:
train_rows_list.append(group_to_rows[g])
train_rows = np.concatenate(train_rows_list) if train_rows_list else np.array([], dtype=int)
@@ -367,11 +397,12 @@ def group_purged_kfold_splits(
test_rows.sort()
embargoed_count = sum(len(group_to_rows[g]) for g in embargo_groups_set)
+ purged_count = sum(len(group_to_rows[g]) for g in purge_groups_set)
yield Split(
train=train_rows,
test=test_rows,
- purged=0,
+ purged=purged_count,
embargoed=embargoed_count,
test_bounds=(int(test_rows.min()), int(test_rows.max())),
)
diff --git a/agent/tests/quantlib/test_crossvalidation.py b/agent/tests/quantlib/test_crossvalidation.py
index 1fe27f48a..2702d911e 100644
--- a/agent/tests/quantlib/test_crossvalidation.py
+++ b/agent/tests/quantlib/test_crossvalidation.py
@@ -387,6 +387,76 @@ def test_group_purged_kfold_splits_input_validation():
list(group_purged_kfold_splits([], n_folds=2))
+def test_group_purged_kfold_splits_leaks_a_forward_looking_label_without_purging():
+ """Groups adjacent to the test block are embargoed, but a group further
+ back whose own label reaches forward into the test window was never
+ checked at all -- group_purged_kfold_splits hardcoded purged=0 and took
+ no label_end_times, unlike every other splitter in this module. This
+ reproduces the leak directly against the module's own leakage auditor.
+ """
+ n_dates, n_assets = 10, 2
+ dates = np.repeat(np.arange(n_dates), n_assets)
+ n = len(dates)
+ # Each row's label resolves 3 rows (~1.5 dates) later.
+ label_end_times = np.minimum(np.arange(n) + 3, n - 1)
+
+ splits = list(
+ group_purged_kfold_splits(dates, n_folds=5, embargo_fraction=0.3)
+ )
+ reports = [
+ detect_boundary_leakage(split, label_end_times, n_samples=n, embargo_size=3)
+ for split in splits
+ ]
+
+ assert any(not report.clean for report in reports)
+
+
+def test_group_purged_kfold_splits_purges_a_forward_looking_label():
+ n_dates, n_assets = 10, 2
+ dates = np.repeat(np.arange(n_dates), n_assets)
+ n = len(dates)
+ label_end_times = np.minimum(np.arange(n) + 3, n - 1)
+
+ splits = list(
+ group_purged_kfold_splits(
+ dates, label_end_times, n_folds=5, embargo_fraction=0.3
+ )
+ )
+
+ for split in splits:
+ report = detect_boundary_leakage(
+ split, label_end_times, n_samples=n, embargo_size=3
+ )
+ assert report.clean, f"leak: overlapping={report.overlapping.tolist()}"
+
+
+def test_group_purged_kfold_splits_reports_purged_count():
+ n_dates, n_assets = 10, 2
+ dates = np.repeat(np.arange(n_dates), n_assets)
+ n = len(dates)
+ label_end_times = np.minimum(np.arange(n) + 3, n - 1)
+
+ splits = list(
+ group_purged_kfold_splits(
+ dates, label_end_times, n_folds=5, embargo_fraction=0.3
+ )
+ )
+
+ assert any(split.purged > 0 for split in splits)
+
+
+def test_group_purged_kfold_splits_without_label_end_times_is_unchanged():
+ """The default (no label_end_times) path must behave exactly as before:
+ no backward purging, purged always 0."""
+ n_dates, n_assets = 100, 10
+ dates = np.repeat(np.arange(n_dates), n_assets)
+
+ splits = list(group_purged_kfold_splits(dates, n_folds=5, embargo_fraction=0.05))
+
+ for split in splits:
+ assert split.purged == 0
+
+
def test_timestamp_label_index_must_be_ordered_and_unique():
index = pd.to_datetime(["2024-01-01", "2024-01-03", "2024-01-02", "2024-01-04"])
ends = pd.Series(index, index=index)