Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(autoresearch): anchor deferred shadow scoring on the champion's people (#119301)

Andrew Maguire25ecfcb83c55
diff --git a/products/autoresearch/backend/dataset/labeling.py b/products/autoresearch/backend/dataset/labeling.py
index 7398241a4146..aa8a4937df46 100644
--- a/products/autoresearch/backend/dataset/labeling.py
+++ b/products/autoresearch/backend/dataset/labeling.py
@@ -1305,6 +1305,23 @@ def build_training_features_sql(
return sql, values
+def build_fixed_anchors_sql(*, anchor_person_ids: list[str], cutoff_ts: int | None) -> tuple[str, dict[str, Any]]:
+ """
+ Anchors at ``cutoff_ts`` for the persons in ``anchor_person_ids`` that are still live.
+
+ The live filter is the one every population uses (``_person_rows_sql``). A person deleted since
+ the ids were stored, including the losing side of a merge, drops out, so nothing scores a
+ deleted person or an obsolete id that events no longer resolve to.
+ """
+ if cutoff_ts is None:
+ raise ValueError("Fixed anchors need an explicit cutoff_ts")
+ live = _person_rows_sql("SELECT toUUID(arrayJoin({anchor_person_ids}))", [])
+ return (
+ f"SELECT id AS person_id, toInt({{cutoff_ts}}) AS cutoff_ts FROM ({live})",
+ {"anchor_person_ids": anchor_person_ids, "cutoff_ts": cutoff_ts},
+ )
+
+
def build_inference_features_sql(
*,
feature_sql: str,
@@ -1315,6 +1332,7 @@ def build_inference_features_sql(
target_definition: dict[str, Any] | None = None,
team: "Team | None" = None,
rolling: RollingSelection | None = None,
+ anchor_person_ids: list[str] | None = None,
) -> tuple[str, dict[str, Any]]:
"""
Build the inference-time query: the agent's feature_sql with {anchors}
@@ -1325,16 +1343,22 @@ def build_inference_features_sql(
Caller substitutes {lookback_days} in feature_sql before calling.
``rolling`` selects a rolling subset of the anchors, as in ``build_inference_anchors_sql``.
+ ``anchor_person_ids`` replaces the population with those of these persons who are still live
+ (``build_fixed_anchors_sql``), so a later query scores the people an earlier run scored even if
+ population data changed since.
"""
- anchors_sql, anchors_values = build_inference_anchors_sql(
- lookback_days=lookback_days,
- inference_population=inference_population,
- cutoff_ts=cutoff_ts,
- target_event=target_event,
- target_definition=target_definition,
- team=team,
- rolling=rolling,
- )
+ if anchor_person_ids is not None:
+ anchors_sql, anchors_values = build_fixed_anchors_sql(anchor_person_ids=anchor_person_ids, cutoff_ts=cutoff_ts)
+ else:
+ anchors_sql, anchors_values = build_inference_anchors_sql(
+ lookback_days=lookback_days,
+ inference_population=inference_population,
+ cutoff_ts=cutoff_ts,
+ target_event=target_event,
+ target_definition=target_definition,
+ team=team,
+ rolling=rolling,
+ )
# Wrap the inference anchors query as the {anchors} subquery — agent's
# feature_sql references columns (person_id, cutoff_ts) just like training.
anchors_subquery = f"({anchors_sql.strip()})"
diff --git a/products/autoresearch/backend/inference/AGENTS.md b/products/autoresearch/backend/inference/AGENTS.md
index ee151f199045..7020b7c0cc16 100644
--- a/products/autoresearch/backend/inference/AGENTS.md
+++ b/products/autoresearch/backend/inference/AGENTS.md
@@ -72,7 +72,7 @@ After a live champion run completes, `_score_shadow_set()` in `scoring.py` score
- **One feature query per distinct `features.sql`.** The digest of the bundle's SQL (`features_sql_digest()`) keys the materialized rows. The champion's rows serve a shadow model with the same SQL, and each other SQL runs once. A failed materialization is cached too. A result is released after the last model that uses it, so the phase does not hold one result per distinct SQL.
- **Person-less events.** A shadow event has `$autoresearch_model_role` `shadow`, no `$set`, and no person processing, and its `distinct_id` is unique per model and person (`_shadow_distinct_id()`), because ingestion deduplicates on timestamp, distinct_id, token, and event name, not on the UUID. Validation keys on `$autoresearch_person_id`. A shadow score never reaches the output person property. Right before its batch goes out, a shadow model checks that it is still in the set (`_require_still_in_shadow_set()`), in place of `_require_still_champion()`.
- **Isolated runs.** Each shadow model records its own inference run, with `metrics["shadow"]` set and `scheduled` false, so `find_unscorable_champion()` never reads it, even after the model becomes the champion. A shadow failure fails only that run. Repeatable shadow failures on `UNSCORABLE_AFTER_FAILED_DAYS` scoring days remove the model from the shadow set (`models_failing_repeatably_in_shadow()`). The champion's run records `metrics["shadow_models"]`: the models that completed, failed, or were skipped.
-- **Deferred in the daily sweep.** The sweep scores the champion with `defer_shadows`, and scores the shadow set later in its own activity, through `score_shadow_set_for_run()`, after every champion of the sweep has finished. A live run records `metrics["scored_persons_digest"]` (`persons_digest()` of the people it scored), and the deferred phase pairs each feature result against that digest, so the person list never crosses the Temporal payload. The deferred phase queries the champion's SQL again, because the champion's rows are gone with its activity. It scores nothing when the prediction date has passed. A model promoted since the champion's run does not score as a shadow. Manual scoring and `autoresearch_score` still score the shadow set in the same call.
+- **Deferred in the daily sweep.** The sweep scores the champion with `defer_shadows`, and scores the shadow set later in its own activity, through `score_shadow_set_for_run()`, after every champion of the sweep has finished. A live run records `metrics["scored_persons_digest"]` (`persons_digest()` of the people it scored), and the deferred phase pairs each feature result against that digest, so the person list never crosses the Temporal payload. The champion's run also stores the person ids it scored in object storage (`metrics["scored_persons_key"]`, `scored_persons_key()` in `../training/artifacts.py`). The deferred phase binds its anchors to those people (`anchor_person_ids`, `build_fixed_anchors_sql()`), so a population-property change during the sweep does not change who the shadows score. The fixed anchors keep only persons who are still live, through the same `_person_rows_sql()` filter every population uses, so a person deleted or merged away since the champion's run is neither scored nor emitted. The shadows then pair on the champion's people who are still live: their rows must key only stored people, and the live count is the expected row count. A shadow run records the champion's `rows_eligible`, so coverage stays true for a rolling champion. The deferred phase deletes the stored list when it ends, including when the activity skips the run or the run is gone with a deleted pipeline. Without a stored list it selects the population again. The deferred phase queries the champion's SQL again, because the champion's rows are gone with its activity. It scores nothing when the prediction date has passed. A model promoted since the champion's run does not score as a shadow. Manual scoring and `autoresearch_score` still score the shadow set in the same call.
- **The rolling ranking ignores shadow events.** `build_inference_anchors_sql()` counts only non-shadow predictions as scores, so shadow scoring does not change who the champion scores next.
- **Cost bound.** No new shadow model starts after `SHADOW_TIME_BUDGET_S` (20 minutes). The rest are skipped for that cadence. The inference activity timeout covers the champion's worst case, the budget, and one more model. The shadow activity timeout covers the budget and one more model.
- **Timing.** A live champion run records `metrics["scoring_seconds"]`, completed or failed. The deferred phase records `metrics["shadow_seconds"]` on the champion's run. A sweep run also records `metrics["coordinator_concurrency"]`.
diff --git a/products/autoresearch/backend/inference/sandbox.py b/products/autoresearch/backend/inference/sandbox.py
index e5ab08ac4901..705bb0775d32 100644
--- a/products/autoresearch/backend/inference/sandbox.py
+++ b/products/autoresearch/backend/inference/sandbox.py
@@ -71,6 +71,7 @@ from products.autoresearch.backend.dataset.labeling import (
MATERIALIZE_ROW_LIMIT,
TrainingSample,
TrainingSampleTooLarge,
+ build_fixed_anchors_sql,
build_inference_anchors_sql,
build_inference_features_sql,
build_prediction_coverage_sql,
@@ -541,6 +542,7 @@ def _materialize_score_data(
cutoff_ts: int | None = None,
user: User | None = None,
query_context: QueryContext = INTERACTIVE_QUERY,
+ anchor_person_ids: list[str] | None = None,
) -> InferenceRows:
"""
Predict run materialization: the bundle's feature SQL against the inference anchors
@@ -550,11 +552,29 @@ def _materialize_score_data(
A population at or above the cap scores a rolling subset: the ``ROLLING_SCORE_LIMIT``
persons whose last score by this pipeline is oldest.
+
+ ``anchor_person_ids`` scores those of these persons who are still live instead of the population,
+ with no rolling selection. Deferred shadow scoring passes the people the champion scored.
"""
- eligible = count_inference_anchors(
- team=team, pipeline=pipeline, cutoff_ts=cutoff_ts, user=user, query_context=query_context
- )
- rolling = rolling_selection(eligible=eligible, pipeline_id=str(pipeline.pk), cadence_days=pipeline.cadence_days)
+ if anchor_person_ids is not None:
+ anchors_sql, anchors_values = build_fixed_anchors_sql(anchor_person_ids=anchor_person_ids, cutoff_ts=cutoff_ts)
+ eligible = int(
+ _count_rows(
+ team=team,
+ sql=f"SELECT count() FROM ({anchors_sql})",
+ values=anchors_values,
+ user=user,
+ what="Fixed anchor count",
+ query_context=query_context,
+ )[0]
+ or 0
+ )
+ rolling = None
+ else:
+ eligible = count_inference_anchors(
+ team=team, pipeline=pipeline, cutoff_ts=cutoff_ts, user=user, query_context=query_context
+ )
+ rolling = rolling_selection(eligible=eligible, pipeline_id=str(pipeline.pk), cadence_days=pipeline.cadence_days)
feature_sql_resolved = feature_sql.replace("{lookback_days}", str(_feature_lookback_days(pipeline)))
score_sql, score_values = build_inference_features_sql(
feature_sql=feature_sql_resolved,
@@ -565,6 +585,7 @@ def _materialize_score_data(
target_definition=pipeline.target_definition,
team=team,
rolling=rolling,
+ anchor_person_ids=anchor_person_ids,
)
score_rows = _materialize_rows(
team=team, sql=score_sql, values=score_values, user=user, query_context=query_context
diff --git a/products/autoresearch/backend/inference/scoring.py b/products/autoresearch/backend/inference/scoring.py
index cced66ee6538..57cacb843834 100644
--- a/products/autoresearch/backend/inference/scoring.py
+++ b/products/autoresearch/backend/inference/scoring.py
@@ -101,7 +101,14 @@ from products.autoresearch.backend.inference.sandbox import (
)
from products.autoresearch.backend.models import AutoresearchModel, AutoresearchPipeline, AutoresearchRun
from products.autoresearch.backend.query import INTERACTIVE_QUERY, HogQLResult, QueryContext, run_hogql
-from products.autoresearch.backend.training.artifacts import ArtifactBundle, read_bundle
+from products.autoresearch.backend.training.artifacts import (
+ ArtifactBundle,
+ delete_scored_persons,
+ read_bundle,
+ read_scored_persons,
+ scored_persons_key,
+ write_scored_persons,
+)
from products.autoresearch.backend.training.recipe_validation import (
RecipeValidationError,
validate_model_class,
@@ -388,6 +395,8 @@ def run_inference_for_pipeline(
if not window.is_backfill:
_record_coverage(team=team, pipeline=pipeline, run=run, scored=scored, window=window, user=acting_user)
+ if not window.is_backfill and defer_shadows and scored.rows:
+ _store_scored_persons(run=run, person_ids=[str(row["distinct_id"]) for row in scored.rows])
if not window.is_backfill and not defer_shadows:
try:
outcome = _score_shadow_set(
@@ -396,6 +405,7 @@ def run_inference_for_pipeline(
champion=model,
persons=run.metrics["scored_persons_digest"] if scored.rows else None,
features=scored.features,
+ champion_rows_eligible=scored.rows_eligible,
window=window,
user=acting_user,
query_context=query_context,
@@ -410,6 +420,73 @@ def run_inference_for_pipeline(
return run
+def _pairs_with_champion(data: InferenceRows, *, persons: str, anchor_person_ids: list[str] | None) -> bool:
+ """
+ True when feature rows key the people the champion scored.
+
+ Fixed anchors already drop a person deleted or merged away since the champion's run, and the
+ materialization checks the rows against the live count, so the rows pair when they key only the
+ champion's people. Without fixed anchors the rows must key exactly the champion's people.
+ """
+ scored = {str(row["distinct_id"]) for row in data.rows}
+ if anchor_person_ids is not None:
+ return scored <= set(anchor_person_ids)
+ return persons_digest(scored) == persons
+
+
+def _store_scored_persons(*, run: AutoresearchRun, person_ids: list[str]) -> None:
+ """
+ Store the people a live run scored, so deferred shadow scoring anchors on exactly them.
+
+ A failure here never fails the champion's run. The deferred phase then selects the people
+ from the population again, as manual scoring does.
+ """
+ key = scored_persons_key(team_id=run.team_id, pipeline_id=str(run.pipeline_id), inference_run_id=str(run.pk))
+ try:
+ write_scored_persons(key, person_ids)
+ run.metrics["scored_persons_key"] = key
+ run.save(update_fields=["metrics"])
+ except Exception:
+ # The run already completed and emitted, so an error here must not reach the activity and start a retry.
+ logger.exception("autoresearch_scored_persons_store_failed", run_id=str(run.pk))
+ run.metrics.pop("scored_persons_key", None)
+ try:
+ delete_scored_persons(key)
+ except Exception:
+ logger.exception("autoresearch_scored_persons_delete_failed", run_id=str(run.pk))
+
+
+def _load_scored_persons(run: AutoresearchRun) -> list[str] | None:
+ key = (run.metrics or {}).get("scored_persons_key")
+ if not key:
+ return None
+ try:
+ return read_scored_persons(key)
+ except Exception:
+ logger.exception("autoresearch_scored_persons_read_failed", run_id=str(run.pk))
+ return None
+
+
+def discard_scored_persons_for(*, team_id: int, pipeline_id: str, inference_run_id: str) -> None:
+ """Delete the stored people of a run that no longer exists, from its ids alone."""
+ key = scored_persons_key(team_id=team_id, pipeline_id=pipeline_id, inference_run_id=inference_run_id)
+ try:
+ delete_scored_persons(key)
+ except Exception:
+ logger.exception("autoresearch_scored_persons_delete_failed", run_id=inference_run_id)
+
+
+def discard_scored_persons(run: AutoresearchRun) -> None:
+ """Delete the people a run stored for deferred shadow scoring. Every path that ends the deferred phase calls it."""
+ key = (run.metrics or {}).get("scored_persons_key")
+ if not key:
+ return
+ try:
+ delete_scored_persons(key)
+ except Exception:
+ logger.exception("autoresearch_scored_persons_delete_failed", run_id=str(run.pk))
+
+
def persons_digest(person_ids: Iterable[Any]) -> str:
"""An order-free digest of a set of person ids, so a later query can prove it selected the same people."""
return hashlib.sha256("\n".join(sorted({str(person_id) for person_id in person_ids})).encode("utf-8")).hexdigest()
@@ -808,6 +885,8 @@ def _score_shadow_set(
user: User,
query_context: QueryContext = INTERACTIVE_QUERY,
clock: Callable[[], float] = time.monotonic,
+ anchor_person_ids: list[str] | None = None,
+ champion_rows_eligible: int | None = None,
) -> ShadowScoringOutcome | None:
"""
Score every shadow-set model other than the champion against the people the champion's run
@@ -854,6 +933,8 @@ def _score_shadow_set(
window=window,
user=user,
query_context=query_context,
+ anchor_person_ids=anchor_person_ids,
+ champion_rows_eligible=champion_rows_eligible,
)
(completed if ok else failed).append(str(model.pk))
digest = digests[index]
@@ -878,39 +959,47 @@ def score_shadow_set_for_run(
Temporal payload. Each feature query binds the run's cutoff and must select exactly those people.
Returns None, and scores nothing, when the run did not complete, scored nobody, or its prediction
date has passed: a shadow score emitted after that date would not pair with the champion's.
+
+ The anchors are the people the run stored (``scored_persons_key``), and the stored list is
+ deleted once this phase ends, scored or not. Without a stored list the population is selected again.
"""
- started = time.monotonic()
- metrics = run.metrics or {}
- digest = metrics.get("scored_persons_digest")
- if (
- run.status != AutoresearchRun.Status.COMPLETED
- or metrics.get("shadow")
- or not digest
- or not run.rows_scored
- or run.model is None
- ):
- return None
- window = ScoringWindow.for_date(date.fromisoformat(metrics["prediction_date"]))
- if window.is_backfill:
- logger.warning("autoresearch_shadow_scoring_after_prediction_date", run_id=str(run.pk))
- return None
- pipeline = run.pipeline
- team = pipeline.team
- outcome = _score_shadow_set(
- team=team,
- pipeline=pipeline,
- champion=run.model,
- persons=digest,
- features=None,
- window=window,
- user=_acting_user(team=team, pipeline=pipeline, user=None),
- query_context=query_context,
- )
- if outcome is not None:
- run.metrics["shadow_models"] = outcome.as_metrics()
- run.metrics["shadow_seconds"] = _elapsed_s(started)
- run.save(update_fields=["metrics"])
- return outcome
+ try:
+ started = time.monotonic()
+ metrics = run.metrics or {}
+ digest = metrics.get("scored_persons_digest")
+ if (
+ run.status != AutoresearchRun.Status.COMPLETED
+ or metrics.get("shadow")
+ or not digest
+ or not run.rows_scored
+ or run.model is None
+ ):
+ return None
+ window = ScoringWindow.for_date(date.fromisoformat(metrics["prediction_date"]))
+ if window.is_backfill:
+ logger.warning("autoresearch_shadow_scoring_after_prediction_date", run_id=str(run.pk))
+ return None
+ pipeline = run.pipeline
+ team = pipeline.team
+ outcome = _score_shadow_set(
+ team=team,
+ pipeline=pipeline,
+ champion=run.model,
+ persons=digest,
+ features=None,
+ anchor_person_ids=_load_scored_persons(run),
+ champion_rows_eligible=metrics.get("rows_eligible"),
+ window=window,
+ user=_acting_user(team=team, pipeline=pipeline, user=None),
+ query_context=query_context,
+ )
+ if outcome is not None:
+ run.metrics["shadow_models"] = outcome.as_metrics()
+ run.metrics["shadow_seconds"] = _elapsed_s(started)
+ run.save(update_fields=["metrics"])
+ return outcome
+ finally:
+ discard_scored_persons(run)
def _read_shadow_bundle(model: AutoresearchModel) -> ArtifactBundle | Exception:
@@ -934,6 +1023,8 @@ def _score_shadow_model(
window: ScoringWindow,
user: User,
query_context: QueryContext,
+ anchor_person_ids: list[str] | None = None,
+ champion_rows_eligible: int | None = None,
) -> bool:
"""
Score one shadow model and record its run. The run is never ``scheduled``, so it cannot
@@ -954,6 +1045,7 @@ def _score_shadow_model(
window=window,
user=user,
query_context=query_context,
+ anchor_person_ids=anchor_person_ids,
)
result = score_via_sandbox(
team=team,
@@ -999,7 +1091,8 @@ def _score_shadow_model(
"stub": False,
"sandbox": True,
"holdout_auc": scored.holdout_auc,
- "rows_eligible": scored.rows_eligible,
+ # Fixed anchors count only the champion's people, so the champion's count keeps the coverage true.
+ "rows_eligible": champion_rows_eligible if champion_rows_eligible is not None else scored.rows_eligible,
}
)
run.completed_at = django_timezone.now()
@@ -1017,14 +1110,17 @@ def _shadow_score_data(
window: ScoringWindow,
user: User,
query_context: QueryContext,
+ anchor_person_ids: list[str] | None = None,
) -> InferenceRows:
"""
The feature rows for ``feature_sql``, materialized once per digest. A failure is cached too,
so a second model with the same broken SQL fails without a second query.
- The anchors bind the run's cutoff, so the selection repeats the champion's. Rows that key a
- different set of people (an event that landed late moved the rolling ranking) fail the
- models that use them, because unpaired scores would not compare with the champion's.
+ The anchors bind the run's cutoff, so the selection repeats the champion's. With
+ ``anchor_person_ids`` the anchors are exactly the people the champion scored, so a person
+ merge or a property change since the champion's run cannot change the selection. Rows that
+ key a different set of people still fail the models that use them, because unpaired scores
+ would not compare with the champion's.
"""
digest = features_sql_digest(feature_sql)
if digest not in materialized:
@@ -1036,8 +1132,9 @@ def _shadow_score_data(
cutoff_ts=window.cutoff_ts,
user=user,
query_context=query_context,
+ anchor_person_ids=anchor_person_ids,
)
- if persons_digest(row["distinct_id"] for row in data.rows) != persons:
+ if not _pairs_with_champion(data, persons=persons, anchor_person_ids=anchor_person_ids):
raise InferenceRunError(
"The shadow feature query did not select the people the champion scored; "
"refusing to emit scores that do not pair with the champion's"
diff --git a/products/autoresearch/backend/inference/test_inference.py b/products/autoresearch/backend/inference/test_inference.py
index 85df1e366ea0..60751724211e 100644
--- a/products/autoresearch/backend/inference/test_inference.py
+++ b/products/autoresearch/backend/inference/test_inference.py
@@ -20,6 +20,7 @@ from posthog.exceptions import (
ClickHouseQueryMemoryLimitExceeded,
ClickHouseQueryTimeOut,
)
+from posthog.storage.object_storage import ObjectStorageError
from products.autoresearch.backend import query as autoresearch_query
from products.autoresearch.backend.dataset.labeling import PREDICTION_EVENT_NAME, ROLLING_SCORE_LIMIT
@@ -1099,6 +1100,7 @@ class _Cadence:
run: AutoresearchRun
capture: MagicMock
materialize: MagicMock
+ stored_persons: dict[str, list[str]]
@time_machine.travel("2026-09-11T12:00:00Z", tick=False)
@@ -1131,7 +1133,12 @@ class TestShadowScoring(TeamScopedTestMixin, BaseTest):
)
def _run(
- self, *, failing: dict[str, Exception] | None = None, shadow_rows=_SHADOW_ROWS, deferred: bool = False
+ self,
+ *,
+ failing: dict[str, Exception] | None = None,
+ shadow_rows=_SHADOW_ROWS,
+ deferred: bool = False,
+ store_fails: bool = False,
) -> _Cadence:
failing = failing or {}
materialize = MagicMock(return_value=InferenceRows(rows=shadow_rows, eligible=len(shadow_rows)))
@@ -1140,7 +1147,8 @@ class TestShadowScoring(TeamScopedTestMixin, BaseTest):
model = kwargs["model"]
if str(model.pk) in failing:
raise failing[str(model.pk)]
- data = kwargs.get("score_data") or InferenceRows(rows=_SHADOW_ROWS, eligible=len(_SHADOW_ROWS))
+ # The champion scores a rolling part of a larger population, so its eligible count exceeds its rows.
+ data = kwargs.get("score_data") or InferenceRows(rows=_SHADOW_ROWS, eligible=50)
features_sql = self.sql_by_prefix[model.artifact_prefix]
return SandboxScoreResult(
scored_rows=[{**row, "p_y": 0.4} for row in data.rows],
@@ -1152,7 +1160,17 @@ class TestShadowScoring(TeamScopedTestMixin, BaseTest):
)
capture = _capture_accepting_everything()
+ stored_persons: dict[str, list[str]] = {}
+
+ def write_scored_persons(key: str, ids: list[str]) -> None:
+ if store_fails:
+ raise ObjectStorageError("storage unavailable")
+ stored_persons[key] = sorted(ids)
+
with (
+ patch.object(scoring, "write_scored_persons", side_effect=write_scored_persons),
+ patch.object(scoring, "read_scored_persons", side_effect=stored_persons.get),
+ patch.object(scoring, "delete_scored_persons", side_effect=lambda key: stored_persons.pop(key, None)),
patch.object(
scoring,
"read_bundle",
@@ -1170,7 +1188,7 @@ class TestShadowScoring(TeamScopedTestMixin, BaseTest):
assert capture.call_count == 1
score_shadow_set_for_run(AutoresearchRun.objects.get(pk=run.pk))
run.refresh_from_db()
- return _Cadence(run=run, capture=capture, materialize=materialize)
+ return _Cadence(run=run, capture=capture, materialize=materialize, stored_persons=stored_persons)
def _shadow_runs(self, model: AutoresearchModel) -> list[AutoresearchRun]:
return list(AutoresearchRun.objects.filter(model=model, run_type=AutoresearchRun.RunType.INFERENCE))
@@ -1179,18 +1197,33 @@ class TestShadowScoring(TeamScopedTestMixin, BaseTest):
# again. Either way the two models with the other SQL share one query, although the shadow set
# scores same_sql between them.
@parameterized.expand(
- [("in_process", False, [_OTHER_FEATURE_SQL]), ("deferred", True, [_ANCHORS_FEATURE_SQL, _OTHER_FEATURE_SQL])]
+ [
+ ("in_process", False, False, [_OTHER_FEATURE_SQL]),
+ ("deferred", True, False, [_ANCHORS_FEATURE_SQL, _OTHER_FEATURE_SQL]),
+ ("deferred_without_stored_people", True, True, [_ANCHORS_FEATURE_SQL, _OTHER_FEATURE_SQL]),
+ ]
)
- def test_each_shadow_model_scores_the_champions_people_person_less(self, _name, deferred, expected_queries):
+ def test_each_shadow_model_scores_the_champions_people_person_less(
+ self, _name, deferred, store_fails, expected_queries
+ ):
other_a = self._model("other_a", _OTHER_FEATURE_SQL)
same_sql = self._model("same_sql", _ANCHORS_FEATURE_SQL)
other_b = self._model("other_b", _OTHER_FEATURE_SQL)
- cadence = self._run(deferred=deferred)
+ cadence = self._run(deferred=deferred, store_fails=store_fails)
run, capture, materialize = cadence.run, cadence.capture, cadence.materialize
assert run.status == AutoresearchRun.Status.COMPLETED
assert sorted(c.kwargs["feature_sql"] for c in materialize.call_args_list) == sorted(expected_queries)
+ # A deferred phase anchors on the people the champion scored, not on a second population query.
+ # A failed store leaves the champion's run complete, and the phase selects the population again.
+ champion_people = (
+ sorted(str(row["distinct_id"]) for row in _SHADOW_ROWS) if deferred and not store_fails else None
+ )
+ assert [c.kwargs.get("anchor_person_ids") for c in materialize.call_args_list] == [champion_people] * len(
+ expected_queries
+ )
+ assert cadence.stored_persons == {}
assert sorted(run.metrics["shadow_models"]["completed"]) == sorted(
str(m.pk) for m in (same_sql, other_a, other_b)
)
@@ -1201,6 +1234,7 @@ class TestShadowScoring(TeamScopedTestMixin, BaseTest):
assert shadow_run.scheduled is False
assert shadow_run.metrics["prediction_date"] == run.metrics["prediction_date"]
assert shadow_run.metrics["horizon_days"] == run.metrics["horizon_days"]
+ assert shadow_run.metrics["rows_eligible"] == run.metrics["rows_eligible"] == 50
champion_call, *shadow_calls = capture.call_args_list
for event in champion_call.kwargs["events"]:
@@ -1229,7 +1263,14 @@ class TestShadowScoring(TeamScopedTestMixin, BaseTest):
"model_load_failed",
True,
),
- ("deferred_unpaired_rows", None, _SHADOW_ROWS[:1], "other", True),
+ # With fixed anchors, rows that key someone the champion did not score cannot pair.
+ (
+ "deferred_unpaired_rows",
+ None,
+ [_SHADOW_ROWS[0], {"distinct_id": "stranger", "events_total": 1}],
+ "other",
+ True,
+ ),
]
)
def test_a_failing_shadow_model_never_touches_the_champion(self, _name, error, shadow_rows, failure_kind, deferred):
diff --git a/products/autoresearch/backend/inference/test_sandbox_inference.py b/products/autoresearch/backend/inference/test_sandbox_inference.py
index 0d4002aee34e..47aa4ebc4a48 100644
--- a/products/autoresearch/backend/inference/test_sandbox_inference.py
+++ b/products/autoresearch/backend/inference/test_sandbox_inference.py
@@ -1,6 +1,8 @@
import io
import json
+import uuid
import base64
+from datetime import UTC, datetime
from decimal import Decimal
from typing import Any
@@ -15,6 +17,7 @@ from parameterized import parameterized
from posthog.api.capture import CaptureInternalResult
from posthog.hogql_queries.query_runner import ExecutionMode
from posthog.models.user import User
+from posthog.test.persons import create_person, delete_person
from products.autoresearch.backend.dataset.labeling import ROLLING_SCORE_LIMIT
from products.autoresearch.backend.evaluation.segment_thresholds import training_base_rate
@@ -207,6 +210,27 @@ class TestMaterializeData(TeamScopedTestMixin, BaseTest):
assert [r["distinct_id"] for r in score_data.rows] == ["s1", "s2"]
assert score_data.eligible == 2
+ def test_score_data_with_fixed_anchors_scores_only_the_live_people(self):
+ pipeline = self._pipeline()
+ live = [create_person(team=self.team, distinct_ids=[f"live-{i}"], is_identified=True) for i in range(2)]
+ deleted = create_person(team=self.team, distinct_ids=["deleted"], is_identified=True)
+ delete_person(deleted)
+ people = [str(p.uuid) for p in [*live, deleted]] + [str(uuid.uuid4())]
+
+ with patch.object(sandbox_inference, "count_inference_anchors") as count:
+ score_data = _materialize_score_data(
+ team=self.team,
+ pipeline=pipeline,
+ feature_sql="SELECT a.person_id AS distinct_id, count() AS events_total FROM {anchors} a GROUP BY a.person_id",
+ cutoff_ts=int(datetime(2026, 9, 11, tzinfo=UTC).timestamp()),
+ anchor_person_ids=people,
+ )
+
+ count.assert_not_called()
+ # A person deleted (or merged away) since the ids were stored, or never present, is not scored.
+ assert sorted(str(row["distinct_id"]) for row in score_data.rows) == sorted(str(p.uuid) for p in live)
+ assert score_data.eligible == 2
+
@parameterized.expand(
[
("rolling_subset_scores", ROLLING_SCORE_LIMIT, False),
diff --git a/products/autoresearch/backend/temporal/test_workflows.py b/products/autoresearch/backend/temporal/test_workflows.py
index 5f793e1e43c9..e7997de5d255 100644
--- a/products/autoresearch/backend/temporal/test_workflows.py
+++ b/products/autoresearch/backend/temporal/test_workflows.py
@@ -62,11 +62,13 @@ from products.autoresearch.backend.temporal.workflows import (
activity_mark_daily_sweep,
activity_remove_person_property,
activity_run_inference,
+ activity_run_shadow_scoring,
activity_run_validation,
daily_sweep_key,
evaluate_pipeline_outcome,
)
from products.autoresearch.backend.testing import TeamScopedTestMixin
+from products.autoresearch.backend.training.artifacts import scored_persons_key
def _inference(status: str = "completed", error: Optional[str] = None) -> InferenceWorkflowResult:
@@ -367,6 +369,41 @@ class TestCoordinatorActivities(TeamScopedTestMixin, BaseTest):
assert mock_inference.call_args.kwargs["run"] == (earlier if reused else None)
assert mock_inference.call_args.kwargs["extra_metrics"] == {"workflow_run_id": env.info.workflow_run_id}
+ @parameterized.expand(
+ [
+ ("paused", AutoresearchPipeline.Status.PAUSED),
+ ("archived", AutoresearchPipeline.Status.ARCHIVED),
+ ("run_deleted_with_its_pipeline", None),
+ ]
+ )
+ def test_shadow_activity_that_skips_a_run_deletes_its_stored_people(
+ self, _name: str, status: Optional[str]
+ ) -> None:
+ pipeline = self._create_pipeline()
+ run = AutoresearchRun.objects.create(
+ pipeline=pipeline,
+ run_type=AutoresearchRun.RunType.INFERENCE,
+ scheduled=True,
+ status=AutoresearchRun.Status.COMPLETED,
+ started_at=django_timezone.now() - timedelta(minutes=10),
+ metrics={"prediction_date": "2026-09-11", "horizon_days": 30},
+ )
+ key = scored_persons_key(team_id=self.team.id, pipeline_id=str(pipeline.id), inference_run_id=str(run.pk))
+ AutoresearchRun.objects.filter(pk=run.pk).update(metrics={**run.metrics, "scored_persons_key": key})
+ if status is None:
+ AutoresearchRun.objects.filter(pk=run.pk).delete()
+ else:
+ AutoresearchPipeline.objects.filter(pk=pipeline.pk).update(status=status)
+
+ with patch("products.autoresearch.backend.inference.scoring.delete_scored_persons") as delete:
+ result = ActivityEnvironment().run(
+ activity_run_shadow_scoring,
+ RunShadowScoringInput(pipeline_id=str(pipeline.id), team_id=self.team.id, run_id=str(run.pk)),
+ )
+
+ assert result.status == "skipped"
+ delete.assert_called_once_with(key)
+
def test_load_active_pipelines_fails_inference_runs_left_running(self) -> None:
pipeline = self._create_pipeline()
diff --git a/products/autoresearch/backend/temporal/workflows.py b/products/autoresearch/backend/temporal/workflows.py
index 35aec7bd3380..55ae1ba4390f 100644
--- a/products/autoresearch/backend/temporal/workflows.py
+++ b/products/autoresearch/backend/temporal/workflows.py
@@ -47,7 +47,12 @@ with workflow.unsafe.imports_passed_through():
)
from products.autoresearch.backend.inference.person_property import remove_output_person_property
from products.autoresearch.backend.inference.sandbox import SandboxInferenceError, _resolve_acting_user
- from products.autoresearch.backend.inference.scoring import run_inference_for_pipeline, score_shadow_set_for_run
+ from products.autoresearch.backend.inference.scoring import (
+ discard_scored_persons,
+ discard_scored_persons_for,
+ run_inference_for_pipeline,
+ score_shadow_set_for_run,
+ )
from products.autoresearch.backend.models import (
AutoresearchModel,
AutoresearchPipeline,
@@ -343,7 +348,12 @@ def activity_run_shadow_scoring(inp: RunShadowScoringInput) -> RunShadowScoringR
.filter(pk=inp.run_id, pipeline__pk=inp.pipeline_id, run_type=AutoresearchRun.RunType.INFERENCE)
.first()
)
- if run is None or run.pipeline.status not in _LIVE_STATUSES or _sweep_access_block(run.pipeline):
+ if run is None:
+ # A deleted pipeline cascades to its runs, so the key comes from the input to clean up the stored people.
+ discard_scored_persons_for(team_id=inp.team_id, pipeline_id=inp.pipeline_id, inference_run_id=inp.run_id)
+ return RunShadowScoringResult(completed=0, failed=0, skipped=0, status="skipped")
+ if run.pipeline.status not in _LIVE_STATUSES or _sweep_access_block(run.pipeline):
+ discard_scored_persons(run)
return RunShadowScoringResult(completed=0, failed=0, skipped=0, status="skipped")
outcome = score_shadow_set_for_run(run, query_context=SCORING_QUERY)
if outcome is None:
diff --git a/products/autoresearch/backend/training/artifacts.py b/products/autoresearch/backend/training/artifacts.py
index f90d1f1638ef..88a91aa74e17 100644
--- a/products/autoresearch/backend/training/artifacts.py
+++ b/products/autoresearch/backend/training/artifacts.py
@@ -14,6 +14,7 @@ naturally and bundles never collide across tenants.
from __future__ import annotations
import re
+import gzip
import hashlib
from pathlib import Path
@@ -182,6 +183,29 @@ def bundle_prefix(*, team_id: int, pipeline_id: str, training_run_id: str) -> st
return f"{folder}/autoresearch/team_{team_id}/pipeline_{pipeline_id}/run_{training_run_id}"
+def scored_persons_key(*, team_id: int, pipeline_id: str, inference_run_id: str) -> str:
+ """Object-storage key for the person ids one live inference run scored."""
+ folder = settings.OBJECT_STORAGE_TASKS_FOLDER
+ return f"{folder}/autoresearch/team_{team_id}/pipeline_{pipeline_id}/inference_{inference_run_id}/scored_persons.txt.gz"
+
+
+def write_scored_persons(key: str, person_ids: list[str]) -> None:
+ _require_storage()
+ object_storage.write(key, gzip.compress("\n".join(sorted(person_ids)).encode("utf-8")))
+
+
+def read_scored_persons(key: str) -> list[str] | None:
+ """The stored person ids, or None when the object is absent."""
+ content = object_storage.read_bytes(key, missing_ok=True)
+ if content is None:
+ return None
+ return [line for line in gzip.decompress(content).decode("utf-8").split("\n") if line]
+
+
+def delete_scored_persons(key: str) -> None:
+ object_storage.delete(key)
+
+
def write_bundle(prefix: str, bundle: ArtifactBundle) -> None:
"""Write all bundle files under ``prefix``, with the same checks as a per-file upload."""
for name, content in bundle.as_files().items():