Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): score the newest versions of the served family as a recent role (#118967)

posthog[bot]3f680b3b7b4c
diff --git a/docs/internal/inbox-ranking-training.md b/docs/internal/inbox-ranking-training.md
index 90f972023026..fb7d63202b9e 100644
--- a/docs/internal/inbox-ranking-training.md
+++ b/docs/internal/inbox-ranking-training.md
@@ -189,7 +189,7 @@ Models trained before thresholds existed have none, and no other value stands in
- Population: D's newborn pool.
- Score: per report and model key, the earliest scored event in D with this deployment's `environment`.
-- `model_role`: `served` when the event's roles hold it, else the first role (`daily_candidate`, `cross_family`, `pinned`), else `candidate`.
+- `model_role`: `served` when the event's roles hold it, else the first role (`recent`, `cross_family`, `pinned`), else `candidate`. Grades written before the `recent` role carry `daily_candidate` for the same read.
- `classification_threshold` comes from `threshold_<head>`, and is null when the event has none.
- Asset metadata: `served_pool_coverage`, `pool_coverage_by_model`, rows per model version, and the heads without a threshold.
diff --git a/posthog/settings/object_storage.py b/posthog/settings/object_storage.py
index a6df872cb704..03956619829c 100644
--- a/posthog/settings/object_storage.py
+++ b/posthog/settings/object_storage.py
@@ -115,6 +115,10 @@ INBOX_RANKING_LABELS_REFRESH_MAX_RUNS = get_from_env("INBOX_RANKING_LABELS_REFRE
# no mirror.
INBOX_RANKING_SERVING_MIRROR_BUCKET = os.getenv("INBOX_RANKING_SERVING_MIRROR_BUCKET", "")
INBOX_RANKING_SERVING_MIRROR_REGION = os.getenv("INBOX_RANKING_SERVING_MIRROR_REGION", "")
+# Serving manifest (products/signals/dags/inbox_ranking/training/serving.py): how many of the served
+# family's newest versions the sweep also scores, read from the last 2N days. Each one costs the
+# sweep one more model per pass, and above 4 they take the pinned and cross-family entries' places.
+INBOX_RANKING_SERVING_RECENT_VERSIONS = get_from_env("INBOX_RANKING_SERVING_RECENT_VERSIONS", 3, type_cast=int)
# Scorer (products/signals/backend/ranking/scorer.py): report ids per ClickHouse vector read. A
# larger call is paged at this size. The sweep also gives one scorer call at most this many ids, which
# bounds the vectors and matrices one call holds.
diff --git a/products/signals/backend/ranking/serving_manifest.py b/products/signals/backend/ranking/serving_manifest.py
index 481adfce6279..8dc6c4dba352 100644
--- a/products/signals/backend/ranking/serving_manifest.py
+++ b/products/signals/backend/ranking/serving_manifest.py
@@ -21,9 +21,10 @@ from products.signals.backend.artefact_schemas import MAX_RANKING_MODEL_RESULTS,
# The one role a reader acts on, taken from the artefact schema so the manifest and the
# `ranking_score` rows it produces name the served model the same way.
SERVED_ROLE = RANKING_SERVED_ROLE
-# The served family's model for the partition being trained, carried so a paired read compares the
-# champion against the day's fit without a second scoring pass.
-DAILY_CANDIDATE_ROLE = "daily_candidate"
+# One of the newest versions of the served model's family, carried so each version collects online
+# scores for several days and a paired read compares it against the served model without a second
+# scoring pass.
+RECENT_ROLE = "recent"
# Another family's champion, scored alongside so a later interleaving needs no rescoring.
CROSS_FAMILY_ROLE = "cross_family"
# A model an owner keeps in the manifest through the `pin` override, so a `served` override can
diff --git a/products/signals/backend/test/test_ranking_scoring.py b/products/signals/backend/test/test_ranking_scoring.py
index d57e0ca2d9d4..f6fcf01a396e 100644
--- a/products/signals/backend/test/test_ranking_scoring.py
+++ b/products/signals/backend/test/test_ranking_scoring.py
@@ -40,12 +40,12 @@ from products.signals.backend.ranking.overrides import RankingOverrides
from products.signals.backend.ranking.scorer import NO_PROJECT_VECTORS, NO_VECTOR, ScoringError, score_reports
from products.signals.backend.ranking.serving_manifest import (
CROSS_FAMILY_ROLE,
- DAILY_CANDIDATE_ROLE,
MANIFEST_SERVED_ROLE,
METADATA_FILE,
ONNX_MODEL_FILE,
ONNX_MODEL_KIND,
PINNED_ROLE,
+ RECENT_ROLE,
SERVED_OVERRIDE_ROLE,
SERVED_ROLE,
ServingManifest,
@@ -339,12 +339,12 @@ class TestModelStore(_StoreTestMixin, SimpleTestCase):
load_serving_set()
model_reads = [key for key in self.store.reads if key != serving_manifest_key(PREFIX)]
- self.store.publish_manifest([served.model_copy(update={"roles": [SERVED_ROLE, DAILY_CANDIDATE_ROLE]})])
+ self.store.publish_manifest([served.model_copy(update={"roles": [SERVED_ROLE, RECENT_ROLE]})])
serving = load_serving_set()
assert serving is not None
assert [key for key in self.store.reads if key != serving_manifest_key(PREFIX)] == model_reads
- assert serving.served.entry.roles == [SERVED_ROLE, DAILY_CANDIDATE_ROLE]
+ assert serving.served.entry.roles == [SERVED_ROLE, RECENT_ROLE]
class _FakeVectors:
@@ -408,7 +408,7 @@ class TestScorer(_ScorerTestMixin, SimpleTestCase):
def test_one_vector_read_per_rendering_whatever_the_number_of_models(self) -> None:
served = self._served()
candidate = self.store.publish_model(
- "report_embeddings", REPORT_EMBEDDINGS_FEATURE_SET, version=OLDER_VERSION, roles=[DAILY_CANDIDATE_ROLE]
+ "report_embeddings", REPORT_EMBEDDINGS_FEATURE_SET, version=OLDER_VERSION, roles=[RECENT_ROLE]
)
challenger = self._challenger("nn", REPORT_EMBEDDINGS_FEATURE_SET, model_kind=ONNX_MODEL_KIND)
self.store.publish_manifest([served, candidate, challenger])
@@ -553,7 +553,7 @@ class TestScorerPersists(_ScorerTestMixin, BaseTest):
self.team_id = self.team.id
served = self._served(thresholds={"open": 0.0})
candidate = self.store.publish_model(
- "report_embeddings", REPORT_EMBEDDINGS_FEATURE_SET, version=OLDER_VERSION, roles=[DAILY_CANDIDATE_ROLE]
+ "report_embeddings", REPORT_EMBEDDINGS_FEATURE_SET, version=OLDER_VERSION, roles=[RECENT_ROLE]
)
manifest = self.store.publish_manifest([served, candidate])
scored, unscored = self._report(self.team.id), self._report(self.team.id)
diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.md
index 8bfd9dad7ab3..d106af6504ec 100644
--- a/products/signals/dags/inbox_ranking/AGENTS.md
+++ b/products/signals/dags/inbox_ranking/AGENTS.md
@@ -34,6 +34,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- The serving path reads the deployment's own object store, never the dataset bucket: the Temporal workers have no credential for that bucket, and keeping it that way is why the manifest asset copies what the sweep needs. `serving/models/<key>/` holds a version's `metadata.json` and its model files, `serving/manifest.json` names the models one pass scores. The holdout boosters grade a promotion and must stay behind.
- An entry's files come from `ServingManifestEntry.files`: one `<head>.ubj` per head for `xgboost`, and the one `model.onnx` that `model_file` names for `onnx`. The copy step and the model store read that property, and the pin check reads the same names from each head's `file` in `metadata.json`, so a new model kind with another file layout keeps the two in step. Keep `onnxruntime` and `xgboost` imports inside the handlers' `load`: a test pins that importing the sweep loads neither.
- The manifest schema and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, next to the feature contract, because the dag writes them and the sweep reads them. Its validators run at write time, so the dag refuses a manifest the sweep could not act on; the entry cap is the `ranking_score` artefact's, so a manifest can never produce a row the artefact refuses.
+- The `recent` role keeps the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` versions of the served `model_name`, read from the last 2N partitions, so each version gets several days of online scores. Each one passes the pin check before composition, because one missing file fails the publish. Keep the order under the cap as served, recent, pinned, cross-family. Each recent entry is one more model per sweep pass, so measure the sweep's duration, peak RSS and scored and deferred counts on the `inbox_ranking_sweep_finished` log before and after a change to N.
- Publishing order is load-bearing. Every model copy finishes before `manifest.json` is written, so a failed copy leaves the previous manifest and the previous models serving. A model version is immutable, so a version whose `metadata.json` is already at the target is not copied again. A run with nothing to publish, such as a run without `served.json`, writes nothing rather than an empty manifest. The served model passes the same check as a pin (`_pin_metadata`) before composition.
- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror: they are per-team data of this deployment, so the mirror's manifest names none. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.
- `HEADS` holds only the heads the inbox reads (`HeadRanker` in `backend/ranking/inbox_sort.py`). Add a head there only together with a surface that reads it. The promotion gate skips a champion head that is not in `HEADS`, so a trim does not block promotion.
diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md
index a1c0109293be..0fb3795477e4 100644
--- a/products/signals/dags/inbox_ranking/README.md
+++ b/products/signals/dags/inbox_ranking/README.md
@@ -149,7 +149,7 @@ s3://<bucket>/<prefix>/
└── inbox_ranking_served_scores/v1/dt=YYYY-MM-DD/ # the scoring sweep's scores of the reports born that day, per model key
```
-- **A model is a `(model_name, model_version, model_role)`.** `model_name` is the recipe: which features, which learner and which training settings: `report_embeddings`, per-head XGBoost, is the one trained here. `nn`, one ONNX network, is trained by the `signals_ml` location, and each `MODEL_FAMILIES` entry names its `learner` and the `location` that trains it. `model_version` is the partition day it was fit on, and `model_role` is the role it was scored or trained under (`candidate` on the training events; `served`, `daily_candidate` or `cross_family` on the online grade). Each family owns a prefix under the models path, so two families trained on the same day cannot collide. A candidate is graded only against the served model of its own name. The served pointer can name any family, so a rollback can cross families.
+- **A model is a `(model_name, model_version, model_role)`.** `model_name` is the recipe: which features, which learner and which training settings: `report_embeddings`, per-head XGBoost, is the one trained here. `nn`, one ONNX network, is trained by the `signals_ml` location, and each `MODEL_FAMILIES` entry names its `learner` and the `location` that trains it. `model_version` is the partition day it was fit on, and `model_role` is the role it was scored or trained under (`candidate` on the training events; `served`, `recent`, `pinned` or `cross_family` on the online grade, and `daily_candidate` on grades written before the `recent` role). Each family owns a prefix under the models path, so two families trained on the same day cannot collide. A candidate is graded only against the served model of its own name. The served pointer can name any family, so a rollback can cross families.
- **A recipe change gets a new `model_name`.** A candidate's `metadata.json` carries `recipe` and `recipe_hash`, a stable hash of the recipe (`training/recipe.py`): the learner, the feature set and its schema version, the project-vector kinds and their history settings, the booster params or network config, each head's cohort, label, horizon, label columns and status-label flag, `lookback_days`, `holdout_days`, the example grain and row budget, and `dataset_version`. Dates, `run_id`, row counts and metrics stay out, so a daily retrain keeps the hash. The candidate asset compares its hash with the newest version of the same name, at or before its partition, that has one, and on a mismatch it fails before it writes the partition and names the changed inputs. A re-run of a partition therefore compares with that partition's own saved recipe. The search covers every dataset version, so a `DATASET_VERSION` bump also needs a new name. A version without a hash is not compared. This includes an env change such as `INBOX_RANKING_TRAINING_LOOKBACK_DAYS`. To start a new recipe, add a `MODEL_FAMILIES` entry with a new name (`nn_v2` after `nn`) and remove the old entry: the old versions stay servable by key, and the promotion gate never compares two recipes. Two recipes can train side by side for a comparison, but each costs a manifest slot, so do not do it by default. `recipe_hash` is on `inbox_ranking_candidate_trained` and `inbox_ranking_promotion_decided` (with `incumbent_recipe_hash`), and `inbox_ranking_serving_manifest_published` carries `model_recipe_hashes` per model key.
- **A feature set is one feature universe.** `products/signals/backend/ranking/features.py` holds a `FeatureSet` per universe: its name, its schema version, its ordered feature names, the report-state columns it reads, `build_matrix`, and how many rows it wants per report and per head. `report_embeddings` is the one today. A candidate records the set it was fit on in its `metadata.json`, so the grader checks a model against its own set rather than one global contract, the examples asset writes one Parquet per set, and the unseen scorer builds one matrix per set and shares it across the families that read it. Adding a set means an entry in `FEATURE_SETS`; a model naming a set this build cannot produce is logged and left unscored.
- **Examples are one row per report at its birth, by default.** For partition `dt=D` the examples asset reads the report-state and labels snapshots `dt=D-lookback..D` and emits one row per report, on the snapshot of the day it was created, labeled from the snapshot `horizon_days` later (7 for open, 14 for action, 30 for fixed). Features are built by the feature set from that snapshot's state columns plus `age_hours`, the report's age at the snapshot. Labels are aligned to the state spine, so a report with no label event is a negative (all-zero labels), not absent. Label-only rows (no Postgres state) are skipped, as are state rows read long after their snapshot day (backfills carry current Postgres state; see `features_observed_at`) and, for the heads that read the status stream (`action` and `fixed`), rows whose status telemetry fails the dataset's `label_provenance_ok` check. Birth is the moment serving scores a report, and the label that comes out is a per-report probability, which is what the inbox ordering needs. A report born before the window, or one whose birth-day snapshot is missing from it, is no example; `reports_missing_birth_snapshot` on the asset counts the second case, which is a gap in the partitions.
@@ -186,7 +186,7 @@ Each entry names its `model_kind`, and the sweep loads it through that kind's ha
- `xgboost`, the default for a model without the field: one booster per head, `<head>.ubj`. The entry's `heads` list names the files.
- `onnx`: one `model.onnx` for every head. The entry names it in `model_file`, and `heads` still lists the heads it scores. The graph takes one float32 input, `report`, of shape (batch, feature-set width), and gives one output, `probabilities`, of shape (batch, heads). The column order of that output is `output_heads` in the model's `metadata.json`, and each head entry in `heads` there names `model.onnx` as its `file`. At load, the handler checks the input names and widths against what the model's feature set builds, and a mismatch skips the model like an unloadable booster. The session runs on CPU with a bounded thread count, and `onnxruntime` is imported only when a model loads. The project-vector inputs are not served yet, so the sweep skips an `nn` model, whose graph also takes `project` and `has_history`, until it feeds them.
-The manifest holds one entry per model with its `roles`: `served` for the model `served.json` names, `daily_candidate` for the candidate of the same `model_name` on this partition when it is a different, readable model, `pinned` for each model the `pin` override keeps (see [Overrides](#overrides)), and `cross_family` for every other family's `champion.json` that has a readable head. Nothing writes those files anymore, so they name the last version the retired automatic promotion chose. A pass scores all of them, so an online paired read and a later interleaving need no rescoring; the entry count is capped at the `ranking_score` artefact's own limit, dropping `cross_family` entries first and `pinned` entries next. The schema, the role names and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, and its validators run at write time, so the dag refuses a manifest the sweep could not act on. A model version is immutable, so a version already in the store is not copied again, and `manifest.json` is written last: a failed copy leaves the previous manifest and the previous models serving. When `served.json` is missing, or names a model that fails the pin check, the asset writes nothing and says why, and the previous manifest keeps serving.
+The manifest holds one entry per model with its `roles`: `served` for the model `served.json` names, `recent` for each of the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` (default 3) versions of the served `model_name`, `pinned` for each model the `pin` override keeps (see [Overrides](#overrides)), and `cross_family` for every other family's `champion.json` that has a readable head. Nothing writes those files anymore, so they name the last version the retired automatic promotion chose. The dag reads the `recent` versions newest first from `<model_name>/dt=D/metadata.json` in the 2N days up to the partition, where N is the setting. It skips a version with no readable head, the served version, and a version that fails the pin check, and it removes duplicate keys. So each version gets online scores for about N days, not for one. A pass scores all of them, so an online paired read and a later interleaving need no rescoring; the entry count is capped at the `ranking_score` artefact's own limit (5). The order is `served`, `recent`, `pinned`, `cross_family`, so the cap drops `cross_family` entries first and `pinned` entries next. Each `recent` version adds one model to every sweep pass, so a higher N costs sweep time and pod memory. The schema, the role names and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, and its validators run at write time, so the dag refuses a manifest the sweep could not act on. A model version is immutable, so a version already in the store is not copied again, and `manifest.json` is written last: a failed copy leaves the previous manifest and the previous models serving. When `served.json` is missing, or names a model that fails the pin check, the asset writes nothing and says why, and the previous manifest keeps serving.
A region that does not train serves the same models through a mirror. When `INBOX_RANKING_SERVING_MIRROR_BUCKET` is set, the asset repeats the publish into that bucket after the primary publish succeeds, in the same order and with the same skip of versions already there. A mirror failure does not fail the asset: it logs, and the `inbox_ranking_serving_manifest_published` event carries `mirror_published=false`. The property is absent when no mirror is set.
@@ -223,7 +223,7 @@ During a `served` override, the override model carries the roles `served` and `s
Examples, each with an `expires_at` at most 14 days ahead:
-- Serve today's candidate now: set `served` to the `daily_candidate` key of the current manifest.
+- Serve a recent version now: set `served` to a `recent` key of the current manifest.
- Roll back to an older model without a Dagster launch: add its key to `pin`, wait for the next manifest run (or rematerialize `inbox_ranking_serving_manifest`), then set `served` to the same key. For a lasting rollback, run `inbox_ranking_promote_job` with the key.
Both regions read the same flag, so one override applies everywhere, if the region's `posthoganalytics` client evaluates flags in the internal project. A region that cannot read the flag runs with no override. Nothing deletes old model prefixes in the dataset bucket or old `serving/models/<key>/` copies. If the bucket has a lifecycle rule, a rollback through `pin` works only while the model files are still there.
@@ -266,7 +266,7 @@ The model is read in three layers: **training** (train metrics), **test** (the r
The holdout grades the recipe, not the model that ships: the shipped booster is refit on train plus holdout, and every holdout row comes from the same snapshots the trainer saw.
Two assets add the missing number, from the scores the sweep actually wrote for reports no model trained on.
-- **`inbox_ranking_served_scores` (dt=D)** (`training/served.py`) takes the dt=D state rows of the reports **created on D**, the pool the rows tag `pool=newborn`. A newborn has no scoring moment before D, and the dt=D examples stop at D minus the head's horizon (7 days at the shortest), so no example can cover it. It reads this deployment's `inbox_ranking_report_scored` events for that pool inside D and keeps the earliest `scored` event per (report, model key), so every model the manifest named is graded: the served champion, the served family's `daily_candidate`, and the other families' `cross_family` champions. `model_role` comes from the event's roles: `served` when they hold it, else the first role, else `candidate`. The sweep scores a candidate only after a manifest names it, so a candidate's online cohort starts the day after it was trained. Every head a model fit is scored, readable or not, and each row carries `head_readable` and the model's own `classification_threshold`. One Parquet row per (report, model key, head) carries the score, `age_hours`, `label_at_scoring` (the head outcome that had already happened when the report was scored, which for a newborn is one that landed on its own birth day, so the grade keeps it; a status-label head's rows are still dropped on it, because no scores column carries the scoring-day `label_provenance_ok` that the example builder checks), `model_name`, `model_role`, and `pool`. `served_pool_coverage` and `pool_coverage_by_model` say how much of the pool each model covered. A re-run that reads no event is refused when the partition already holds rows, and so is one that reads fewer families than the object already holds. Rows of a retired family (`RETIRED_MODEL_NAMES`) are dropped on read, and the rewrite guard does not require them.
+- **`inbox_ranking_served_scores` (dt=D)** (`training/served.py`) takes the dt=D state rows of the reports **created on D**, the pool the rows tag `pool=newborn`. A newborn has no scoring moment before D, and the dt=D examples stop at D minus the head's horizon (7 days at the shortest), so no example can cover it. It reads this deployment's `inbox_ranking_report_scored` events for that pool inside D and keeps the earliest `scored` event per (report, model key), so every model the manifest named is graded: the served model, the served family's `recent` versions, the `pinned` models, and the other families' `cross_family` champions. `model_role` comes from the event's roles: `served` when they hold it, else the first role, else `candidate`. The sweep scores a candidate only after a manifest names it, so a candidate's online cohort starts the day after it was trained, and it stays a `recent` entry until N newer versions replace it. Every head a model fit is scored, readable or not, and each row carries `head_readable` and the model's own `classification_threshold`. One Parquet row per (report, model key, head) carries the score, `age_hours`, `label_at_scoring` (the head outcome that had already happened when the report was scored, which for a newborn is one that landed on its own birth day, so the grade keeps it; a status-label head's rows are still dropped on it, because no scores column carries the scoring-day `label_provenance_ok` that the example builder checks), `model_name`, `model_role`, and `pool`. `served_pool_coverage` and `pool_coverage_by_model` say how much of the pool each model covered. A re-run that reads no event is refused when the partition already holds rows, and so is one that reads fewer families than the object already holds. Rows of a retired family (`RETIRED_MODEL_NAMES`) are dropped on read, and the rewrite guard does not require them.
- **`inbox_ranking_unseen_graded` (dt=D)** reads saved scores from D back through each head's horizon and evaluates them against the dt=D labels. Daily head evaluations include both unbaked and baked metrics, marked by `is_mature`; existing head, report and calibration grade events remain baked-only at `D - horizon_days`. See [baked and unbaked unseen metrics](../../../../docs/internal/inbox-ranking-training.md#baked-and-unbaked-unseen-metrics) for the event contract and chart revision rules. It keeps the rows whose cohort holds at D, reads the outcome with `Head.label`, and reports the AUC per model and head next to two comparison lines on the same rows: `recency_auc`, the AUC of ranking newest first by `report_created_at`, and `null_auc` / `null_auc_std`, the mean and spread of the AUC over seeded permutations of the scores. The mean is 0.5 by construction; the spread is the noise band a per-day AUC has to clear before a gap between two families means anything. It also reads the calibration of those scores, which no AUC can see: the in-cohort rows are cut into score deciles by rank (`training/calibration.py`, shared with the trainer so both sides cut them the same way), and each decile's mean score is compared against the rate the outcome actually happened at. A run of equal scores stays in one decile, so a head reports fewer deciles rather than a gap that only reflects the row order. Read each grade with the model's `pool_coverage_by_model`: a key that joined the manifest part of the way through D grades only part of the pool.
At the full horizon, the cohort, label and provenance rules are the trainer's own, so the baked online AUC is directly comparable to the holdout AUC of the same head and model version. The gap between the two series is the overfitting read the promotion gate cannot see.
@@ -371,15 +371,16 @@ dagster job launch -w .dagster_home/workspace.yaml --location posthog.dags.locat
## Configuration
-| Setting | Default | Meaning |
-| -------------------------------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `INBOX_RANKING_DATASET_S3_BUCKET` | unset | Destination bucket. Unset on Cloud makes every asset log and skip, so the dag can deploy before the bucket exists. Unset elsewhere falls back to the deployment's object-storage service (SeaweedFS in dev and CI). |
-| `INBOX_RANKING_DATASET_S3_PREFIX` | `inbox_ranking` | Key prefix under the bucket. |
-| `INBOX_RANKING_TRAINING_LOOKBACK_DAYS` | `60` | How many daily snapshots back the training examples reach. |
-| `INBOX_RANKING_TRAINING_HOLDOUT_DAYS` | `7` | Trailing days of reports that grade a candidate. |
-| `INBOX_RANKING_SERVING_MIRROR_BUCKET` | unset | A second deployment's object-storage bucket that receives a copy of every serving publish, through ambient AWS config. Unset means no mirror. |
-| `INBOX_RANKING_SERVING_MIRROR_REGION` | unset | AWS region of the mirror bucket. |
-| `INBOX_RANKING_SCORING_BATCH_SIZE` | `500` | Report ids per ClickHouse vector read in the scorer. A larger call is paged at this size. |
+| Setting | Default | Meaning |
+| --------------------------------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `INBOX_RANKING_DATASET_S3_BUCKET` | unset | Destination bucket. Unset on Cloud makes every asset log and skip, so the dag can deploy before the bucket exists. Unset elsewhere falls back to the deployment's object-storage service (SeaweedFS in dev and CI). |
+| `INBOX_RANKING_DATASET_S3_PREFIX` | `inbox_ranking` | Key prefix under the bucket. |
+| `INBOX_RANKING_TRAINING_LOOKBACK_DAYS` | `60` | How many daily snapshots back the training examples reach. |
+| `INBOX_RANKING_TRAINING_HOLDOUT_DAYS` | `7` | Trailing days of reports that grade a candidate. |
+| `INBOX_RANKING_SERVING_MIRROR_BUCKET` | unset | A second deployment's object-storage bucket that receives a copy of every serving publish, through ambient AWS config. Unset means no mirror. |
+| `INBOX_RANKING_SERVING_MIRROR_REGION` | unset | AWS region of the mirror bucket. |
+| `INBOX_RANKING_SERVING_RECENT_VERSIONS` | `3` | How many of the served family's newest versions the manifest carries under the `recent` role, read from the last 2N days. Each one adds one model to every sweep pass. |
+| `INBOX_RANKING_SCORING_BATCH_SIZE` | `500` | Report ids per ClickHouse vector read in the scorer. A larger call is paged at this size. |
Writes use boto3: ambient AWS config (the node role) when the dedicated bucket is set, the `OBJECT_STORAGE_*` endpoint and credentials otherwise. Readers (project-level warehouse tables, model training) use a separate read-only credential provisioned with the bucket.
diff --git a/products/signals/dags/inbox_ranking/tests/test_nn.py b/products/signals/dags/inbox_ranking/tests/test_nn.py
index c7ec1b5bed2e..7fc04465565e 100644
--- a/products/signals/dags/inbox_ranking/tests/test_nn.py
+++ b/products/signals/dags/inbox_ranking/tests/test_nn.py
@@ -21,9 +21,9 @@ from products.signals.backend.ranking.model_store import ONNX_OUTPUT_HEADS_KEY
from products.signals.backend.ranking.overrides import NO_OVERRIDES
from products.signals.backend.ranking.serving_manifest import (
CROSS_FAMILY_ROLE,
- DAILY_CANDIDATE_ROLE,
ONNX_MODEL_FILE,
ONNX_MODEL_KIND,
+ RECENT_ROLE,
SERVED_ROLE,
)
from products.signals.dags.inbox_ranking.common import DATASET_VERSION, partition_object_key
@@ -187,10 +187,9 @@ def test_the_manifest_carries_an_nn_champion_as_an_onnx_entry():
[
FamilyModels(
name=REPORT_EMBEDDINGS_MODEL_NAME,
- candidate=None,
champion=_xgboost_metadata(REPORT_EMBEDDINGS_MODEL_NAME, "2026-08-15"),
),
- FamilyModels(name=NN_MODEL_NAME, candidate=None, champion=_nn_metadata("2026-08-16")),
+ FamilyModels(name=NN_MODEL_NAME, champion=_nn_metadata("2026-08-16")),
],
served=_xgboost_metadata(REPORT_EMBEDDINGS_MODEL_NAME, "2026-08-15"),
prefix=PREFIX,
@@ -210,24 +209,21 @@ def test_the_manifest_serves_an_nn_model_with_the_xgboost_champion_as_a_challeng
[
FamilyModels(
name=REPORT_EMBEDDINGS_MODEL_NAME,
- candidate=None,
champion=_xgboost_metadata(REPORT_EMBEDDINGS_MODEL_NAME, "2026-08-15"),
),
- FamilyModels(
- name=NN_MODEL_NAME,
- candidate=_nn_metadata(PARTITION, heads=NN_HEAD_NAMES),
- champion=_nn_metadata("2026-08-16", heads=NN_HEAD_NAMES),
- ),
+ FamilyModels(name=NN_MODEL_NAME, champion=_nn_metadata("2026-08-16", heads=NN_HEAD_NAMES)),
],
served=_nn_metadata("2026-08-16", heads=NN_HEAD_NAMES),
prefix=PREFIX,
now=NOW,
+ recent=[_nn_metadata(PARTITION, heads=NN_HEAD_NAMES)],
+ recent_versions=3,
)
assert decision.manifest is not None
assert [(entry.model_name, entry.model_version, entry.roles) for entry in decision.manifest.models] == [
(NN_MODEL_NAME, "2026-08-16", [SERVED_ROLE]),
- (NN_MODEL_NAME, PARTITION, [DAILY_CANDIDATE_ROLE]),
+ (NN_MODEL_NAME, PARTITION, [RECENT_ROLE]),
(REPORT_EMBEDDINGS_MODEL_NAME, "2026-08-15", [CROSS_FAMILY_ROLE]),
]
served = decision.manifest.served
diff --git a/products/signals/dags/inbox_ranking/tests/test_training.py b/products/signals/dags/inbox_ranking/tests/test_training.py
index 435b7c6eb045..8f1c9c89641a 100644
--- a/products/signals/dags/inbox_ranking/tests/test_training.py
+++ b/products/signals/dags/inbox_ranking/tests/test_training.py
@@ -38,11 +38,11 @@ from products.signals.backend.ranking.model_contract import classification_thres
from products.signals.backend.ranking.overrides import RankingOverrides
from products.signals.backend.ranking.serving_manifest import (
CROSS_FAMILY_ROLE,
- DAILY_CANDIDATE_ROLE,
DEFAULT_MODEL_KIND,
ONNX_MODEL_FILE,
ONNX_MODEL_KIND,
PINNED_ROLE,
+ RECENT_ROLE,
SERVED_ROLE,
ServingManifest,
ServingManifestEntry,
@@ -2323,27 +2323,25 @@ def test_a_manifest_entry_the_sweep_could_not_load_is_refused(overrides, message
)
-def _compose(families, served):
+def _compose(families, served, recent=(), recent_versions=3, pinned=()):
return compose_manifest(
families,
served=served,
prefix="inbox_ranking",
now=datetime.datetime(2026, 8, 19, 6, tzinfo=datetime.UTC),
+ recent=recent,
+ recent_versions=recent_versions,
+ pinned=pinned,
)
def test_no_served_pointer_publishes_nothing():
- # Composing a manifest off a champion or the day's candidate instead would move the served
+ # Composing a manifest off a champion or a recent version instead would move the served
# model without a person choosing it.
decision = _compose(
- [
- FamilyModels(
- name=EMBEDDINGS_MODEL_NAME,
- candidate=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-19"),
- champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
- )
- ],
+ [FamilyModels(name=EMBEDDINGS_MODEL_NAME, champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"))],
served=None,
+ recent=[_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-19")],
)
assert decision.manifest is None
assert "no served pointer" in decision.reason
@@ -2355,12 +2353,9 @@ def test_the_served_pointer_wins_over_the_champion_of_its_own_family():
[
FamilyModels(
name=EMBEDDINGS_MODEL_NAME,
- candidate=None,
champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
),
- FamilyModels(
- name=STATE_MODEL_NAME, candidate=None, champion=_serving_metadata(STATE_MODEL_NAME, "2026-08-10")
- ),
+ FamilyModels(name=STATE_MODEL_NAME, champion=_serving_metadata(STATE_MODEL_NAME, "2026-08-10")),
],
served=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-01"),
)
@@ -2372,32 +2367,39 @@ def test_the_served_pointer_wins_over_the_champion_of_its_own_family():
@pytest.mark.parametrize(
- "candidate_version,candidate_readable,expected_roles",
+ "recent,recent_versions,expected_recent",
[
- # A distinct, readable candidate is the paired read the sweep exists for.
- ("2026-08-19", True, {SERVED_ROLE: "2026-08-15", DAILY_CANDIDATE_ROLE: "2026-08-19"}),
- # The candidate at the champion's own version is the champion: a second entry for it would
- # spend a result slot on a duplicate score.
- ("2026-08-15", True, {SERVED_ROLE: "2026-08-15"}),
- # An unreadable candidate has no holdout number to pair the champion against.
- ("2026-08-19", False, {SERVED_ROLE: "2026-08-15"}),
+ # The newest versions are the paired reads the sweep exists for, newest first.
+ (
+ [("2026-08-19", True), ("2026-08-18", True), ("2026-08-17", True), ("2026-08-16", True)],
+ 3,
+ ["2026-08-19", "2026-08-18", "2026-08-17"],
+ ),
+ # The served version is already an entry, so it gains no second one and frees its slot.
+ (
+ [("2026-08-19", True), ("2026-08-15", True), ("2026-08-14", True), ("2026-08-13", True)],
+ 3,
+ ["2026-08-19", "2026-08-14", "2026-08-13"],
+ ),
+ # An unreadable version has no holdout number to pair the served model against.
+ ([("2026-08-19", False), ("2026-08-18", True)], 3, ["2026-08-18"]),
+ # The same version read twice is one entry.
+ ([("2026-08-19", True), ("2026-08-19", True), ("2026-08-18", True)], 3, ["2026-08-19", "2026-08-18"]),
+ ([("2026-08-19", True)], 0, []),
],
)
-def test_the_served_family_candidate_is_paired_only_when_it_is_a_second_model(
- candidate_version, candidate_readable, expected_roles
-):
+def test_the_served_family_keeps_its_newest_distinct_readable_versions(recent, recent_versions, expected_recent):
decision = _compose(
- [
- FamilyModels(
- name=EMBEDDINGS_MODEL_NAME,
- candidate=_serving_metadata(EMBEDDINGS_MODEL_NAME, candidate_version, readable=candidate_readable),
- champion=None,
- )
- ],
+ [],
served=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
+ recent=[_serving_metadata(EMBEDDINGS_MODEL_NAME, version, readable=readable) for version, readable in recent],
+ recent_versions=recent_versions,
)
assert decision.manifest is not None
- assert {entry.roles[0]: entry.model_version for entry in decision.manifest.models} == expected_roles
+ assert [(entry.roles, entry.model_version) for entry in decision.manifest.models] == [
+ ([SERVED_ROLE], "2026-08-15"),
+ *(([RECENT_ROLE], version) for version in expected_recent),
+ ]
def test_a_family_with_no_readable_head_is_left_out():
@@ -2406,15 +2408,11 @@ def test_a_family_with_no_readable_head_is_left_out():
[
FamilyModels(
name=EMBEDDINGS_MODEL_NAME,
- candidate=None,
champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
),
- FamilyModels(
- name=STATE_MODEL_NAME, candidate=None, champion=_serving_metadata(STATE_MODEL_NAME, "2026-08-10")
- ),
+ FamilyModels(name=STATE_MODEL_NAME, champion=_serving_metadata(STATE_MODEL_NAME, "2026-08-10")),
FamilyModels(
name=NN_MODEL_NAME,
- candidate=None,
champion=_serving_metadata(NN_MODEL_NAME, "2026-08-12", readable=False),
),
],
@@ -2426,57 +2424,50 @@ def test_a_family_with_no_readable_head_is_left_out():
def test_the_cap_drops_cross_family_entries_before_the_paired_read():
# `RankingScore` refuses a row over the cap, so the manifest is capped at the same number. The
- # served entry and its paired candidate are the reads the sweep exists for.
+ # served entry and its recent version are the reads the sweep exists for.
families = [
- FamilyModels(
- name=EMBEDDINGS_MODEL_NAME,
- candidate=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-19"),
- champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
- ),
+ FamilyModels(name=EMBEDDINGS_MODEL_NAME, champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15")),
*(
- FamilyModels(
- name=f"family_{index}", candidate=None, champion=_serving_metadata(f"family_{index}", "2026-08-10")
- )
+ FamilyModels(name=f"family_{index}", champion=_serving_metadata(f"family_{index}", "2026-08-10"))
for index in range(5)
),
]
- decision = _compose(families, served=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"))
+ decision = _compose(
+ families,
+ served=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
+ recent=[_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-19")],
+ )
assert decision.manifest is not None
assert [entry.roles[0] for entry in decision.manifest.models] == [
SERVED_ROLE,
- DAILY_CANDIDATE_ROLE,
+ RECENT_ROLE,
*[CROSS_FAMILY_ROLE] * (MAX_RANKING_MODEL_RESULTS - 2),
]
-def test_a_pinned_model_is_kept_and_capped_before_the_paired_read():
+def test_a_pinned_model_is_kept_and_capped_after_the_recent_versions():
families = [
- FamilyModels(
- name=EMBEDDINGS_MODEL_NAME,
- candidate=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-19"),
- champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
- ),
- FamilyModels(name=STATE_MODEL_NAME, candidate=None, champion=_serving_metadata(STATE_MODEL_NAME, "2026-08-10")),
+ FamilyModels(name=EMBEDDINGS_MODEL_NAME, champion=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15")),
+ FamilyModels(name=STATE_MODEL_NAME, champion=_serving_metadata(STATE_MODEL_NAME, "2026-08-10")),
]
pinned = [_serving_metadata(EMBEDDINGS_MODEL_NAME, f"2026-08-0{day}", readable=False) for day in range(1, 5)]
# A pin of a model the manifest already names gains no second entry.
- pinned.append(_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-19"))
+ pinned.insert(0, _serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-19"))
- decision = compose_manifest(
+ decision = _compose(
families,
served=_serving_metadata(EMBEDDINGS_MODEL_NAME, "2026-08-15"),
- prefix="inbox_ranking",
- now=datetime.datetime(2026, 8, 19, 6, tzinfo=datetime.UTC),
+ recent=[_serving_metadata(EMBEDDINGS_MODEL_NAME, f"2026-08-{day}") for day in (19, 18, 17)],
pinned=pinned,
)
assert decision.manifest is not None
assert [(entry.roles[0], entry.model_version) for entry in decision.manifest.models] == [
(SERVED_ROLE, "2026-08-15"),
- (DAILY_CANDIDATE_ROLE, "2026-08-19"),
+ (RECENT_ROLE, "2026-08-19"),
+ (RECENT_ROLE, "2026-08-18"),
+ (RECENT_ROLE, "2026-08-17"),
(PINNED_ROLE, "2026-08-01"),
- (PINNED_ROLE, "2026-08-02"),
- (PINNED_ROLE, "2026-08-03"),
]
@@ -2670,6 +2661,66 @@ def test_project_vectors_are_published_before_the_manifest_that_names_them(
assert mirror.objects == {}
+def test_recent_versions_come_from_the_window_and_skip_a_version_missing_its_booster(monkeypatch):
+ # A booster deleted by a re-run would fail the whole publish, and a version older than the
+ # window would keep a stale model in every pass.
+ prefix = settings.INBOX_RANKING_DATASET_S3_PREFIX
+ client = _serving_dataset_s3(prefix, "2026-08-19")
+ for version, booster in (("2026-08-18", True), ("2026-08-17", False), ("2026-08-13", True)):
+ folder = model_object_key(prefix, EMBEDDINGS_MODEL_NAME, version, "")
+ client.objects[folder + METADATA_FILE] = json.dumps(
+ _serving_metadata(EMBEDDINGS_MODEL_NAME, version, heads=("open",), **_FEATURE_FIELDS)
+ ).encode()
+ if booster:
+ client.objects[folder + "open.ubj"] = b"booster"
+ monkeypatch.setattr(settings, "INBOX_RANKING_SERVING_RECENT_VERSIONS", 3)
+ store = _AppObjectStore()
+
+ _run_serving_manifest(monkeypatch, store, dataset_client=client)
+
+ manifest = ServingManifest.model_validate_json(store.objects[serving_manifest_key(prefix)])
+ assert [(entry.model_version, entry.roles) for entry in manifest.models] == [
+ ("2026-08-15", [SERVED_ROLE]),
+ ("2026-08-19", [RECENT_ROLE]),
+ ("2026-08-18", [RECENT_ROLE]),
+ ]
+
+
+class _FailingVersionS3(_ModelStoreS3):
+ def __init__(self, objects: dict[str, bytes], *, failing_prefix: str, code: str):
+ super().__init__(objects)
+ self.failing_prefix = failing_prefix
+ self.code = code
+
+ def get_object(self, *, Bucket, Key):
+ if Key.startswith(self.failing_prefix):
+ raise ClientError({"Error": {"Code": self.code}}, "GetObject")
+ return super().get_object(Bucket=Bucket, Key=Key)
+
+
[email protected]("code,published", [("AccessDenied", True), ("SlowDown", False), ("InternalError", False)])
+def test_a_recent_version_read_error_skips_only_a_missing_key(monkeypatch, code, published):
+ # A throttled read that skipped the version would publish a manifest without it and bypass
+ # the asset retry, so only the 403 S3 gives for a missing key may drop a recent version.
+ prefix = settings.INBOX_RANKING_DATASET_S3_PREFIX
+ base = _serving_dataset_s3(prefix, "2026-08-19")
+ client = _FailingVersionS3(
+ base.objects,
+ failing_prefix=model_object_key(prefix, EMBEDDINGS_MODEL_NAME, "2026-08-19", ""),
+ code=code,
+ )
+ store = _AppObjectStore()
+
+ if published:
+ _run_serving_manifest(monkeypatch, store, dataset_client=client)
+ manifest = ServingManifest.model_validate_json(store.objects[serving_manifest_key(prefix)])
+ assert [entry.model_version for entry in manifest.models] == ["2026-08-15"]
+ else:
+ with pytest.raises(ClientError):
+ _run_serving_manifest(monkeypatch, store, dataset_client=client)
+ assert serving_manifest_key(prefix) not in store.objects
+
+
def test_a_version_already_in_the_store_is_not_copied_again(monkeypatch):
# A model version is immutable, so re-copying it every day would move the boosters of every
# entry, one of which is 1536 columns wide, for no change.
@@ -2682,7 +2733,7 @@ def test_a_version_already_in_the_store_is_not_copied_again(monkeypatch):
assert store.objects[f"{champion_prefix}/{METADATA_FILE}"] == b"already here"
assert f"{champion_prefix}/open.ubj" not in store.objects
- # The day's candidate was not there, so it was copied and the manifest still names both.
+ # The recent version was not there, so it was copied and the manifest still names both.
manifest = ServingManifest.model_validate_json(store.objects[serving_manifest_key(prefix)])
assert len(manifest.models) == 2
@@ -2803,7 +2854,7 @@ _CANDIDATE_EVENT = {
"model_key": model_key(REPORT_EMBEDDINGS_MODEL_NAME, CANDIDATE_VERSION),
"model_name": REPORT_EMBEDDINGS_MODEL_NAME,
"model_version": CANDIDATE_VERSION,
- "roles": [DAILY_CANDIDATE_ROLE],
+ "roles": [RECENT_ROLE],
}
_NN_EVENT = {"model_name": NN_MODEL_NAME, "model_version": V1, "roles": [CROSS_FAMILY_ROLE]}
_SERVED_POOL = ["tie", "low", "legacy_low", "legacy_high", "challenger_only", "late"]
@@ -2870,14 +2921,14 @@ def test_served_score_rows_keep_the_earliest_score_of_each_newborn_per_model_key
zip(others["report_id"], others["model_name"], others["model_version"], others["model_role"], others["score"])
) == [
("challenger_only", NN_MODEL_NAME, V1, CROSS_FAMILY_ROLE, 0.5),
- ("tie", REPORT_EMBEDDINGS_MODEL_NAME, CANDIDATE_VERSION, DAILY_CANDIDATE_ROLE, 0.7),
+ ("tie", REPORT_EMBEDDINGS_MODEL_NAME, CANDIDATE_VERSION, RECENT_ROLE, 0.7),
]
metadata = served_metadata(_state(_SERVED_POOL), rows)
assert metadata["served_pool_coverage"] == dagster.MetadataValue.float(4 / 6)
assert metadata["pool_coverage_by_model"].value == {
f"{REPORT_EMBEDDINGS_MODEL_NAME}@{V1}/{SERVED_ROLE}": 2 / 6,
f"{REPORT_EMBEDDINGS_MODEL_NAME}@{V2}/{SERVED_ROLE}": 3 / 6,
- f"{REPORT_EMBEDDINGS_MODEL_NAME}@{CANDIDATE_VERSION}/{DAILY_CANDIDATE_ROLE}": 1 / 6,
+ f"{REPORT_EMBEDDINGS_MODEL_NAME}@{CANDIDATE_VERSION}/{RECENT_ROLE}": 1 / 6,
f"{NN_MODEL_NAME}@{V1}/{CROSS_FAMILY_ROLE}": 1 / 6,
}
@@ -2885,8 +2936,8 @@ def test_served_score_rows_keep_the_earliest_score_of_each_newborn_per_model_key
@pytest.mark.parametrize(
"roles,expected",
[
- ([SERVED_ROLE, DAILY_CANDIDATE_ROLE], SERVED_ROLE),
- ([DAILY_CANDIDATE_ROLE, CROSS_FAMILY_ROLE], DAILY_CANDIDATE_ROLE),
+ ([SERVED_ROLE, RECENT_ROLE], SERVED_ROLE),
+ ([RECENT_ROLE, CROSS_FAMILY_ROLE], RECENT_ROLE),
([], CANDIDATE_ROLE),
(None, CANDIDATE_ROLE),
],
@@ -2929,7 +2980,7 @@ def test_served_scores_grade_per_version_at_the_threshold_that_was_served(monkey
assert set(graded_by_key) == {
(REPORT_EMBEDDINGS_MODEL_NAME, V1, SERVED_ROLE),
(REPORT_EMBEDDINGS_MODEL_NAME, V2, SERVED_ROLE),
- (REPORT_EMBEDDINGS_MODEL_NAME, CANDIDATE_VERSION, DAILY_CANDIDATE_ROLE),
+ (REPORT_EMBEDDINGS_MODEL_NAME, CANDIDATE_VERSION, RECENT_ROLE),
(NN_MODEL_NAME, V1, CROSS_FAMILY_ROLE),
}
graded = {version: row for (_, version, role), row in graded_by_key.items() if role == SERVED_ROLE}
diff --git a/products/signals/dags/inbox_ranking/training/dag.py b/products/signals/dags/inbox_ranking/training/dag.py
index 7b6b9818af4e..fe73a86f06ec 100644
--- a/products/signals/dags/inbox_ranking/training/dag.py
+++ b/products/signals/dags/inbox_ranking/training/dag.py
@@ -1000,15 +1000,26 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
families = [
FamilyModels(
name=family.name,
- candidate=_read_json_if_exists(
- client, bucket, model_object_key(prefix, family.name, partition_key, METADATA_FILE)
- ),
champion=_read_json_if_exists(client, bucket, champion_object_key(prefix, family.name)),
)
for family in MODEL_FAMILIES
]
+ recent_versions = settings.INBOX_RANKING_SERVING_RECENT_VERSIONS
+ recent = (
+ _recent_models(context, client, bucket, prefix, served["model_name"], partition_key, recent_versions)
+ if served is not None
+ else []
+ )
decision = (
- compose_manifest(families, served=served, prefix=prefix, now=datetime.datetime.now(datetime.UTC), pinned=pinned)
+ compose_manifest(
+ families,
+ served=served,
+ prefix=prefix,
+ now=datetime.datetime.now(datetime.UTC),
+ recent=recent,
+ recent_versions=recent_versions,
+ pinned=pinned,
+ )
if served_reason is None
else ManifestDecision(manifest=None, reason=served_reason)
)
@@ -1068,8 +1079,9 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
model_key(metadata["model_name"], metadata["model_version"]): metadata.get(RECIPE_HASH_KEY)
for metadata in (
*([served] if served is not None else []),
+ *recent,
*pinned,
- *(record for family in families for record in (family.candidate, family.champion) if record),
+ *(family.champion for family in families if family.champion),
)
},
)
@@ -1095,6 +1107,9 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
}
+_ACCESS_DENIED_CODES = ("403", "AccessDenied")
+
+
def _object_exists(client, bucket: str, key: str) -> bool:
try:
client.head_object(Bucket=bucket, Key=key)
@@ -1148,6 +1163,36 @@ def _pinned_models(
return pinned, dropped
+def _recent_models(
+ context: RunContext, client, bucket: str, prefix: str, model_name: str, partition_key: str, versions: int
+) -> list[dict[str, Any]]:
+ """The metadata of each `model_name` version fit in the 2 * `versions` days up to the partition,
+ newest first, that this build can serve.
+
+ A family does not fit a model every day, so the window is wider than `versions`. A version
+ passes the same check as a pin, because one missing file fails the whole publish.
+ """
+ end = datetime.date.fromisoformat(partition_key)
+ recent: list[dict[str, Any]] = []
+ for offset in range(2 * versions):
+ key = model_key(model_name, (end - datetime.timedelta(days=offset)).isoformat())
+ try:
+ metadata, reason = _pin_metadata(client, bucket, prefix, key)
+ except ValueError as error:
+ metadata, reason = None, f"could not read the model: {error!r}"
+ except ClientError as error:
+ # S3 answers 403 for a missing key when the role cannot list the prefix. Any other error,
+ # such as throttling, fails the publish so the retry policy runs instead of a smaller manifest.
+ if error.response.get("Error", {}).get("Code") not in _ACCESS_DENIED_CODES:
+ raise
+ metadata, reason = None, f"could not read the model: {error!r}"
+ if metadata is None:
+ context.log.info(f"recent model {key} skipped: {reason}")
+ continue
+ recent.append(metadata)
+ return recent
+
+
def _pin_metadata(client, bucket: str, prefix: str, key: str) -> tuple[dict[str, Any] | None, str | None]:
"""The metadata of one pinned model, or None and the reason it cannot be served."""
model_name, model_version = key.split("@", 1)
diff --git a/products/signals/dags/inbox_ranking/training/served.py b/products/signals/dags/inbox_ranking/training/served.py
index 672170d45f15..5a1f8ec12ce5 100644
--- a/products/signals/dags/inbox_ranking/training/served.py
+++ b/products/signals/dags/inbox_ranking/training/served.py
@@ -1,7 +1,7 @@
"""The scoring sweep's own birth-day scores, graded per model key.
The sweep scores each report with every model the serving manifest names: the served model, the
-daily candidate of its `model_name`, and the other families' champions. This module reads the sweep's
+recent versions of its `model_name`, and the other families' champions. This module reads the sweep's
`inbox_ranking_report_scored` events for D's newborn pool and writes them in `SCORES_SCHEMA`, so
`inbox_ranking_unseen_graded` grades every model key on live scores through one path.
diff --git a/products/signals/dags/inbox_ranking/training/serving.py b/products/signals/dags/inbox_ranking/training/serving.py
index d3d30218ae51..92a74e2c0216 100644
--- a/products/signals/dags/inbox_ranking/training/serving.py
+++ b/products/signals/dags/inbox_ranking/training/serving.py
@@ -1,13 +1,14 @@
"""Composing the serving manifest from the day's models.
-The training dag is the only place that knows every family's candidate and champion at once, so it
+The training dag is the only place that knows every family's versions and champion at once, so it
picks the set the scoring sweep runs. This module is that choice as pure functions over the
`metadata.json` records; `training/dag.py` owns the S3 reads, the publishing and the telemetry.
The served entry is the model the `served.json` pointer names, so the served model changes only
when a person runs `inbox_ranking_promote_job`. The other entries are there so one pass produces several scores: an online paired read
-and a later interleaving then need no rescoring of the same reports. A pinned entry is there so an
-owner's `served` override can name it.
+and a later interleaving then need no rescoring of the same reports. The recent entries keep a
+version in the manifest for several days, so its online grade reads more than one day's cohort. A
+pinned entry is there so an owner's `served` override can name it.
"""
import datetime
@@ -21,11 +22,11 @@ from products.signals.backend.ranking.model_contract import readable_head_names,
from products.signals.backend.ranking.model_store import ONNX_PROJECT_VECTOR_KINDS_KEY
from products.signals.backend.ranking.serving_manifest import (
CROSS_FAMILY_ROLE,
- DAILY_CANDIDATE_ROLE,
DEFAULT_MODEL_KIND,
ONNX_MODEL_FILE,
ONNX_MODEL_KIND,
PINNED_ROLE,
+ RECENT_ROLE,
SERVED_ROLE,
ServingManifest,
ServingManifestEntry,
@@ -37,13 +38,11 @@ from products.signals.dags.inbox_ranking.training.heads import HEADS_BY_NAME
@frozen
class FamilyModels:
- """One family's two records for the partition: the candidate fit that day, and the family's
- `champion.json`. Either can be absent. Nothing writes `champion.json` anymore, so a champion is
- the last version the retired automatic promotion chose, and it serves only as a cross-family
- read."""
+ """One family's `champion.json`, or None when it has none. Nothing writes `champion.json`
+ anymore, so a champion is the last version the retired automatic promotion chose, and it serves
+ only as a cross-family read."""
name: str
- candidate: Mapping[str, Any] | None
champion: Mapping[str, Any] | None
@@ -92,11 +91,14 @@ def compose_manifest(
served: Mapping[str, Any] | None,
prefix: str,
now: datetime.datetime,
+ recent: Sequence[Mapping[str, Any]] = (),
+ recent_versions: int = 0,
pinned: Sequence[Mapping[str, Any]] = (),
) -> ManifestDecision:
- """`served` holds the metadata of the model the `served.json` pointer names, and `pinned` the
- metadata of the models the `pin` override keeps. The caller checks that their files exist,
- because a manifest that names a missing file is never published."""
+ """`served` holds the metadata of the model the `served.json` pointer names, `recent` the
+ metadata of the served family's versions newest first, and `pinned` the metadata of the models
+ the `pin` override keeps. At most `recent_versions` of `recent` get the recent role. The caller
+ checks that their files exist, because a manifest that names a missing file is never published."""
if served is None:
return ManifestDecision(manifest=None, reason="no served pointer")
served_family = served["model_name"]
@@ -112,23 +114,23 @@ def compose_manifest(
)
entries = [served_entry]
- served_models = next((family for family in families if family.name == served_family), None)
- candidate = served_models.candidate if served_models is not None else None
- # A candidate at the champion's own version is the champion, so scoring it twice would only
- # spend the artefact's result budget on a duplicate.
- if (
- candidate is not None
- and candidate["model_version"] != served_entry.model_version
- and readable_head_names(candidate)
- ):
- daily = _entry(
- candidate,
- roles=[DAILY_CANDIDATE_ROLE],
- labels={"reason": f"today's {served_family} candidate, paired against the served model"},
+ kept_recent = 0
+ for metadata in recent:
+ if kept_recent >= recent_versions:
+ break
+ # The served version is already an entry, so scoring it twice would only spend the
+ # artefact's result budget on a duplicate.
+ if metadata["model_version"] == served_entry.model_version or not readable_head_names(metadata):
+ continue
+ version = _entry(
+ metadata,
+ roles=[RECENT_ROLE],
+ labels={"reason": f"recent {served_family} version, paired against the served model"},
prefix=prefix,
)
- if daily is not None:
- entries.append(daily)
+ if version is not None and version.key not in {entry.key for entry in entries}:
+ entries.append(version)
+ kept_recent += 1
for metadata in pinned:
pin = _entry(
@@ -152,7 +154,7 @@ def compose_manifest(
if cross is not None and cross.key not in {entry.key for entry in entries}:
entries.append(cross)
- # The served entry and the paired candidate are the reads the sweep exists for, so the cap
+ # The served entry and the recent versions are the reads the sweep exists for, so the cap
# falls on the pinned and cross-family entries, cross-family first. They are last in
# `entries`, the cross-family ones in family registration order.
return ManifestDecision(