:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): train nn_with_prio inbox ranking network with report priority (#119824)
posthog[bot]e6eff73b0ee7
diff --git a/posthog/dags/locations/signals_ml.py b/posthog/dags/locations/signals_ml.pyindex 9bdd1d1b87cf..72d4f616e649 100644--- a/posthog/dags/locations/signals_ml.py+++ b/posthog/dags/locations/signals_ml.py@@ -14,6 +14,8 @@ if is_inbox_ranking_registered():inbox_ranking_nn.inbox_ranking_nn_examples,inbox_ranking_nn.inbox_ranking_nn_model_candidate,inbox_ranking_nn.inbox_ranking_nn_model_champion,+ inbox_ranking_nn.inbox_ranking_nn_with_prio_model_candidate,+ inbox_ranking_nn.inbox_ranking_nn_with_prio_model_champion,],jobs=[inbox_ranking_nn.inbox_ranking_nn_training_job],schedules=[inbox_ranking_nn.inbox_ranking_nn_training_schedule],diff --git a/posthog/settings/object_storage.py b/posthog/settings/object_storage.pyindex 03956619829c..8d94d29023cb 100644--- a/posthog/settings/object_storage.py+++ b/posthog/settings/object_storage.py@@ -117,8 +117,9 @@ INBOX_RANKING_SERVING_MIRROR_BUCKET = os.getenv("INBOX_RANKING_SERVING_MIRROR_BUINBOX_RANKING_SERVING_MIRROR_REGION = os.getenv("INBOX_RANKING_SERVING_MIRROR_REGION", "")# Serving manifest (products/signals/dags/inbox_ranking/training/serving.py): how many of the served# family's newest versions the sweep also scores, read from the last 2N days. Each one costs the-# sweep one more model per pass, and above 4 they take the pinned and cross-family entries' places.-INBOX_RANKING_SERVING_RECENT_VERSIONS = get_from_env("INBOX_RANKING_SERVING_RECENT_VERSIONS", 3, type_cast=int)+# sweep one more model per pass. The served entry, these versions and one challenger per other family+# share the manifest's 5 entries, so a higher N leaves less room for the challengers.+INBOX_RANKING_SERVING_RECENT_VERSIONS = get_from_env("INBOX_RANKING_SERVING_RECENT_VERSIONS", 2, type_cast=int)# Scorer (products/signals/backend/ranking/scorer.py): report ids per ClickHouse vector read. A# larger call is paged at this size. The sweep also gives one scorer call at most this many ids, which# bounds the vectors and matrices one call holds.diff --git a/products/signals/backend/briefing_reports.py b/products/signals/backend/briefing_reports.pyindex 1a9739cd7a1a..0346a052304e 100644--- a/products/signals/backend/briefing_reports.py+++ b/products/signals/backend/briefing_reports.py@@ -16,7 +16,7 @@ from posthog.dataclasses import frozenfrom posthog.models import Userfrom products.signals.backend.artefact_attribution import ArtefactAttribution-from products.signals.backend.artefact_schemas import ActionabilityChoice, priority_from_judgment+from products.signals.backend.artefact_schemas import ActionabilityChoicefrom products.signals.backend.implementation_pr import (fetch_implementation_pr_state_for_reports,implementation_pr_report_filter,@@ -28,6 +28,7 @@ from products.signals.backend.report_content_gates import team_report_monitoringfrom products.signals.backend.report_lifecycle import effective_report_statusfrom products.signals.backend.report_metric_access import ReportMetricAccessPolicyfrom products.signals.backend.report_metrics import ReportMetricSnapshot, saved_metric_snapshots+from products.signals.backend.report_priority import latest_report_prioritiesfrom products.signals.backend.suggested_reviewer_index import report_ids_naming_reviewers_OPEN_STATUSES = (SignalReport.Status.READY, SignalReport.Status.PENDING_INPUT)@@ -69,31 +70,6 @@ class BriefingReportDetails:charts: list[ReportChartSnapshot]-def _latest_artefacts(team_id: int, report_ids: Sequence[str], artefact_type: str) -> dict[str, str]:- """The newest artefact content of one type per report, by report id."""- # Pick the newest id per report first, so Postgres reads `content` once per report, not once per row.- latest_ids = (- SignalReportArtefact.objects.filter(team_id=team_id, report_id__in=report_ids, type=artefact_type)- .order_by("report_id", "-created_at")- .distinct("report_id")- .values("id")- )- rows = SignalReportArtefact.objects.filter(team_id=team_id, id__in=latest_ids).values_list("report_id", "content")- return {str(report_id): content for report_id, content in rows}---def _priorities(team_id: int, report_ids: Sequence[str]) -> dict[str, str]:- """Latest priority judgment per report, read the same way the inbox serializer reads it."""- latest: dict[str, str] = {}- for report_id, content in _latest_artefacts(- team_id, report_ids, SignalReportArtefact.ArtefactType.PRIORITY_JUDGMENT- ).items():- priority = priority_from_judgment(content)- if priority is not None:- latest[report_id] = priority- return latest--def _json_object_content() -> Q:# Content is written from a pydantic schema. The guard only keeps a legacy or truncated row out# of the jsonb cast, because a failed cast fails the whole query.@@ -285,7 +261,7 @@ def report_details())found_ids = [str(report.id) for report in reports]- priorities = _priorities(team_id, found_ids)+ priorities = latest_report_priorities(team_id, found_ids)pull_requests = fetch_implementation_pr_state_for_reports(found_ids, team_id=team_id)monitoring_enabled = (team_report_monitoring_enabled(team_id)diff --git a/products/signals/backend/ranking/features.py b/products/signals/backend/ranking/features.pyindex fffa55400c1b..cd59d5325292 100644--- a/products/signals/backend/ranking/features.py+++ b/products/signals/backend/ranking/features.py@@ -6,11 +6,13 @@ matrix. A model records the set it was fit on in its `metadata.json`, so a run csets at once: the examples Parquet, the training matrix and the scoring matrix are all built perset, and a model whose declared set this build cannot produce is left unscored.-The one set is the report's combined-text embedding vector. It is the served contract: training-(`products/signals/dags/inbox_ranking/training/`) and serving (the scoring sweep in this package)-build features through the same `build_matrix`, so the booster's `feature_names` match the serving-matrix by construction. The sweep reads the vector at scoring time and needs nothing else from the-report.+Two sets live here. `report_embeddings` is the report's combined-text embedding vector, and+`report_embeddings_priority` is that vector plus the report's priority, one column per level. They+are the served contract: training (`products/signals/dags/inbox_ranking/training/`) and serving+(the scoring sweep in this package) build features through the same `build_matrix`, so the model's+`feature_names` match the serving matrix by construction. The sweep reads the vector at scoring+time, and reads the latest priority judgment only for a model whose set declares the `priority`+state column."""import abc@@ -37,6 +39,13 @@ EMBEDDING_INSERTED_AT_COLUMN = "embedding_inserted_at"# The width of text-embedding-3-small-1536, the model the report documents are embedded with. A# row whose vector is a different length is not this model's, so it is treated as missing.EMBEDDING_DIMENSIONS = 1536+# Position order, so `emb_i` is the vector's ith component in every matrix an embedding set builds.+EMBEDDING_FEATURE_NAMES = tuple(f"emb_{index}" for index in range(EMBEDDING_DIMENSIONS))++# The report-state column that holds the report's latest priority judgment as of the moment, and+# the levels a set encodes it with.+PRIORITY_COLUMN = "priority"+PRIORITY_LEVELS = ("P0", "P1", "P2", "P3", "P4")# The team's project-vector kinds, in the order a network that reads them takes them. The dataset# dag's `inbox_project_vectors` writes one row per (team, kind), and the training and serving@@ -139,9 +148,8 @@ class ReportEmbeddingsFeatureSet(FeatureSet):"""schema_version = 1- # Position order, so `emb_i` is the vector's ith component in every matrix this set builds.- feature_names = tuple(f"emb_{index}" for index in range(EMBEDDING_DIMENSIONS))- state_columns = ()+ feature_names = EMBEDDING_FEATURE_NAMES+ state_columns: tuple[str, ...] = ()# 1536 float32 columns, so a head's Parquet slice and its training matrix both scale with this.# Sized so every head of this family fits one partition's examples object and the fits stay# inside the training job's runtime budget. The budget cuts the oldest report-creation days, so@@ -160,7 +168,8 @@ class ReportEmbeddingsFeatureSet(FeatureSet):matrix = np.full((len(rows), EMBEDDING_DIMENSIONS), np.nan, dtype=np.float32)if buildable.any():matrix[buildable] = np.vstack(vectors.to_numpy()[buildable])- return pd.DataFrame(matrix, index=rows.index, columns=list(self.feature_names))+ # The vector columns, not `feature_names`: a subclass adds columns after these.+ return pd.DataFrame(matrix, index=rows.index, columns=list(EMBEDDING_FEATURE_NAMES))def buildable(self, rows: pd.DataFrame, extras: Extras = NO_EXTRAS, *, as_of: datetime.datetime | None = None@@ -196,13 +205,44 @@ class ReportEmbeddingsFeatureSet(FeatureSet):).astype(bool)+class ReportEmbeddingsPriorityFeatureSet(ReportEmbeddingsFeatureSet):+ """The report vector, then the report's priority as one column per level in `PRIORITY_LEVELS`.++ The priority comes from the `priority` state column: the dataset dag cuts it at each snapshot+ from the latest priority judgment, and the scorer reads the latest judgment at scoring time. No+ judgment, or a value outside `PRIORITY_LEVELS`, sets every level column to zero. `buildable` is+ the parent's, so this set keeps the same rows as `report_embeddings`.+ """++ schema_version = 1+ feature_names = (*EMBEDDING_FEATURE_NAMES, *(f"priority_{level}" for level in PRIORITY_LEVELS))+ state_columns = (PRIORITY_COLUMN,)++ def build_matrix(+ self, rows: pd.DataFrame, extras: Extras = NO_EXTRAS, *, as_of: datetime.datetime | None = None+ ) -> pd.DataFrame:+ # A missing column must not read as "no priority" on every row: the model would then score+ # every report as unjudged and nothing would say so.+ if PRIORITY_COLUMN not in rows:+ raise ValueError(f"the {self.name} feature set needs the {PRIORITY_COLUMN} column")+ matrix = super().build_matrix(rows, extras, as_of=as_of)+ priorities = rows[PRIORITY_COLUMN]+ for level in PRIORITY_LEVELS:+ matrix[f"priority_{level}"] = (priorities == level).to_numpy(dtype=np.float32)+ return matrix++REPORT_EMBEDDINGS_FEATURE_SET = ReportEmbeddingsFeatureSet(name="report_embeddings", extras_key=REPORT_EMBEDDINGS_EXTRA)+REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET = ReportEmbeddingsPriorityFeatureSet(+ name="report_embeddings_priority", extras_key=REPORT_EMBEDDINGS_EXTRA+)# Every set this build can produce, by name. A model that names a set absent from here cannot be# scored, the same way a model whose feature names have moved on cannot.FEATURE_SETS: Mapping[str, FeatureSet] = MappingProxyType({REPORT_EMBEDDINGS_FEATURE_SET.name: REPORT_EMBEDDINGS_FEATURE_SET,+ REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET.name: REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,})diff --git a/products/signals/backend/ranking/scorer.py b/products/signals/backend/ranking/scorer.pyindex bdaff0591e60..1b1ea8c7fc51 100644--- a/products/signals/backend/ranking/scorer.py+++ b/products/signals/backend/ranking/scorer.py@@ -10,6 +10,12 @@ a call writes nothing.A model that reads its team's project vectors gets the team's row of the day the manifest names,repeated for each report, through the same `project_inputs` the training dag builds them with.++A model whose set declares the `priority` state column gets each report's latest priority judgment,+read once per call and only when a model needs it. The sweep rescores a report when its vector or+the manifest changes, not when its priority changes, so a priority edited in the inbox after scoring+keeps the old score until the next vector. The summary workflow writes the first priority judgment+before the rewrite that re-embeds the report, so a newborn's score normally sees that judgment."""from collections.abc import Mapping, Sequence@@ -27,6 +33,7 @@ from products.signals.backend.artefact_schemas import RankingModelResult, Rankinfrom products.signals.backend.ranking.features import (EMBEDDING_COLUMN,EMBEDDING_INSERTED_AT_COLUMN,+ PRIORITY_COLUMN,REPORT_EMBEDDINGS_EXTRA,FeatureSet,ProjectInputs,@@ -44,6 +51,7 @@ from products.signals.backend.ranking.serving_manifest import ServingManifestEntfrom products.signals.backend.ranking.sinks import persist_scoresfrom products.signals.backend.report_embedding_reader import ReportVector, latest_report_vectorsfrom products.signals.backend.report_embeddings import EMBEDDING_RENDERING_TITLE_SUMMARY+from products.signals.backend.report_priority import latest_report_prioritieslogger = structlog.get_logger(__name__)@@ -125,6 +133,16 @@ def pass_project_vectors(serving: ServingSet, team_ids: Sequence[int]) -> Projecreturn ProjectVectors(available=False, by_team={})+def _state_rows(team_id: int, report_ids: Sequence[str], columns: set[str]) -> pd.DataFrame:+ """The report-state columns of `columns` that the scorer can read, indexed by report id. A+ column is read only when a model of the pass declares it."""+ rows = pd.DataFrame(index=pd.Index(list(report_ids), name="report_id"))+ if PRIORITY_COLUMN in columns:+ priorities = latest_report_priorities(team_id, report_ids)+ rows[PRIORITY_COLUMN] = [priorities.get(report_id) for report_id in report_ids]+ return rows++def _extras_frame(report_ids: Sequence[str], vectors: Mapping[str, ReportVector]) -> pd.DataFrame:present = [report_id for report_id in report_ids if report_id in vectors]return pd.DataFrame(@@ -139,17 +157,25 @@ def _extras_frame(report_ids: Sequence[str], vectors: Mapping[str, ReportVector]def _score_model(model: LoadedModel,report_ids: Sequence[str],+ rows: pd.DataFrame,vectors_by_extra: Mapping[str, Mapping[str, ReportVector]],team_project: ProjectInputs | None,) -> _ModelScores:+ """`rows` holds the report-state columns of the pass, one row per id of `report_ids`."""extra = _served_extra(model.feature_set)if extra is None:return _ModelScores(model=model, by_report={}, skip_reason=f"feature set {model.feature_set.name} is not served yet")+ unread = [column for column in model.feature_set.state_columns if column not in rows]+ if unread:+ return _ModelScores(+ model=model,+ by_report={},+ skip_reason=f"state columns {unread} of {model.feature_set.name} are not served yet",+ )if model.reads_project_vectors and team_project is None:return _ModelScores(model=model, by_report={}, skip_reason=NO_PROJECT_VECTORS)- rows = pd.DataFrame(index=pd.Index(list(report_ids), name="report_id"))extras = {extra: _extras_frame(report_ids, vectors_by_extra[extra])}# `as_of=None`: serving scores the latest vector, where training reads the one of its moment.buildable = model.feature_set.buildable(rows, extras, as_of=None).to_numpy()@@ -247,7 +273,10 @@ def score_reports(vectors_by_extra = {extra: latest_report_vectors(team_id, ids, rendering=RENDERING_BY_EXTRA[extra]) for extra in needed_extras}- scored_models = {model.entry.key: _score_model(model, ids, vectors_by_extra, team_project) for model in models}+ rows = _state_rows(team_id, ids, {column for model in models for column in model.feature_set.state_columns})+ scored_models = {+ model.entry.key: _score_model(model, ids, rows, vectors_by_extra, team_project) for model in models+ }served_key = serving.manifest.served.keyserved_vectors = vectors_by_extra[served_extra]diff --git a/products/signals/backend/ranking/serving_manifest.py b/products/signals/backend/ranking/serving_manifest.pyindex 8dc6c4dba352..778722bf015f 100644--- a/products/signals/backend/ranking/serving_manifest.py+++ b/products/signals/backend/ranking/serving_manifest.py@@ -25,6 +25,9 @@ SERVED_ROLE = RANKING_SERVED_ROLE# scores for several days and a paired read compares it against the served model without a second# scoring pass.RECENT_ROLE = "recent"+# The newest version of another trained family, scored alongside so its online grade reads the same+# reports as the served model's, before anyone promotes it.+CHALLENGER_ROLE = "challenger"# Another family's champion, scored alongside so a later interleaving needs no rescoring.CROSS_FAMILY_ROLE = "cross_family"# A model an owner keeps in the manifest through the `pin` override, so a `served` override candiff --git a/products/signals/backend/report_priority.py b/products/signals/backend/report_priority.pyindex f8cc4facb71f..29b1e48bec73 100644--- a/products/signals/backend/report_priority.py+++ b/products/signals/backend/report_priority.py@@ -1,3 +1,5 @@+from collections.abc import Sequence+from django.db import transactionfrom products.signals.backend.artefact_attribution import ArtefactAttribution@@ -37,3 +39,29 @@ def update_report_priority(*, report: SignalReport, priority: ReportPriority, at),)report.save(update_fields=["updated_at"])+++def _latest_artefacts(team_id: int, report_ids: Sequence[str], artefact_type: str) -> dict[str, str]:+ """The newest artefact content of one type per report, by report id."""+ # Pick the newest id per report first, so Postgres reads `content` once per report, not once per row.+ latest_ids = (+ SignalReportArtefact.objects.filter(team_id=team_id, report_id__in=report_ids, type=artefact_type)+ .order_by("report_id", "-created_at")+ .distinct("report_id")+ .values("id")+ )+ rows = SignalReportArtefact.objects.filter(team_id=team_id, id__in=latest_ids).values_list("report_id", "content")+ return {str(report_id): content for report_id, content in rows}+++def latest_report_priorities(team_id: int, report_ids: Sequence[str]) -> dict[str, str]:+ """Latest priority judgment per report, read the same way the inbox serializer reads it. A report+ without a judgment, or whose latest judgment has no priority, has no entry."""+ latest: dict[str, str] = {}+ for report_id, content in _latest_artefacts(+ team_id, report_ids, SignalReportArtefact.ArtefactType.PRIORITY_JUDGMENT+ ).items():+ priority = priority_from_judgment(content)+ if priority is not None:+ latest[report_id] = priority+ return latestdiff --git a/products/signals/backend/test/test_ranking_scoring.py b/products/signals/backend/test/test_ranking_scoring.pyindex f6fcf01a396e..4286f5ab8fe0 100644--- a/products/signals/backend/test/test_ranking_scoring.py+++ b/products/signals/backend/test/test_ranking_scoring.py@@ -33,6 +33,7 @@ from products.signals.backend.ranking.features import (MIN_PROJECT_HISTORY_REPORTS,PROJECT_VECTOR_KINDS,REPORT_EMBEDDINGS_FEATURE_SET,+ REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,FeatureSet,)from products.signals.backend.ranking.model_store import ModelLoadError, load_serving_set@@ -598,6 +599,43 @@ class TestScorerPersists(_ScorerTestMixin, BaseTest):assert self.store.reads.count(serving_manifest_key(PREFIX)) == 1assert captured.scopes == (0 if shared_pass else 1)+ def _judge(self, report_id: str, priority: str, created_at: datetime.datetime) -> None:+ artefact = SignalReportArtefact.objects.create(+ team=self.team,+ report_id=report_id,+ type=SignalReportArtefact.ArtefactType.PRIORITY_JUDGMENT,+ content=json.dumps({"priority": priority}),+ )+ SignalReportArtefact.objects.filter(pk=artefact.pk).update(created_at=created_at)++ def test_a_priority_model_scores_each_report_by_its_latest_priority_judgment(self) -> None:+ self.team_id = self.team.id+ feature_names = REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET.feature_names+ rng = np.random.default_rng(0)+ x = pd.DataFrame(np.zeros((40, len(feature_names))), columns=list(feature_names))+ x["priority_P0"] = np.arange(40) % 2+ x["emb_0"] = rng.normal(size=40)+ booster = xgb.XGBClassifier(n_estimators=3, max_depth=2).fit(x, x["priority_P0"]).get_booster()+ served = self.store.publish_model("nn_with_prio", REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET)+ for head in HEADS:+ self.store.objects[f"{served.prefix}/{head}.ubj"] = bytes(booster.save_raw("ubj"))+ self.store.publish_manifest([served])+ now_p0, now_p4 = self._report(self.team.id), self._report(self.team.id)+ self._judge(now_p0, "P4", NOW - datetime.timedelta(days=2))+ self._judge(now_p0, "P0", NOW - datetime.timedelta(days=1))+ self._judge(now_p4, "P0", NOW - datetime.timedelta(days=2))+ self._judge(now_p4, "P4", NOW - datetime.timedelta(days=1))+ vector = ReportVector(embedding=_vector(1), inserted_at=LANDED)++ outcomes, _, _ = self._score(+ [now_p0, now_p4], {EMBEDDING_RENDERING_TITLE_SUMMARY: {now_p0: vector, now_p4: vector}}, persist=False+ )++ scores = {+ outcome.report_id: outcome.score.results[served.key].scores["open"] for outcome in outcomes if outcome.score+ }+ assert scores[now_p0] > scores[now_p4]+def test_without_persist_nothing_is_written_or_captured(self) -> None:self.team_id = self.team.idself.store.publish_manifest([self._served()])diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.mdindex d106af6504ec..0b0eb0af28f7 100644--- a/products/signals/dags/inbox_ranking/AGENTS.md+++ b/products/signals/dags/inbox_ranking/AGENTS.md@@ -34,11 +34,11 @@ Read `README.md` first for what the dataset is and how partitions behave. This f- The serving path reads the deployment's own object store, never the dataset bucket: the Temporal workers have no credential for that bucket, and keeping it that way is why the manifest asset copies what the sweep needs. `serving/models/<key>/` holds a version's `metadata.json` and its model files, `serving/manifest.json` names the models one pass scores. The holdout boosters grade a promotion and must stay behind.- An entry's files come from `ServingManifestEntry.files`: one `<head>.ubj` per head for `xgboost`, and the one `model.onnx` that `model_file` names for `onnx`. The copy step and the model store read that property, and the pin check reads the same names from each head's `file` in `metadata.json`, so a new model kind with another file layout keeps the two in step. Keep `onnxruntime` and `xgboost` imports inside the handlers' `load`: a test pins that importing the sweep loads neither.- The manifest schema and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, next to the feature contract, because the dag writes them and the sweep reads them. Its validators run at write time, so the dag refuses a manifest the sweep could not act on; the entry cap is the `ranking_score` artefact's, so a manifest can never produce a row the artefact refuses.-- The `recent` role keeps the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` versions of the served `model_name`, read from the last 2N partitions, so each version gets several days of online scores. Each one passes the pin check before composition, because one missing file fails the publish. Keep the order under the cap as served, recent, pinned, cross-family. Each recent entry is one more model per sweep pass, so measure the sweep's duration, peak RSS and scored and deferred counts on the `inbox_ranking_sweep_finished` log before and after a change to N.+- The `recent` role keeps the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` versions of the served `model_name`, read from the last 2N partitions, so each version gets several days of online scores. The `challenger` role keeps the newest readable version of each other `MODEL_FAMILIES` entry, read from its last 2 partitions. Each one passes the pin check before composition, because one missing file fails the publish. Keep the order under the cap as served, recent, challenger, pinned, cross-family. Each recent entry is one more model per sweep pass, so measure the sweep's duration, peak RSS and scored and deferred counts on the `inbox_ranking_sweep_finished` log before and after a change to N.- Publishing order is load-bearing. Every model copy finishes before `manifest.json` is written, so a failed copy leaves the previous manifest and the previous models serving. A model version is immutable, so a version whose `metadata.json` is already at the target is not copied again. A run with nothing to publish, such as a run without `served.json`, writes nothing rather than an empty manifest. The served model passes the same check as a pin (`_pin_metadata`) before composition.- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror: they are per-team data of this deployment, so the mirror's manifest names none. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.- `HEADS` holds only the heads the inbox reads (`HeadRanker` in `backend/ranking/inbox_sort.py`). Add a head there only together with a surface that reads it. The promotion gate skips a champion head that is not in `HEADS`, so a trim does not block promotion.-- `nn` is trained by the `signals_ml` location (`nn/dag.py`), never by the main one. torch is imported only inside the candidate asset's body (through `nn/train.py` and `multihead/network.py`). `signals_ml` must import without torch, and `test_multihead_network.py` pins that. Its decision goes through `decide_family_champion`, the same rule as the boosters, graded through the incumbent's `model.holdout.onnx`. Keep `signals_ml` commented out in `.dagster_home/workspace.yaml`, and keep its schedule (04:30 UTC) running by default on prod US only.+- `nn_v2` and `nn_with_prio` are trained by the `signals_ml` location (`nn/dag.py`), never by the main one. They share one `inbox_ranking_nn_examples` object, and each has its own candidate and champion asset from `family_assets`. The `nn_v2` assets keep the names `inbox_ranking_nn_model_{candidate,champion}`, so their Dagster history continues. A later family's candidate depends on the previous family's champion, for memory and not for data: two candidates at once can exceed the job's memory limit. A family whose set reads a state column the examples object lacks skips with a warning rather than training on zeros. The main examples asset builds only the sets of `families_trained_in(MAIN_LOCATION)`, so registering a set for `signals_ml` does not add a 1536-column object to the main job. torch is imported only inside the candidate asset's body (through `nn/train.py` and `multihead/network.py`). `signals_ml` must import without torch, and `test_multihead_network.py` pins that. Its decision goes through `decide_family_champion`, the same rule as the boosters, graded through the incumbent's `model.holdout.onnx`. Keep `signals_ml` commented out in `.dagster_home/workspace.yaml`, and keep its schedule (04:30 UTC) running by default on prod US only.## Invariants — do not breakdiff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.mdindex c4745d9287e2..4de50674f6ea 100644--- a/products/signals/dags/inbox_ranking/README.md+++ b/products/signals/dags/inbox_ranking/README.md@@ -149,9 +149,9 @@ s3://<bucket>/<prefix>/└── inbox_ranking_served_scores/v1/dt=YYYY-MM-DD/ # the scoring sweep's scores of the reports born that day, per model key```-- **A model is a `(model_name, model_version, model_role)`.** `model_name` is the recipe: which features, which learner and which training settings: `report_embeddings_v2`, per-head XGBoost, is the one trained here. `nn_v2`, one ONNX network, is trained by the `signals_ml` location, and each `MODEL_FAMILIES` entry names its `learner` and the `location` that trains it. `model_version` is the partition day it was fit on, and `model_role` is the role it was scored or trained under (`candidate` on the training events; `served`, `recent`, `pinned` or `cross_family` on the online grade, and `daily_candidate` on grades written before the `recent` role). Each family owns a prefix under the models path, so two families trained on the same day cannot collide. A candidate is graded only against the served model of its own name. The served pointer can name any family, so a rollback can cross families.+- **A model is a `(model_name, model_version, model_role)`.** `model_name` is the recipe: which features, which learner and which training settings: `report_embeddings_v2`, per-head XGBoost, is the one trained here. `nn_v2`, one ONNX network, and `nn_with_prio`, the same network with the report's priority as an input, are trained by the `signals_ml` location, and each `MODEL_FAMILIES` entry names its `learner` and the `location` that trains it. `model_version` is the partition day it was fit on, and `model_role` is the role it was scored or trained under (`candidate` on the training events; `served`, `recent`, `challenger`, `pinned` or `cross_family` on the online grade, and `daily_candidate` on grades written before the `recent` role). Each family owns a prefix under the models path, so two families trained on the same day cannot collide. A candidate is graded only against the served model of its own name. The served pointer can name any family, so a rollback can cross families.- **A recipe change gets a new `model_name`.** `report_embeddings` and `nn` became `report_embeddings_v2` and `nn_v2` when `action` started to count a thumbs up. A candidate's `metadata.json` carries `recipe` and `recipe_hash`, a stable hash of the recipe (`training/recipe.py`): the learner, the feature set and its schema version, the project-vector kinds and their history settings, the booster params or network config, each head's cohort, label, horizon, label columns and status-label flag, `lookback_days`, `holdout_days`, the example grain and row budget, and `dataset_version`. Dates, `run_id`, row counts and metrics stay out, so a daily retrain keeps the hash. The candidate asset compares its hash with the newest version of the same name, at or before its partition, that has one, and on a mismatch it fails before it writes the partition and names the changed inputs. A re-run of a partition therefore compares with that partition's own saved recipe. The search covers every dataset version, so a `DATASET_VERSION` bump also needs a new name. A version without a hash is not compared. This includes an env change such as `INBOX_RANKING_TRAINING_LOOKBACK_DAYS`. To start a new recipe, add a `MODEL_FAMILIES` entry with a new name (`nn_v2` after `nn`) and remove the old entry: the old versions stay servable by key, and the promotion gate never compares two recipes. Two recipes can train side by side for a comparison, but each costs a manifest slot, so do not do it by default. `recipe_hash` is on `inbox_ranking_candidate_trained` and `inbox_ranking_promotion_decided` (with `incumbent_recipe_hash`), and `inbox_ranking_serving_manifest_published` carries `model_recipe_hashes` per model key.-- **A feature set is one feature universe.** `products/signals/backend/ranking/features.py` holds a `FeatureSet` per universe: its name, its schema version, its ordered feature names, the report-state columns it reads, `build_matrix`, and how many rows it wants per report and per head. `report_embeddings` is the one today. A candidate records the set it was fit on in its `metadata.json`, so the grader checks a model against its own set rather than one global contract, the examples asset writes one Parquet per set, and the unseen scorer builds one matrix per set and shares it across the families that read it. Adding a set means an entry in `FEATURE_SETS`; a model naming a set this build cannot produce is logged and left unscored.+- **A feature set is one feature universe.** `products/signals/backend/ranking/features.py` holds a `FeatureSet` per universe: its name, its schema version, its ordered feature names, the report-state columns it reads, `build_matrix`, and how many rows it wants per report and per head. `report_embeddings` is the report vector, and `report_embeddings_priority` is that vector plus one column per priority level (`priority_P0` to `priority_P4`, all zero when the report has no judgment), read from the `priority` state column. A candidate records the set it was fit on in its `metadata.json`, so the grader checks a model against its own set rather than one global contract, the examples asset writes one Parquet per set, and the unseen scorer builds one matrix per set and shares it across the families that read it. Adding a set means an entry in `FEATURE_SETS`; a model naming a set this build cannot produce is logged and left unscored. The main examples asset writes a Parquet only for the sets of the families the main location trains, so a set only `signals_ml` reads costs it nothing.- **Examples are one row per report at its birth, by default.** For partition `dt=D` the examples asset reads the report-state and labels snapshots `dt=D-lookback..D` and emits one row per report, on the snapshot of the day it was created, labeled from the snapshot `horizon_days` later (7 for open, 14 for action, 30 for fixed). Features are built by the feature set from that snapshot's state columns plus `age_hours`, the report's age at the snapshot. Labels are aligned to the state spine, so a report with no label event is a negative (all-zero labels), not absent. Label-only rows (no Postgres state) are skipped, as are state rows read long after their snapshot day (backfills carry current Postgres state; see `features_observed_at`) and, for the heads that read the status stream (`action` and `fixed`), rows whose status telemetry fails the dataset's `label_provenance_ok` check. Birth is the moment serving scores a report, and the label that comes out is a per-report probability, which is what the inbox ordering needs. A report born before the window, or one whose birth-day snapshot is missing from it, is no example; `reports_missing_birth_snapshot` on the asset counts the second case, which is a gap in the partitions.- **An outcome already visible on the birth day is a future positive.** A report born on day D has no scoring moment before D, so an outcome visible at D belongs to the moment being built rather than to an earlier one; the labels of D therefore read as their defaults for a report born on D. Most outcomes land on the birth day, so this is where the positives are: censoring on them costs the `pr_created` head most of its training signal. `<set>_<head>_birth_day_positives` on the examples asset, and `birth_day_positives` on the `inbox_ranking_examples_built` event, say how many positives the rule keeps. These rows carry a hindsight the other rows do not: the state snapshot reads `signal_count`, `total_weight`, `run_count` and the text sizes live from Postgres a few hours after day D ends, so a birth-day outcome happened before its own feature read. Both the holdout AUC and the newborn unseen grade therefore read optimistically on these rows, until the scoring sweep's timestamped score log replaces the daily snapshot as this table's source.- **Every head asks whether the outcome happened within N days of the scoring moment.**@@ -186,7 +186,7 @@ Each entry names its `model_kind`, and the sweep loads it through that kind's ha- `xgboost`, the default for a model without the field: one booster per head, `<head>.ubj`. The entry's `heads` list names the files.- `onnx`: one `model.onnx` for every head. The entry names it in `model_file`, and `heads` still lists the heads it scores. The graph takes one float32 input, `report`, of shape (batch, feature-set width), and gives one output, `probabilities`, of shape (batch, heads). The column order of that output is `output_heads` in the model's `metadata.json`, and each head entry in `heads` there names `model.onnx` as its `file`. At load, the handler checks the input names and widths against what the model's feature set builds, and a mismatch skips the model like an unloadable booster. The session runs on CPU with a bounded thread count, and `onnxruntime` is imported only when a model loads. The project-vector inputs are not served yet, so the sweep skips an `nn` model, whose graph also takes `project` and `has_history`, until it feeds them.-The manifest holds one entry per model with its `roles`: `served` for the model `served.json` names, `recent` for each of the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` (default 3) versions of the served `model_name`, `pinned` for each model the `pin` override keeps (see [Overrides](#overrides)), and `cross_family` for every other family's `champion.json` that has a readable head. Nothing writes those files anymore, so they name the last version the retired automatic promotion chose. The dag reads the `recent` versions newest first from `<model_name>/dt=D/metadata.json` in the 2N days up to the partition, where N is the setting. It skips a version with no readable head, the served version, and a version that fails the pin check, and it removes duplicate keys. So each version gets online scores for about N days, not for one. A pass scores all of them, so an online paired read and a later interleaving need no rescoring; the entry count is capped at the `ranking_score` artefact's own limit (5). The order is `served`, `recent`, `pinned`, `cross_family`, so the cap drops `cross_family` entries first and `pinned` entries next. Each `recent` version adds one model to every sweep pass, so a higher N costs sweep time and pod memory. The schema, the role names and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, and its validators run at write time, so the dag refuses a manifest the sweep could not act on. A model version is immutable, so a version already in the store is not copied again, and `manifest.json` is written last: a failed copy leaves the previous manifest and the previous models serving. When `served.json` is missing, or names a model that fails the pin check, the asset writes nothing and says why, and the previous manifest keeps serving.+The manifest holds one entry per model with its `roles`: `served` for the model `served.json` names, `recent` for each of the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` (default 2) versions of the served `model_name`, `challenger` for the newest readable version of each other `MODEL_FAMILIES` entry, read from its last 2 days because an `nn` candidate can land after the manifest runs, `pinned` for each model the `pin` override keeps (see [Overrides](#overrides)), and `cross_family` for every other family's `champion.json` that has a readable head. Nothing writes those files anymore, so they name the last version the retired automatic promotion chose. The dag reads the `recent` versions newest first from `<model_name>/dt=D/metadata.json` in the 2N days up to the partition, where N is the setting. It skips a version with no readable head, the served version, and a version that fails the pin check, and it removes duplicate keys. So each version gets online scores for about N days, not for one. A pass scores all of them, so an online paired read and a later interleaving need no rescoring; the entry count is capped at the `ranking_score` artefact's own limit (5). The order is `served`, `recent`, `challenger`, `pinned`, `cross_family`, so the cap drops `cross_family` entries first and `pinned` entries next. Challengers come in family registration order, so when there are more challengers than free entries, the last registered families lose their place. A challenger is how a new recipe gets online scores on the same reports as the served model before anyone promotes it. Each `recent` version adds one model to every sweep pass, so a higher N costs sweep time and pod memory. The schema, the role names and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, and its validators run at write time, so the dag refuses a manifest the sweep could not act on. A model version is immutable, so a version already in the store is not copied again, and `manifest.json` is written last: a failed copy leaves the previous manifest and the previous models serving. When `served.json` is missing, or names a model that fails the pin check, the asset writes nothing and says why, and the previous manifest keeps serving.A region that does not train serves the same models through a mirror. When `INBOX_RANKING_SERVING_MIRROR_BUCKET` is set, the asset repeats the publish into that bucket after the primary publish succeeds, in the same order and with the same skip of versions already there. A mirror failure does not fail the asset: it logs, and the `inbox_ranking_serving_manifest_published` event carries `mirror_published=false`. The property is absent when no mirror is set.@@ -345,21 +345,24 @@ The network leaves out every report that is in any head's holdout, so the compar### The nn family-`nn` is one multi-head network (`multihead/network.py`) over the report vector and the team's `typical`, `open`, `action` and `dismiss` project vectors, with the heads `open`, `action` and `fixed`. It is a model family like the per-head ones: it has its own prefix, the main job's serving manifest names a served or cross-family `nn` model as an `onnx` entry, and the online grader grades it. The main location's candidate and champion assets never fit it.+`nn` is one multi-head network (`multihead/network.py`) over the report vector and the team's `typical`, `open`, `action` and `dismiss` project vectors, with the heads `open`, `action` and `fixed`. It is a model family like the per-head ones: it has its own prefix, the main job's serving manifest names a served, challenger or cross-family `nn` model as an `onnx` entry, and the online grader grades it. The main location's candidate and champion assets never fit it.++Two `nn` families train every day. `nn_v2` reads the `report_embeddings` set. `nn_with_prio` reads `report_embeddings_priority`: its `report` input carries the 1536 vector components and then the five priority levels, and those five columns skip the shared projection and go straight into the trunk input. Neither is served by default; a person serves one with `inbox_ranking_promote_job` after comparing their online grades.`inbox_ranking_nn_training_job` lives on its own code location, `posthog/dags/locations/signals_ml.py`, which runs on an image with the `inbox-ranking` group. torch is imported only inside the candidate asset's body, so the location module imports on the main image too, and `test_multihead_network.py` fails if torch reaches it. The job writes:```texts3://<bucket>/<prefix>/-├── inbox_ranking_nn_examples/v1/dt=YYYY-MM-DD/ # rows, a label and a holdout flag per head; no vectors-└── inbox_ranking_models/v1/nn_v2/- ├── dt=YYYY-MM-DD/model.onnx + model.holdout.onnx + metadata.json- └── champion.json # retired; nothing writes it+├── inbox_ranking_nn_examples/v1/dt=YYYY-MM-DD/ # rows, a label and a holdout flag per head, priority; no vectors+├── inbox_ranking_models/v1/nn_v2/+│ ├── dt=YYYY-MM-DD/model.onnx + model.holdout.onnx + metadata.json+│ └── champion.json # retired; nothing writes it+└── inbox_ranking_models/v1/nn_with_prio/dt=YYYY-MM-DD/model.onnx + model.holdout.onnx + metadata.json```-- **Examples** are the prototype's: each head's `example_moments` over the `report_embeddings` set, capped by `cap_examples` and split by `holdout_mask`, so every head's rows and holdout are the ones its `report_embeddings` champion is graded on. The object holds no vector. The candidate and champion assets rebuild the report vectors and the project vectors from the dt=D embeddings and labels snapshots.-- **Candidate**: the network trains on the rows in no head's holdout and is graded on each head's own holdout through the same `head_metrics` the boosters use, so `metadata.json` carries the same per-head fields (pooled and within-team AUC, average precision, ECE, readability, row counts, example window). Readability uses the boosters' permutation null: three networks fit on permuted labels. `model.onnx` is a refit on every labeled row, and `model.holdout.onnx` is the train-only fit. `metadata.json` has `model_kind: onnx`, `feature_set: report_embeddings`, and `output_heads`, the graph's column order.-- **Champion**: the same rule as the other families (`decide_family_champion`), recorded and never written to a pointer. The incumbent is the served model when it is an `nn` model, graded through its `model.holdout.onnx` on the candidate's holdout, with `onnxruntime`.+- **Examples** are the prototype's: each head's `example_moments` over the `report_embeddings` set, capped by `cap_examples` and split by `holdout_mask`, so every head's rows and holdout are the ones its `report_embeddings` champion is graded on. The object holds no vector, and one object serves both families. Each row carries its `priority` from its own snapshot. The candidate and champion assets rebuild the report vectors and the project vectors from the dt=D embeddings and labels snapshots. An object written before the `priority` column existed makes the `nn_with_prio` candidate skip with a warning; re-run the examples asset for that day.+- **Candidate**: the network trains on the rows in no head's holdout and is graded on each head's own holdout through the same `head_metrics` the boosters use, so `metadata.json` carries the same per-head fields (pooled and within-team AUC, average precision, ECE, readability, row counts, example window). Readability uses the boosters' permutation null: three networks fit on permuted labels. `model.onnx` is a refit on every labeled row, and `model.holdout.onnx` is the train-only fit. `metadata.json` has `model_kind: onnx`, the family's `feature_set`, and `output_heads`, the graph's column order. The `nn_with_prio` candidate runs after the `nn_v2` champion, for memory and not for data: each candidate holds an embeddings snapshot and the project vectors.+- **Champion**: the same rule as the other families (`decide_family_champion`), recorded and never written to a pointer. The incumbent is the served model when it has the candidate's `model_name`, graded through its `model.holdout.onnx` on the candidate's holdout, with `onnxruntime`.- **A missing partition is a skip.** If the dt=D state, labels or embeddings snapshot is missing, each asset logs and writes nothing, so the partition keeps the objects it holds.- **Schedule**: 04:30 UTC, after the dataset job's 2h budget (02:30 UTC start) and before the 05:00 UTC training job when the run is short enough. A candidate that lands after that day's manifest is paired by the next day's manifest. The schedule runs by default on prod US and is stopped everywhere else.@@ -380,7 +383,7 @@ dagster job launch -w .dagster_home/workspace.yaml --location posthog.dags.locat| `INBOX_RANKING_TRAINING_HOLDOUT_DAYS` | `7` | Trailing days of reports that grade a candidate. || `INBOX_RANKING_SERVING_MIRROR_BUCKET` | unset | A second deployment's object-storage bucket that receives a copy of every serving publish, through ambient AWS config. Unset means no mirror. || `INBOX_RANKING_SERVING_MIRROR_REGION` | unset | AWS region of the mirror bucket. |-| `INBOX_RANKING_SERVING_RECENT_VERSIONS` | `3` | How many of the served family's newest versions the manifest carries under the `recent` role, read from the last 2N days. Each one adds one model to every sweep pass. |+| `INBOX_RANKING_SERVING_RECENT_VERSIONS` | `2` | How many of the served family's newest versions the manifest carries under the `recent` role, read from the last 2N days. Each one adds one model to every sweep pass and takes an entry a challenger could use. || `INBOX_RANKING_SCORING_BATCH_SIZE` | `500` | Report ids per ClickHouse vector read in the scorer. A larger call is paged at this size. |Writes use boto3: ambient AWS config (the node role) when the dedicated bucket is set, the `OBJECT_STORAGE_*` endpoint and credentials otherwise. Readers (project-level warehouse tables, model training) use a separate read-only credential provisioned with the bucket.diff --git a/products/signals/dags/inbox_ranking/multihead/inputs.py b/products/signals/dags/inbox_ranking/multihead/inputs.pyindex ff6787c4934d..9e28c392d198 100644--- a/products/signals/dags/inbox_ranking/multihead/inputs.py+++ b/products/signals/dags/inbox_ranking/multihead/inputs.py@@ -36,7 +36,7 @@ class ProjectContext:@frozenclass NetworkInputs:- # (rows, D) float32.+ # (rows, D + report features) float32: the report vector, then any columns the feature set adds.reports: np.ndarraycontext: ProjectContext | None = Nonediff --git a/products/signals/dags/inbox_ranking/multihead/network.py b/products/signals/dags/inbox_ranking/multihead/network.pyindex 8b74b0775188..60199d065f72 100644--- a/products/signals/dags/inbox_ranking/multihead/network.py+++ b/products/signals/dags/inbox_ranking/multihead/network.py@@ -8,6 +8,10 @@ One projection serves every vector, so the report and the team's history land inand their element-wise product can say "this report looks like what the team engaged with". Weightdecay is decoupled (AdamW), and dropout sits on the trunk's input and output.+The `report` input can carry `report_features` columns after the vector, such as the report's+priority levels. Those columns skip the projection and go straight into the trunk input. With the+default of zero, the graph is the vector-only one.+A label is NaN where the head has no example for the row, and a NaN label adds no gradient, so onenetwork trains every head on rows that only some heads cover.@@ -36,13 +40,14 @@ ONNX_OPSET = 17class _Model(nn.Module):- def __init__(self, *, heads: int, kinds: int, config: NetworkConfig) -> None:+ def __init__(self, *, heads: int, kinds: int, config: NetworkConfig, report_features: int = 0) -> None:super().__init__()self.kinds = kinds+ self.report_features = report_featuresk, h = config.projection_dim, config.trunk_dimself.projection = nn.Linear(EMBEDDING_DIMENSIONS, k)self.input_dropout = nn.Dropout(config.dropout)- self.trunk = nn.Linear(k + kinds * (2 * k + 1), h)+ self.trunk = nn.Linear(k + report_features + kinds * (2 * k + 1), h)self.output_dropout = nn.Dropout(config.dropout)self.heads = nn.Linear(h, heads)for layer in (self.projection, self.trunk, self.heads):@@ -52,9 +57,14 @@ class _Model(nn.Module):def forward(self, report: torch.Tensor, project: torch.Tensor | None = None, has_history: torch.Tensor | None = None) -> torch.Tensor:- """(N, heads) logits from `report` (N, D), `project` (N, kinds, D) and `has_history` (N, kinds)."""- projected = self.projection(report * INPUT_SCALE)+ """(N, heads) logits from `report` (N, D + report_features), `project` (N, kinds, D) and+ `has_history` (N, kinds)."""+ # Sliced only when there are extra columns, so a vector-only graph stays as it was exported.+ vector = report[:, :EMBEDDING_DIMENSIONS] if self.report_features else report+ projected = self.projection(vector * INPUT_SCALE)parts = [projected]+ if self.report_features:+ parts.append(report[:, EMBEDDING_DIMENSIONS:])if self.kinds:if project is None or has_history is None:raise ValueError("a network built with context needs project vectors")@@ -89,16 +99,27 @@ def masked_binary_cross_entropy(logits: torch.Tensor, labels: torch.Tensor) -> tclass MultiHeadNetwork:- def __init__(self, *, heads: Sequence[str], with_context: bool, kinds: int, config: NetworkConfig) -> None:+ def __init__(+ self,+ *,+ heads: Sequence[str],+ with_context: bool,+ kinds: int,+ config: NetworkConfig,+ report_features: int = 0,+ ) -> None:self.head_names = tuple(heads)self.with_context = with_contextself.kinds = kinds if with_context else 0+ self.report_features = report_featuresself.config = config# Every random draw (initialization, shuffle, dropout) comes from this seeded state, and the# global torch state stays as it was.self._rng_state = torch.Generator().manual_seed(config.seed).get_state()with self._seeded():- self.model = _Model(heads=len(self.head_names), kinds=self.kinds, config=config)+ self.model = _Model(+ heads=len(self.head_names), kinds=self.kinds, config=config, report_features=report_features+ )weights = [parameter for name, parameter in self.model.named_parameters() if name.endswith("weight")]biases = [parameter for name, parameter in self.model.named_parameters() if name.endswith("bias")]self._optimizer = torch.optim.AdamW(@@ -171,13 +192,13 @@ class MultiHeadNetwork:def export_onnx(network: MultiHeadNetwork, path: Path) -> tuple[str, ...]:"""Write the network to `path` as an ONNX graph and return its head order.- The inputs are `report` (N, D) and, for a network with context, `project` (N, kinds, D) and+ The inputs are `report` (N, D + report_features) and, for a network with context, `project` (N, kinds, D) and`has_history` (N, kinds), all float32. The one output is `probabilities` (N, heads), in thereturned head order. N is a dynamic axis."""module = _Probabilities(network.model).eval()batch = 2- example: tuple[torch.Tensor, ...] = (torch.zeros(batch, EMBEDDING_DIMENSIONS),)+ example: tuple[torch.Tensor, ...] = (torch.zeros(batch, EMBEDDING_DIMENSIONS + network.report_features),)if network.with_context:example += (torch.zeros(batch, network.kinds, EMBEDDING_DIMENSIONS), torch.zeros(batch, network.kinds))input_names = list(ONNX_INPUTS[: len(example)])diff --git a/products/signals/dags/inbox_ranking/multihead/prototype.py b/products/signals/dags/inbox_ranking/multihead/prototype.pyindex d224f2da825c..1577d2ec6d68 100644--- a/products/signals/dags/inbox_ranking/multihead/prototype.py+++ b/products/signals/dags/inbox_ranking/multihead/prototype.py@@ -34,8 +34,10 @@ from posthog.dataclasses import frozenfrom products.signals.backend.ranking.features import (EMBEDDING_DIMENSIONS,+ PRIORITY_COLUMN,REPORT_EMBEDDINGS_FEATURE_SET,Extras,+ FeatureSet,project_inputs,)from products.signals.dags.inbox_ranking.common import snapshot_bounds@@ -60,7 +62,7 @@ ABLATIONS = (REPORT_ONLY, WITH_PROJECT_VECTORS)class ExampleTable:"""One row per report moment, with a label and a holdout flag per head in `PROTOTYPE_HEADS` order."""- # report_id, snapshot_date, report_team_id.+ # report_id, snapshot_date, report_team_id, and priority when the snapshots carry it.rows: pd.DataFrame# (rows, heads) float32, NaN where the head has no example for the row.labels: np.ndarray@@ -106,19 +108,33 @@ def example_table(joined = pd.concat(frames, axis=1, join="outer").sort_index()rows = joined.index.to_frame(index=False)rows["report_team_id"] = [snapshots[date].state.at[report_id, "report_team_id"] for report_id, date in joined.index]+ if all(PRIORITY_COLUMN in snapshot.state for snapshot in snapshots.values()):+ # Read per row from its own snapshot, so a moment carries the priority it had then.+ rows[PRIORITY_COLUMN] = [+ _priority_or_none(snapshots[date].state.at[report_id, PRIORITY_COLUMN]) for report_id, date in joined.index+ ]labels = np.column_stack([joined[f"{head.name}:label"].to_numpy(dtype=np.float32) for head in PROTOTYPE_HEADS])holdout = np.column_stack([joined[f"{head.name}:holdout"].eq(True).to_numpy() for head in PROTOTYPE_HEADS])return ExampleTable(rows=rows, labels=labels, holdout=holdout)-def report_matrix(rows: pd.DataFrame, extras: Extras) -> np.ndarray:- """(rows, D) float32 report vectors, built by the set's own `build_matrix` at each row's moment,- so a row carries the vector its champion example carries."""- matrix = np.empty((len(rows), EMBEDDING_DIMENSIONS), dtype=np.float32)+def _priority_or_none(value: object) -> str | None:+ return value if isinstance(value, str) else None+++def report_matrix(rows: pd.DataFrame, extras: Extras, feature_set: FeatureSet = FEATURE_SET) -> np.ndarray:+ """(rows, features) float32 report inputs, built by the set's own `build_matrix` at each row's+ moment, so a row carries the vector its champion example carries. The set's state columns come+ from `rows`, and a set raises when one is missing."""+ matrix = np.empty((len(rows), len(feature_set.feature_names)), dtype=np.float32)+ state_columns = [column for column in feature_set.state_columns if column in rows]for date, group in rows.groupby("snapshot_date", sort=True):_, snapshot_end = snapshot_bounds(date.isoformat())- frame = pd.DataFrame(index=pd.Index(group["report_id"], name="report_id"))- matrix[group.index.to_numpy()] = FEATURE_SET.build_matrix(frame, extras, as_of=snapshot_end).to_numpy(+ frame = pd.DataFrame(+ {column: group[column].to_numpy() for column in state_columns},+ index=pd.Index(group["report_id"], name="report_id"),+ )+ matrix[group.index.to_numpy()] = feature_set.build_matrix(frame, extras, as_of=snapshot_end).to_numpy(dtype=np.float32)return matrixdiff --git a/products/signals/dags/inbox_ranking/nn/dag.py b/products/signals/dags/inbox_ranking/nn/dag.pyindex fed303d1c773..5d04fca3a9bd 100644--- a/products/signals/dags/inbox_ranking/nn/dag.py+++ b/products/signals/dags/inbox_ranking/nn/dag.py@@ -1,16 +1,20 @@-"""Daily training for the `nn` model family: one multi-head network over the report vector and the-team's project vectors, exported to ONNX.+"""Daily training for the `nn` model families: one multi-head network over the report vector and the+team's project vectors, exported to ONNX. `nn_v2` reads the vector only, and `nn_with_prio` also+reads the report's priority levels.-Three assets on the dataset dag's daily partition, registered by the `signals_ml` code location,-which runs on the image that carries torch:+One examples asset, and a candidate and a champion asset per family, on the dataset dag's daily+partition, registered by the `signals_ml` code location, which runs on the image that carries torch:- inbox_ranking_nn_examples/v1/dt=D/ rows, labels and holdout flags (`nn/examples.py`)- inbox_ranking_models/v1/nn/dt=D/model.onnx the refit that serves, + model.holdout.onnx and metadata.json- (no object) the champion asset grades the candidate and records the decision+ inbox_ranking_nn_examples/v1/dt=D/ rows, labels, holdout flags and priority (`nn/examples.py`)+ inbox_ranking_models/v1/<model_name>/dt=D/model.onnx the refit that serves, + model.holdout.onnx and metadata.json+ (no object) the champion asset grades the candidate and records the decision-The main location's training job never fits `nn`, and this job never writes another family's-objects or any pointer. A person serves an `nn` version with `inbox_ranking_promote_job`. The main-job's serving manifest and online grader then publish and grade it as an `onnx` entry with no+The families' candidates run one after the other: each holds an embeddings snapshot and the project+vectors, and two at once can exceed the run's memory limit.++The main location's training job never fits these families, and this job never writes another+family's objects or any pointer. A person serves a version with `inbox_ranking_promote_job`. The+main job's serving manifest and online grader then publish and grade it as an `onnx` entry with nochange here. A candidate that lands after that day's manifest is paired by the next day's manifest.All three assets read the dt=D dataset partitions from the dataset bucket, as the main training job@@ -19,7 +23,7 @@ it stay as they are."""import datetime-from collections.abc import Mapping+from collections.abc import Mapping, Sequencefrom typing import Anyimport numpy as np@@ -47,6 +51,7 @@ from products.signals.dags.inbox_ranking.dataset.dag import EMBEDDINGS_TABLE, LAfrom products.signals.dags.inbox_ranking.dataset.project_vectors import EMBEDDING_COLUMNS, LABEL_COLUMNSfrom products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfigfrom products.signals.dags.inbox_ranking.multihead.local import report_vectors+from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTablefrom products.signals.dags.inbox_ranking.nn.examples import (NN_HEADS,build_examples,@@ -83,11 +88,14 @@ from products.signals.dags.inbox_ranking.training.telemetry import (holdout_calibration_events,)from products.signals.dags.inbox_ranking.training.train import HoldoutGrade-from products.signals.dags.inbox_ranking.training.unseen import NN_FAMILY+from products.signals.dags.inbox_ranking.training.unseen import NN_FAMILY, NN_WITH_PRIO_FAMILY, ModelFamilyNN_EXAMPLES_TABLE = "inbox_ranking_nn_examples"+# The `nn_v2` assets keep the names they had as the only family, so their Dagster history continues.NN_CANDIDATE_ASSET = "inbox_ranking_nn_model_candidate"NN_CHAMPION_ASSET = "inbox_ranking_nn_model_champion"+NN_WITH_PRIO_CANDIDATE_ASSET = "inbox_ranking_nn_with_prio_model_candidate"+NN_WITH_PRIO_CHAMPION_ASSET = "inbox_ranking_nn_with_prio_model_champion"# The settings the prototype compared against the champion recipe with.NETWORK_CONFIG = NetworkConfig()@@ -175,37 +183,44 @@ def inbox_ranking_nn_examples(context: dagster.AssetExecutionContext) -> None:)[email protected](name=NN_CANDIDATE_ASSET, deps=[NN_EXAMPLES_TABLE], **NN_ASSET_KWARGS)-def inbox_ranking_nn_model_candidate(context: dagster.AssetExecutionContext) -> None:- if skip_unconfigured(context):- return- context.add_output_metadata(train_candidate(context, s3_client(), dataset_bucket(), context.partition_key))+def _missing_state_columns(family: ModelFamily, table: ExampleTable) -> list[str]:+ """The state columns `family`'s set reads that the examples rows lack. An examples object written+ before a column existed lacks it, and building the set from it would raise."""+ return [column for column in family.feature_set.state_columns if column not in table.rows]def train_candidate(- context: dagster.AssetExecutionContext, client, bucket: str, partition_key: str+ context: dagster.AssetExecutionContext, client, bucket: str, partition_key: str, family: ModelFamily) -> dict[str, dagster.MetadataValue]:- """Fit the day's network, write its graphs and `metadata.json` under `nn/dt=D/`, and return the- asset metadata. A partition without examples or snapshots keeps the objects it already holds,- because the served pointer can name them."""+ """Fit the day's network for `family`, write its graphs and `metadata.json` under+ `<model_name>/dt=D/`, and return the asset metadata. A partition without examples or snapshots+ keeps the objects it already holds, because the served pointer can name them."""prefix = settings.INBOX_RANKING_DATASET_S3_PREFIXarrow = read_parquet_if_exists(client, bucket, partition_object_key(prefix, NN_EXAMPLES_TABLE, partition_key))if arrow is None:- context.log.warning(f"no nn examples for dt={partition_key}; nn keeps this partition as it stands")+ context.log.warning(f"no nn examples for dt={partition_key}; {family.name} keeps this partition as it stands")+ return {"skipped": dagster.MetadataValue.bool(True)}+ examples = examples_from_arrow(arrow)+ missing = _missing_state_columns(family, examples.table)+ if missing:+ # Zeros in place of the missing column would train a model that never saw the input.+ context.log.warning(+ f"the nn examples for dt={partition_key} have no {', '.join(missing)} column; {family.name} keeps this "+ f"partition as it stands. Re-run the {NN_EXAMPLES_TABLE} asset for dt={partition_key}"+ )return {"skipped": dagster.MetadataValue.bool(True)}day = _day_inputs(context, client, bucket, prefix, partition_key)if day is None:return {"skipped": dagster.MetadataValue.bool(True)}- examples = examples_from_arrow(arrow)- inputs = network_inputs(examples.table, day.extras, day.labels, day.embeddings)+ inputs = network_inputs(examples.table, day.extras, day.labels, day.embeddings, family.feature_set)del daycheck_recipe(client,bucket,prefix,- NN_FAMILY.name,+ family.name,partition_key,- network_recipe(NETWORK_CONFIG, network_vector_kinds(inputs)),+ network_recipe(family, NETWORK_CONFIG, network_vector_kinds(inputs)),)from products.signals.dags.inbox_ranking.nn.train import ( # noqa: PLC0415 — keeps torch off the location's import path@@ -214,32 +229,35 @@ def train_candidate(trained = train_network(examples.table, inputs, NN_HEADS, NETWORK_CONFIG)for head in trained.heads:- context.log.info(f"nn {head.metrics.head}: {head.metrics.as_dict()}")+ context.log.info(f"{family.name} {head.metrics.head}: {head.metrics.as_dict()}")for name in trained.skipped:- context.log.warning(f"nn {name}: nothing to fit, skipped")+ context.log.warning(f"{family.name} {name}: nothing to fit, skipped")files = {ONNX_MODEL_FILE: trained.model_onnx, HOLDOUT_MODEL_FILE: trained.holdout_model_onnx}written: set[str] = set()for filename, body in files.items():if body is None:continue- key = model_object_key(prefix, NN_FAMILY.name, partition_key, filename)+ key = model_object_key(prefix, family.name, partition_key, filename)client.put_object(Bucket=bucket, Key=key, Body=body, ContentType="application/octet-stream")written.add(key)metadata = candidate_metadata(partition_key,trained,+ family=family,config=NETWORK_CONFIG,windows=examples.windows,trained_at=datetime.datetime.now(datetime.UTC),run_id=context.run_id,)- metadata_key = model_object_key(prefix, NN_FAMILY.name, partition_key, METADATA_FILE)+ metadata_key = model_object_key(prefix, family.name, partition_key, METADATA_FILE)_put_json(client, bucket, metadata_key, metadata)written.add(metadata_key)- stale = _delete_other_objects(client, bucket, model_object_key(prefix, NN_FAMILY.name, partition_key, ""), written)+ stale = _delete_other_objects(client, bucket, model_object_key(prefix, family.name, partition_key, ""), written)if stale:- context.log.warning(f"removed {len(stale)} stale nn objects from a previous run of dt={partition_key}")+ context.log.warning(+ f"removed {len(stale)} stale {family.name} objects from a previous run of dt={partition_key}"+ )capture_training_events(context,partition_key,@@ -248,7 +266,7 @@ def train_candidate(*holdout_calibration_events(partition_key=partition_key,run_id=context.run_id,- model_name=NN_FAMILY.name,+ model_name=family.name,rows=[row for head in trained.heads for row in bucket_rows(head.calibration, {"head": head.metrics.head})],@@ -277,64 +295,113 @@ def train_candidate(}[email protected](name=NN_CHAMPION_ASSET, deps=[NN_CANDIDATE_ASSET], **NN_ASSET_KWARGS)-def inbox_ranking_nn_model_champion(context: dagster.AssetExecutionContext) -> None:- if skip_unconfigured(context):- return- metadata = decide_champion(context, s3_client(), dataset_bucket(), context.partition_key)- context.add_output_metadata(metadata or {"skipped": dagster.MetadataValue.bool(True)})--def decide_champion(- context: dagster.AssetExecutionContext, client, bucket: str, partition_key: str+ context: dagster.AssetExecutionContext, client, bucket: str, partition_key: str, family: ModelFamily) -> dict[str, dagster.MetadataValue] | None:- """The `nn` promotion decision, through the rule every family shares. Nothing is written: the- decision is recorded for the person who runs `inbox_ranking_promote_job`."""+ """The `family` promotion decision, through the rule every family shares. Nothing is written:+ the decision is recorded for the person who runs `inbox_ranking_promote_job`."""prefix = settings.INBOX_RANKING_DATASET_S3_PREFIXdef grade_champion(champion: Mapping[str, Any]) -> dict[str, HoldoutGrade]:version = champion.get("model_version")if champion.get("model_kind") != ONNX_MODEL_KIND:context.log.warning(- f"served nn model {version} is not an {ONNX_MODEL_KIND} model; compared on its stored AUC"+ f"served {family.name} model {version} is not an {ONNX_MODEL_KIND} model; compared on its stored AUC")return {}holdout_model = _read_bytes_if_exists(- client, bucket, model_object_key(prefix, NN_FAMILY.name, str(version), HOLDOUT_MODEL_FILE)+ client, bucket, model_object_key(prefix, family.name, str(version), HOLDOUT_MODEL_FILE))arrow = read_parquet_if_exists(client, bucket, partition_object_key(prefix, NN_EXAMPLES_TABLE, partition_key))if holdout_model is None or arrow is None:context.log.warning(- f"no served nn {HOLDOUT_MODEL_FILE} or no examples for dt={partition_key}; "- "the served nn model is compared on its stored AUC"+ f"no served {family.name} {HOLDOUT_MODEL_FILE} or no examples for dt={partition_key}; "+ f"the served {family.name} model is compared on its stored AUC"+ )+ return {}+ table = examples_from_arrow(arrow).table+ missing = _missing_state_columns(family, table)+ if missing:+ context.log.warning(+ f"the nn examples for dt={partition_key} have no {', '.join(missing)} column; "+ f"the served {family.name} model is compared on its stored AUC")return {}day = _day_inputs(context, client, bucket, prefix, partition_key)if day is None:return {}- table = examples_from_arrow(arrow).tableholdout = subset(table, np.flatnonzero(table.holdout.any(axis=1)))return champion_grades(- champion, holdout_model, holdout, network_inputs(holdout, day.extras, day.labels, day.embeddings)+ champion,+ holdout_model,+ holdout,+ network_inputs(holdout, day.extras, day.labels, day.embeddings, family.feature_set),)- return decide_family_champion(- context, client, bucket, prefix, partition_key, NN_FAMILY, grade_champion=grade_champion- )+ return decide_family_champion(context, client, bucket, prefix, partition_key, family, grade_champion=grade_champion)+++@frozen+class FamilyAssets:+ candidate: dagster.AssetsDefinition+ champion: dagster.AssetsDefinition+++def family_assets(+ family: ModelFamily, *, candidate_name: str, champion_name: str, after: Sequence[str] = ()+) -> FamilyAssets:+ """The candidate and champion assets of one `nn` family. `after` holds the assets the candidate+ waits for, for memory and not for data."""++ @dagster.asset(name=candidate_name, deps=[NN_EXAMPLES_TABLE, *after], **NN_ASSET_KWARGS)+ def candidate(context: dagster.AssetExecutionContext) -> None:+ if skip_unconfigured(context):+ return+ context.add_output_metadata(+ train_candidate(context, s3_client(), dataset_bucket(), context.partition_key, family)+ )++ @dagster.asset(name=champion_name, deps=[candidate_name], **NN_ASSET_KWARGS)+ def champion(context: dagster.AssetExecutionContext) -> None:+ if skip_unconfigured(context):+ return+ metadata = decide_champion(context, s3_client(), dataset_bucket(), context.partition_key, family)+ context.add_output_metadata(metadata or {"skipped": dagster.MetadataValue.bool(True)})++ return FamilyAssets(candidate=candidate, champion=champion)+++_NN_ASSETS = family_assets(NN_FAMILY, candidate_name=NN_CANDIDATE_ASSET, champion_name=NN_CHAMPION_ASSET)+inbox_ranking_nn_model_candidate = _NN_ASSETS.candidate+inbox_ranking_nn_model_champion = _NN_ASSETS.champion+_NN_WITH_PRIO_ASSETS = family_assets(+ NN_WITH_PRIO_FAMILY,+ candidate_name=NN_WITH_PRIO_CANDIDATE_ASSET,+ champion_name=NN_WITH_PRIO_CHAMPION_ASSET,+ after=[NN_CHAMPION_ASSET],+)+inbox_ranking_nn_with_prio_model_candidate = _NN_WITH_PRIO_ASSETS.candidate+inbox_ranking_nn_with_prio_model_champion = _NN_WITH_PRIO_ASSETS.championinbox_ranking_nn_training_job = dagster.define_asset_job(name="inbox_ranking_nn_training_job",- selection=[NN_EXAMPLES_TABLE, NN_CANDIDATE_ASSET, NN_CHAMPION_ASSET],+ selection=[+ NN_EXAMPLES_TABLE,+ NN_CANDIDATE_ASSET,+ NN_CHAMPION_ASSET,+ NN_WITH_PRIO_CANDIDATE_ASSET,+ NN_WITH_PRIO_CHAMPION_ASSET,+ ],tags={**owner_tags,- # Five network fits per run (the train-only fit, NULL_PERMUTATIONS permuted-label fits and the- # refit that ships), on CPU.+ # Five network fits per family (the train-only fit, NULL_PERMUTATIONS permuted-label fits and+ # the refit that ships), on CPU, one family after the other."dagster/max_runtime": str(3 * 60 * 60),# The examples asset holds the lookback window's snapshots in pandas, as the main training job- # does. The candidate holds one embeddings snapshot twice (as Arrow for the project vectors+ # does. A candidate holds one embeddings snapshot twice (as Arrow for the project vectors# and as the report side input), the report matrix, and one set of project vectors per- # (team, day).+ # (team, day). The candidates run in sequence, so the limit covers one of them."dagster-k8s/config": {"container_config": {"resources": {diff --git a/products/signals/dags/inbox_ranking/nn/examples.py b/products/signals/dags/inbox_ranking/nn/examples.pyindex 6044802bd3e8..346c1defb77d 100644--- a/products/signals/dags/inbox_ranking/nn/examples.py+++ b/products/signals/dags/inbox_ranking/nn/examples.py@@ -5,10 +5,11 @@ and a holdout flag for each of `open`, `action` and `fixed`. Each head's moments`example_moments` over the `report_embeddings` set and are capped by `cap_examples`, so every head'scohort, horizon, row budget and holdout are the ones its `report_embeddings` champion is graded on.-The examples object holds the rows, labels and holdout flags only. The vectors are rebuilt from the-dt=D embeddings and labels snapshots by each asset that reads the object: the report vectors-carry 1536 floats per row and the project vectors four times that per (team, day), and both are a-pure function of those two snapshots.+The examples object holds the rows, labels and holdout flags only, and each row's `priority` as of+its own snapshot. One object serves both `nn` families. The vectors are rebuilt from the dt=D+embeddings and labels snapshots by each asset that reads the object: the report vectors carry 1536+floats per row and the project vectors four times that per (team, day), and both are a pure function+of those two snapshots."""import json@@ -21,7 +22,7 @@ import pyarrow as pafrom posthog.dataclasses import frozen-from products.signals.backend.ranking.features import Extras+from products.signals.backend.ranking.features import PRIORITY_COLUMN, Extras, FeatureSetfrom products.signals.dags.inbox_ranking.multihead.inputs import NetworkInputsfrom products.signals.dags.inbox_ranking.multihead.prototype import (FEATURE_SET,@@ -88,14 +89,22 @@ def examples_arrow(examples: NNExamples) -> pa.Table:frame[f"{name}_label"] = table.labels[:, index].astype(np.float32)frame[f"{name}_holdout"] = table.holdout[:, index]arrow = pa.Table.from_pandas(frame, preserve_index=False)+ if PRIORITY_COLUMN in table.rows:+ # Typed explicitly: a day where no row has a judgment would otherwise write a null column.+ arrow = arrow.append_column(+ pa.field(PRIORITY_COLUMN, pa.string()), pa.array(table.rows[PRIORITY_COLUMN].tolist(), type=pa.string())+ )return arrow.replace_schema_metadata({**(arrow.schema.metadata or {}), EXAMPLE_WINDOWS_METADATA_KEY: json.dumps(examples.windows).encode()})def examples_from_arrow(arrow: pa.Table) -> NNExamples:+ """The examples an `examples_arrow` table holds. An object written before the `priority` column+ existed gives rows without it, so a family that reads priority can tell and skip."""frame = arrow.to_pandas()- rows = frame[list(_ROW_COLUMNS)].reset_index(drop=True)+ row_columns = [*_ROW_COLUMNS, *([PRIORITY_COLUMN] if PRIORITY_COLUMN in frame else [])]+ rows = frame[row_columns].reset_index(drop=True)labels = np.column_stack([frame[f"{name}_label"].to_numpy(dtype=np.float32) for name in NN_HEAD_NAMES])holdout = np.column_stack([frame[f"{name}_holdout"].to_numpy(dtype=bool) for name in NN_HEAD_NAMES])raw = (arrow.schema.metadata or {}).get(EXAMPLE_WINDOWS_METADATA_KEY)@@ -109,9 +118,13 @@ def examples_from_arrow(arrow: pa.Table) -> NNExamples:)-def network_inputs(table: ExampleTable, extras: Extras, labels: pa.Table, embeddings: pa.Table) -> NetworkInputs:- """The report vector of each row as of its moment, and its team's project vectors as of the- start of its snapshot day, the ones serving would read for a report born that day."""+def network_inputs(+ table: ExampleTable, extras: Extras, labels: pa.Table, embeddings: pa.Table, feature_set: FeatureSet = FEATURE_SET+) -> NetworkInputs:+ """The report input of each row as of its moment, built by `feature_set`, and its team's project+ vectors as of the start of its snapshot day, the ones serving would read for a report born that+ day."""return NetworkInputs(- reports=report_matrix(table.rows, extras), context=project_context(table.rows, labels, embeddings)+ reports=report_matrix(table.rows, extras, feature_set),+ context=project_context(table.rows, labels, embeddings),)diff --git a/products/signals/dags/inbox_ranking/nn/model.py b/products/signals/dags/inbox_ranking/nn/model.pyindex 4f731a8897be..0b1ab31fe931 100644--- a/products/signals/dags/inbox_ranking/nn/model.py+++ b/products/signals/dags/inbox_ranking/nn/model.py@@ -1,6 +1,6 @@-"""The `nn` family's model record and its ONNX scoring, without torch.+"""The `nn` families' model record and its ONNX scoring, without torch.-A candidate is two ONNX graphs under `inbox_ranking_models/v1/nn/dt=D/`: `model.onnx`, the refit on+A candidate is two ONNX graphs under `inbox_ranking_models/v1/<model_name>/dt=D/`: `model.onnx`, the refit onevery labeled row that a store serves, and `model.holdout.onnx`, the train-only fit that produced theholdout metrics. A later candidate grades the champion through the second one on its own holdout,as the per-head families grade a champion through `<head>.holdout.ubj`.@@ -21,13 +21,13 @@ from products.signals.backend.ranking.serving_manifest import ONNX_MODEL_FILE, Ofrom products.signals.dags.inbox_ranking.common import DATASET_VERSIONfrom products.signals.dags.inbox_ranking.dataset.project_vectors import VECTOR_KINDSfrom products.signals.dags.inbox_ranking.multihead.inputs import ONNX_INPUTS, ONNX_OUTPUT, NetworkConfig, NetworkInputs-from products.signals.dags.inbox_ranking.multihead.prototype import FEATURE_SET, ExampleTable+from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTablefrom products.signals.dags.inbox_ranking.nn.examples import NN_HEAD_NAMES, NN_HEADSfrom products.signals.dags.inbox_ranking.training.calibration import CalibrationBucketfrom products.signals.dags.inbox_ranking.training.heads import HEADS_BY_NAMEfrom products.signals.dags.inbox_ranking.training.recipe import model_recipe, with_recipefrom products.signals.dags.inbox_ranking.training.train import HeadMetrics, HoldoutGrade, holdout_grade-from products.signals.dags.inbox_ranking.training.unseen import NN_MODEL_NAME+from products.signals.dags.inbox_ranking.training.unseen import ModelFamilyHOLDOUT_MODEL_FILE = "model.holdout.onnx"@@ -59,10 +59,10 @@ def network_vector_kinds(inputs: NetworkInputs) -> tuple[str, ...]:return VECTOR_KINDS if inputs.context is not None else ()-def network_recipe(config: NetworkConfig, vector_kinds: Sequence[str]) -> dict[str, Any]:+def network_recipe(family: ModelFamily, config: NetworkConfig, vector_kinds: Sequence[str]) -> dict[str, Any]:return model_recipe(model_kind=ONNX_MODEL_KIND,- feature_set=FEATURE_SET,+ feature_set=family.feature_set,heads=NN_HEADS,learner_params=dataclasses.asdict(config),project_vector_kinds=vector_kinds,@@ -73,26 +73,28 @@ def candidate_metadata(partition_key: str,trained: TrainedNetwork,*,+ family: ModelFamily,config: NetworkConfig,windows: Mapping[str, Mapping[str, object]],trained_at: datetime.datetime,run_id: str,) -> dict[str, Any]:- """The `metadata.json` of an `nn` candidate, in the layout of the per-head families.+ """The `metadata.json` of a `family` candidate, in the layout of the per-head families.The heads carry the same metric fields as a booster's (`train_head` and this trainer both fill`HeadMetrics`), so the promotion rule, the events and the manifest read every family alike."""+ feature_set = family.feature_setmetadata = {- "model_name": NN_MODEL_NAME,+ "model_name": family.name,"model_version": partition_key,"dataset_version": DATASET_VERSION,"model_kind": ONNX_MODEL_KIND,# The set that builds the `report` input, so the grader and the store check the graph# against it like any other family.- "feature_set": FEATURE_SET.name,- "feature_schema_version": FEATURE_SET.schema_version,- "feature_names": list(FEATURE_SET.feature_names),+ "feature_set": feature_set.name,+ "feature_schema_version": feature_set.schema_version,+ "feature_names": list(feature_set.feature_names),ONNX_OUTPUT_HEADS_KEY: list(trained.output_heads),"project_vector_kinds": list(trained.vector_kinds),"trained_at": trained_at.isoformat(),@@ -116,7 +118,7 @@ def candidate_metadata(],"skipped_heads": list(trained.skipped),}- return with_recipe(metadata, network_recipe(config, trained.vector_kinds))+ return with_recipe(metadata, network_recipe(family, config, trained.vector_kinds))def onnx_probabilities(model: bytes, inputs: NetworkInputs, output_heads: Sequence[str]) -> dict[str, np.ndarray]:diff --git a/products/signals/dags/inbox_ranking/nn/train.py b/products/signals/dags/inbox_ranking/nn/train.pyindex 6e6f46d58ccc..a353361f55d8 100644--- a/products/signals/dags/inbox_ranking/nn/train.py+++ b/products/signals/dags/inbox_ranking/nn/train.py@@ -19,6 +19,7 @@ from sklearn.metrics import roc_auc_scorefrom posthog.dataclasses import frozen+from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONSfrom products.signals.dags.inbox_ranking.dataset.project_vectors import VECTOR_KINDSfrom products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfig, NetworkInputsfrom products.signals.dags.inbox_ranking.multihead.network import MultiHeadNetwork, export_onnx@@ -36,6 +37,7 @@ def _fit(with_context=inputs.context is not None,kinds=len(VECTOR_KINDS),config=config,+ report_features=inputs.reports.shape[1] - EMBEDDING_DIMENSIONS,)network.fit(inputs, labels, rows)return networkdiff --git a/products/signals/dags/inbox_ranking/tests/test_multihead.py b/products/signals/dags/inbox_ranking/tests/test_multihead.pyindex f37861fae0f5..27c39e2bd368 100644--- a/products/signals/dags/inbox_ranking/tests/test_multihead.py+++ b/products/signals/dags/inbox_ranking/tests/test_multihead.py@@ -31,6 +31,8 @@ TEAMS = (2, 3)# One latent outcome per head, each written into its own component of the report vector, so a# network that reads one head's output for another head ranks at chance.HEAD_COMPONENTS = {"open": 0, "action": 1, "fixed": 2}+# Cycled by position rather than drawn, so the random vectors stay the same.+PRIORITIES = ("P0", "P1", "P2", "P3", "P4", None)def _birth(day: int) -> pd.Timestamp:@@ -48,6 +50,7 @@ class _SyntheticInbox:"report_id": f"r-{day}-{index}","born": day,"team": TEAMS[index % len(TEAMS)],+ "priority": PRIORITIES[index % len(PRIORITIES)],**{head: bool(rng.random() < 0.4) for head in HEAD_COMPONENTS},}for day in range(DAYS)@@ -69,6 +72,7 @@ class _SyntheticInbox:"report_age_hours": (day - live["born"]) * 24.0 + 12.0,"signal_count": 1,"report_team_id": live["team"],+ "priority": live["priority"],},index=live.index,)diff --git a/products/signals/dags/inbox_ranking/tests/test_nn.py b/products/signals/dags/inbox_ranking/tests/test_nn.pyindex 7fc04465565e..4907d1fe0b5d 100644--- a/products/signals/dags/inbox_ranking/tests/test_nn.py+++ b/products/signals/dags/inbox_ranking/tests/test_nn.py@@ -20,6 +20,7 @@ from products.signals.backend.ranking.model_contract import model_mismatchfrom products.signals.backend.ranking.model_store import ONNX_OUTPUT_HEADS_KEYfrom products.signals.backend.ranking.overrides import NO_OVERRIDESfrom products.signals.backend.ranking.serving_manifest import (+ CHALLENGER_ROLE,CROSS_FAMILY_ROLE,ONNX_MODEL_FILE,ONNX_MODEL_KIND,@@ -36,6 +37,7 @@ from products.signals.dags.inbox_ranking.nn.examples import (NN_HEAD_NAMES,build_examples,examples_arrow,+ examples_from_arrow,network_inputs,subset,)@@ -64,7 +66,14 @@ from products.signals.dags.inbox_ranking.training.heads import HEADS_BY_NAME, Hefrom products.signals.dags.inbox_ranking.training.recipe import RECIPE_HASH_KEY, RECIPE_KEY, model_recipe, recipe_hashfrom products.signals.dags.inbox_ranking.training.serving import FamilyModels, compose_manifestfrom products.signals.dags.inbox_ranking.training.train import HeadMetrics-from products.signals.dags.inbox_ranking.training.unseen import NN_MODEL_NAME, REPORT_EMBEDDINGS_MODEL_NAME+from products.signals.dags.inbox_ranking.training.unseen import (+ NN_FAMILY,+ NN_MODEL_NAME,+ NN_WITH_PRIO_FAMILY,+ NN_WITH_PRIO_MODEL_NAME,+ REPORT_EMBEDDINGS_MODEL_NAME,+ ModelFamily,+)PREFIX = settings.INBOX_RANKING_DATASET_S3_PREFIXPARTITION = "2026-08-19"@@ -139,9 +148,15 @@ def _head_metrics(head: str) -> HeadMetrics:)-def _nn_metadata(version: str, heads: tuple[str, ...] = ("open",)) -> dict[str, Any]:+def _nn_metadata(version: str, heads: tuple[str, ...] = ("open",), family: ModelFamily = NN_FAMILY) -> dict[str, Any]:return candidate_metadata(- version, _trained_network(heads), config=NetworkConfig(), windows={}, trained_at=NOW, run_id="run-1"+ version,+ _trained_network(heads),+ family=family,+ config=NetworkConfig(),+ windows={},+ trained_at=NOW,+ run_id="run-1",)@@ -218,12 +233,19 @@ def test_the_manifest_serves_an_nn_model_with_the_xgboost_champion_as_a_challengnow=NOW,recent=[_nn_metadata(PARTITION, heads=NN_HEAD_NAMES)],recent_versions=3,+ challengers=[+ # The served family's own newest version is a recent entry, never a challenger.+ _nn_metadata("2026-08-18", heads=NN_HEAD_NAMES),+ _nn_metadata(PARTITION, heads=NN_HEAD_NAMES, family=NN_WITH_PRIO_FAMILY),+ _nn_metadata("2026-08-18", heads=NN_HEAD_NAMES, family=NN_WITH_PRIO_FAMILY),+ ],)assert decision.manifest is not Noneassert [(entry.model_name, entry.model_version, entry.roles) for entry in decision.manifest.models] == [(NN_MODEL_NAME, "2026-08-16", [SERVED_ROLE]),(NN_MODEL_NAME, PARTITION, [RECENT_ROLE]),+ (NN_WITH_PRIO_MODEL_NAME, PARTITION, [CHALLENGER_ROLE]),(REPORT_EMBEDDINGS_MODEL_NAME, "2026-08-15", [CROSS_FAMILY_ROLE]),]served = decision.manifest.served@@ -248,14 +270,17 @@ def test_the_nn_decision_is_graded_against_the_served_nn_model_and_writes_nothinbefore = dict(client.objects)with dagster.build_asset_context(partition_key=PARTITION) as context:- metadata = nn_dag.decide_champion(context, client, "bucket", PARTITION)+ metadata = nn_dag.decide_champion(context, client, "bucket", PARTITION, NN_FAMILY)assert metadata is not Noneassert metadata[f"{NN_MODEL_NAME}_incumbent_champion_version"].value == "2026-08-16"assert client.objects == before-def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_order(monkeypatch, no_side_effects):[email protected]("family", [NN_FAMILY, NN_WITH_PRIO_FAMILY], ids=lambda family: family.name)+def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_order(+ monkeypatch, no_side_effects, family+):pytest.importorskip("torch")onnxruntime = pytest.importorskip("onnxruntime")monkeypatch.setattr(@@ -275,10 +300,11 @@ def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_)with dagster.build_asset_context(partition_key=day) as context:- nn_dag.train_candidate(context, client, "bucket", day)+ nn_dag.train_candidate(context, client, "bucket", day, family)- metadata = json.loads(client.objects[model_object_key(PREFIX, NN_MODEL_NAME, day, METADATA_FILE)])+ metadata = json.loads(client.objects[model_object_key(PREFIX, family.name, day, METADATA_FILE)])assert metadata["model_kind"] == ONNX_MODEL_KIND+ assert metadata["feature_set"] == family.feature_set.nameassert metadata[ONNX_OUTPUT_HEADS_KEY] == list(NN_HEAD_NAMES)assert model_mismatch(metadata) is Noneassert {head["head"] for head in metadata["heads"]} == set(NN_HEAD_NAMES)@@ -286,18 +312,20 @@ def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_(ONNX_MODEL_FILE, HOLDOUT_MODEL_FILE)}served = onnxruntime.InferenceSession(- client.objects[model_object_key(PREFIX, NN_MODEL_NAME, day, ONNX_MODEL_FILE)],+ client.objects[model_object_key(PREFIX, family.name, day, ONNX_MODEL_FILE)],providers=["CPUExecutionProvider"],)assert served.get_outputs()[0].shape[1] == len(metadata[ONNX_OUTPUT_HEADS_KEY])+ report_input = next(node for node in served.get_inputs() if node.name == "report")+ assert report_input.shape[1] == len(family.feature_set.feature_names)# Each head's column of the train-only graph, read through `output_heads`, must reproduce the# holdout AUC recorded for that head. A column read for the wrong head ranks at chance here.table = examples.tableholdout = subset(table, np.flatnonzero(table.holdout.any(axis=1)))probabilities = onnx_probabilities(- client.objects[model_object_key(PREFIX, NN_MODEL_NAME, day, HOLDOUT_MODEL_FILE)],- network_inputs(holdout, extras, labels, embeddings),+ client.objects[model_object_key(PREFIX, family.name, day, HOLDOUT_MODEL_FILE)],+ network_inputs(holdout, extras, labels, embeddings, family.feature_set),metadata[ONNX_OUTPUT_HEADS_KEY],)for head in metadata["heads"]:@@ -308,6 +336,22 @@ def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_assert auc > 0.75+def test_a_priority_family_skips_an_examples_object_written_before_the_priority_column(no_side_effects):+ inbox = _SyntheticInbox()+ examples = build_examples(inbox.snapshots(), report_vectors(inbox.embeddings()), holdout_days=HOLDOUT_DAYS)+ arrow = examples_arrow(examples)+ old_arrow = arrow.drop_columns(["priority"])+ assert "priority" in examples_from_arrow(arrow).table.rows+ day = (D0 + datetime.timedelta(days=DAYS - 1)).isoformat()+ client = _S3({partition_object_key(PREFIX, nn_dag.NN_EXAMPLES_TABLE, day): _parquet(old_arrow)})++ with dagster.build_asset_context(partition_key=day) as context:+ metadata = nn_dag.train_candidate(context, client, "bucket", day, NN_WITH_PRIO_FAMILY)++ assert metadata["skipped"].value is True+ assert not [key for key in client.objects if f"/{NN_WITH_PRIO_MODEL_NAME}/" in key]++def _stored_version(model_name: str, version: str, recipe: dict[str, Any] | None, dataset_version: str = DATASET_VERSION) -> tuple[str, bytes]:@@ -377,9 +421,9 @@ def _action_recipe(head: Head) -> dict[str, Any]:def test_check_recipe_refuses_a_new_recipe_under_an_old_name(monkeypatch, stored, recipe, patch, changed):recipes = {"booster": lambda: booster_recipe(REPORT_EMBEDDINGS_FEATURE_SET),- "network": lambda: network_recipe(NetworkConfig(), ()),- "network_more_epochs": lambda: network_recipe(NetworkConfig(epochs=NetworkConfig().epochs + 5), ()),- "network_with_vectors": lambda: network_recipe(NetworkConfig(), PROJECT_VECTOR_KINDS),+ "network": lambda: network_recipe(NN_FAMILY, NetworkConfig(), ()),+ "network_more_epochs": lambda: network_recipe(NN_FAMILY, NetworkConfig(epochs=NetworkConfig().epochs + 5), ()),+ "network_with_vectors": lambda: network_recipe(NN_FAMILY, NetworkConfig(), PROJECT_VECTOR_KINDS),"action": lambda: _action_recipe(ACTION),"action_without_status_labels": lambda: _action_recipe(dataclasses.replace(ACTION, status_labels=False)),"action_without_label_columns": lambda: _action_recipe(dataclasses.replace(ACTION, label_columns=())),diff --git a/products/signals/dags/inbox_ranking/tests/test_training.py b/products/signals/dags/inbox_ranking/tests/test_training.pyindex dffb4602cbc9..6d351c93e1a5 100644--- a/products/signals/dags/inbox_ranking/tests/test_training.py+++ b/products/signals/dags/inbox_ranking/tests/test_training.py@@ -26,8 +26,10 @@ from products.signals.backend.ranking.features import (EMBEDDING_DIMENSIONS,EMBEDDING_INSERTED_AT_COLUMN,NO_EXTRAS,+ PRIORITY_LEVELS,REPORT_EMBEDDINGS_EXTRA,REPORT_EMBEDDINGS_FEATURE_SET,+ REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,REPORT_GRAIN,SCORING_MOMENT_GRAIN,Extras,@@ -37,6 +39,7 @@ from products.signals.backend.ranking.features import (from products.signals.backend.ranking.model_contract import classification_thresholds, model_mismatchfrom products.signals.backend.ranking.overrides import RankingOverridesfrom products.signals.backend.ranking.serving_manifest import (+ CHALLENGER_ROLE,CROSS_FAMILY_ROLE,DEFAULT_MODEL_KIND,ONNX_MODEL_FILE,@@ -142,6 +145,7 @@ from products.signals.dags.inbox_ranking.training.unseen import (LEGACY_POOL_NAME,MODEL_FAMILIES,NN_MODEL_NAME,+ NN_WITH_PRIO_MODEL_NAME,POOL_NAME,REPORT_EMBEDDINGS_MODEL_NAME,SERVED_SCORES_TABLE,@@ -2016,6 +2020,22 @@ def test_report_embeddings_matrix_puts_the_vector_in_position_order(feature_set)assert matrix.loc["b", ["emb_0", last]].tolist() == [float(EMBEDDING_DIMENSIONS - 1), 0.0]+def test_the_priority_set_puts_one_column_per_level_after_the_vector():+ feature_set = REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET+ rows = _state(["a", "b", "c", "d"]).assign(priority=["P0", "P3", None, "urgent"])+ extras = _report_vectors({report_id: _embedding(first=1.0) for report_id in rows.index}, feature_set=feature_set)++ matrix = feature_set.build_matrix(rows, extras)++ assert list(matrix.columns) == list(feature_set.feature_names)+ assert matrix["emb_0"].tolist() == [1.0] * 4+ levels = matrix[[f"priority_{level}" for level in PRIORITY_LEVELS]].to_numpy().tolist()+ assert levels == [[1, 0, 0, 0, 0], [0, 0, 0, 1, 0], [0] * 5, [0] * 5]+ # A frame without the column must not read as "no priority" on every row.+ with pytest.raises(ValueError, match="priority"):+ feature_set.build_matrix(rows.drop(columns=["priority"]), extras)++@pytest.mark.parametrize("feature_set", EMBEDDING_FEATURE_SETS, ids=EMBEDDING_SET_IDS)@pytest.mark.parametrize("vectors",@@ -2545,14 +2565,14 @@ def _run_serving_manifest(dataset_objects: dict[str, bytes] | None = None,mirror: _AppObjectStore | None = None,dataset_client: _ModelStoreS3 | None = None,+ families: tuple[ModelFamily, ...] = (+ ModelFamily(name=EMBEDDINGS_MODEL_NAME, feature_set=REPORT_EMBEDDINGS_FEATURE_SET),+ ),) -> _FakeClient:partition_key = "2026-08-19"prefix = settings.INBOX_RANKING_DATASET_S3_PREFIXcapture = _patch_capture(monkeypatch, cloud=True, debug=False)- monkeypatch.setattr(- "products.signals.dags.inbox_ranking.training.dag.MODEL_FAMILIES",- (ModelFamily(name=EMBEDDINGS_MODEL_NAME, feature_set=REPORT_EMBEDDINGS_FEATURE_SET),),- )+ monkeypatch.setattr("products.signals.dags.inbox_ranking.training.dag.MODEL_FAMILIES", families)client = dataset_client or (_ModelStoreS3(dataset_objects) if dataset_objects is not None else _serving_dataset_s3(prefix, partition_key))@@ -2663,28 +2683,37 @@ def test_project_vectors_are_published_before_the_manifest_that_names_them(assert mirror.objects == {}-def test_recent_versions_come_from_the_window_and_skip_a_version_missing_its_booster(monkeypatch):+def test_recent_versions_and_challengers_come_from_the_window_and_skip_a_version_missing_its_booster(monkeypatch):# A booster deleted by a re-run would fail the whole publish, and a version older than the- # window would keep a stale model in every pass.+ # window would keep a stale model in every pass. A challenger is one version per other family,+ # from its last 2 days.prefix = settings.INBOX_RANKING_DATASET_S3_PREFIXclient = _serving_dataset_s3(prefix, "2026-08-19")- for version, booster in (("2026-08-18", True), ("2026-08-17", False), ("2026-08-13", True)):- folder = model_object_key(prefix, EMBEDDINGS_MODEL_NAME, version, "")+ for name, version, booster in (+ (EMBEDDINGS_MODEL_NAME, "2026-08-18", True),+ (EMBEDDINGS_MODEL_NAME, "2026-08-17", False),+ (EMBEDDINGS_MODEL_NAME, "2026-08-13", True),+ (NN_WITH_PRIO_MODEL_NAME, "2026-08-19", False),+ (NN_WITH_PRIO_MODEL_NAME, "2026-08-18", True),+ (NN_MODEL_NAME, "2026-08-17", True),+ ):+ folder = model_object_key(prefix, name, version, "")client.objects[folder + METADATA_FILE] = json.dumps(- _serving_metadata(EMBEDDINGS_MODEL_NAME, version, heads=("open",), **_FEATURE_FIELDS)+ _serving_metadata(name, version, heads=("open",), **_FEATURE_FIELDS)).encode()if booster:client.objects[folder + "open.ubj"] = b"booster"monkeypatch.setattr(settings, "INBOX_RANKING_SERVING_RECENT_VERSIONS", 3)store = _AppObjectStore()- _run_serving_manifest(monkeypatch, store, dataset_client=client)+ _run_serving_manifest(monkeypatch, store, dataset_client=client, families=MODEL_FAMILIES)manifest = ServingManifest.model_validate_json(store.objects[serving_manifest_key(prefix)])- assert [(entry.model_version, entry.roles) for entry in manifest.models] == [- ("2026-08-15", [SERVED_ROLE]),- ("2026-08-19", [RECENT_ROLE]),- ("2026-08-18", [RECENT_ROLE]),+ assert [(entry.model_name, entry.model_version, entry.roles) for entry in manifest.models] == [+ (EMBEDDINGS_MODEL_NAME, "2026-08-15", [SERVED_ROLE]),+ (EMBEDDINGS_MODEL_NAME, "2026-08-19", [RECENT_ROLE]),+ (EMBEDDINGS_MODEL_NAME, "2026-08-18", [RECENT_ROLE]),+ (NN_WITH_PRIO_MODEL_NAME, "2026-08-18", [CHALLENGER_ROLE]),]diff --git a/products/signals/dags/inbox_ranking/training/dag.py b/products/signals/dags/inbox_ranking/training/dag.pyindex 5935e290fdbe..ae4196ea54db 100644--- a/products/signals/dags/inbox_ranking/training/dag.py+++ b/products/signals/dags/inbox_ranking/training/dag.py@@ -468,7 +468,10 @@ def inbox_ranking_training_examples(context: dagster.AssetExecutionContext) -> N"excluded_no_training_consent_reports": dagster.MetadataValue.int(excluded.reports),"excluded_no_training_consent_teams": dagster.MetadataValue.int(excluded.teams),}- for feature_set in FEATURE_SETS.values():+ # Only the sets this location trains on. A set only `signals_ml` reads would cost this job a+ # second 1536-column examples object that nothing reads.+ main_sets = {family.feature_set.name: family.feature_set for family in families_trained_in(MAIN_LOCATION)}+ for feature_set in main_sets.values():metadata |= _examples_for_set(context, client, bucket, prefix, partition_key, feature_set, snapshots, backfilled_rows, excluded)@@ -1009,6 +1012,18 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> dif served is not Noneelse [])+ # One version per other family, read from the last 2 days, because a `signals_ml` candidate can+ # land after this asset runs.+ challengers = (+ [+ metadata+ for family in MODEL_FAMILIES+ if family.name != served["model_name"]+ for metadata in _recent_models(context, client, bucket, prefix, family.name, partition_key, 1)+ ]+ if served is not None+ else []+ )decision = (compose_manifest(families,@@ -1017,6 +1032,7 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> dnow=datetime.datetime.now(datetime.UTC),recent=recent,recent_versions=recent_versions,+ challengers=challengers,pinned=pinned,)if served_reason is None@@ -1079,6 +1095,7 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> dfor metadata in (*([served] if served is not None else []),*recent,+ *challengers,*pinned,*(family.champion for family in families if family.champion),)diff --git a/products/signals/dags/inbox_ranking/training/serving.py b/products/signals/dags/inbox_ranking/training/serving.pyindex 92a74e2c0216..0d69e8f86d09 100644--- a/products/signals/dags/inbox_ranking/training/serving.py+++ b/products/signals/dags/inbox_ranking/training/serving.py@@ -8,7 +8,8 @@ The served entry is the model the `served.json` pointer names, so the served modwhen a person runs `inbox_ranking_promote_job`. The other entries are there so one pass produces several scores: an online paired readand a later interleaving then need no rescoring of the same reports. The recent entries keep aversion in the manifest for several days, so its online grade reads more than one day's cohort. A-pinned entry is there so an owner's `served` override can name it.+challenger entry is the newest version of another trained family, so a recipe gets online scores+before anyone serves it. A pinned entry is there so an owner's `served` override can name it."""import datetime@@ -21,6 +22,7 @@ from products.signals.backend.artefact_schemas import MAX_RANKING_MODEL_RESULTSfrom products.signals.backend.ranking.model_contract import readable_head_names, trained_head_filesfrom products.signals.backend.ranking.model_store import ONNX_PROJECT_VECTOR_KINDS_KEYfrom products.signals.backend.ranking.serving_manifest import (+ CHALLENGER_ROLE,CROSS_FAMILY_ROLE,DEFAULT_MODEL_KIND,ONNX_MODEL_FILE,@@ -93,12 +95,15 @@ def compose_manifest(now: datetime.datetime,recent: Sequence[Mapping[str, Any]] = (),recent_versions: int = 0,+ challengers: Sequence[Mapping[str, Any]] = (),pinned: Sequence[Mapping[str, Any]] = (),) -> ManifestDecision:"""`served` holds the metadata of the model the `served.json` pointer names, `recent` the- metadata of the served family's versions newest first, and `pinned` the metadata of the models- the `pin` override keeps. At most `recent_versions` of `recent` get the recent role. The caller- checks that their files exist, because a manifest that names a missing file is never published."""+ metadata of the served family's versions newest first, `challengers` the metadata of the other+ families' versions, each family newest first, and `pinned` the metadata of the models the `pin`+ override keeps. At most `recent_versions` of `recent` get the recent role, and the first readable+ version of each other family gets the challenger role. The caller checks that their files exist,+ because a manifest that names a missing file is never published."""if served is None:return ManifestDecision(manifest=None, reason="no served pointer")served_family = served["model_name"]@@ -132,6 +137,21 @@ def compose_manifest(entries.append(version)kept_recent += 1+ challenged: set[str] = set()+ for metadata in challengers:+ family_name = metadata["model_name"]+ if family_name == served_family or family_name in challenged or not readable_head_names(metadata):+ continue+ challenger = _entry(+ metadata,+ roles=[CHALLENGER_ROLE],+ labels={"reason": f"newest {family_name} version, paired against the served model"},+ prefix=prefix,+ )+ if challenger is not None and challenger.key not in {entry.key for entry in entries}:+ entries.append(challenger)+ challenged.add(family_name)+for metadata in pinned:pin = _entry(metadata,@@ -154,9 +174,10 @@ def compose_manifest(if cross is not None and cross.key not in {entry.key for entry in entries}:entries.append(cross)- # The served entry and the recent versions are the reads the sweep exists for, so the cap- # falls on the pinned and cross-family entries, cross-family first. They are last in- # `entries`, the cross-family ones in family registration order.+ # The served entry, the recent versions and the challengers are the reads the sweep exists for,+ # so the cap falls on the pinned and cross-family entries, cross-family first. They are last in+ # `entries`, the cross-family ones in family registration order. Challengers come in the order+ # the caller gives them, which is family registration order.return ManifestDecision(manifest=ServingManifest(manifest_version=now.isoformat(), models=entries[:MAX_RANKING_MODEL_RESULTS]),reason="composed",diff --git a/products/signals/dags/inbox_ranking/training/unseen.py b/products/signals/dags/inbox_ranking/training/unseen.pyindex 67b1228b4df3..23e8af4e078d 100644--- a/products/signals/dags/inbox_ranking/training/unseen.py+++ b/products/signals/dags/inbox_ranking/training/unseen.py@@ -22,7 +22,11 @@ from sklearn.metrics import roc_auc_scorefrom posthog.dataclasses import frozen-from products.signals.backend.ranking.features import REPORT_EMBEDDINGS_FEATURE_SET, FeatureSet+from products.signals.backend.ranking.features import (+ REPORT_EMBEDDINGS_FEATURE_SET,+ REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,+ FeatureSet,+)from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND, ONNX_MODEL_KINDfrom products.signals.dags.inbox_ranking.training.calibration import (CalibrationBucket,@@ -59,6 +63,7 @@ CANDIDATE_ROLE = "candidate"# servable by key, and their online grades keep their own series.REPORT_EMBEDDINGS_MODEL_NAME = "report_embeddings_v2"NN_MODEL_NAME = "nn_v2"+NN_WITH_PRIO_MODEL_NAME = "nn_with_prio"# The Dagster code locations that train a family: `signals` on the main image, and `signals_ml` on# the image that carries torch.MAIN_LOCATION = "signals"@@ -116,13 +121,21 @@ class ModelFamily:# `nn` is one multi-head network over the report vector and the team's project vectors. It needs# torch, so the `signals_ml` location trains it and the main location never fits it. It declares# the report embeddings set, because that set builds its report input in training and at serving.+# `nn_with_prio` is the same network with the report's priority levels as extra report inputs.MODEL_FAMILIES: tuple[ModelFamily, ...] = (ModelFamily(name=REPORT_EMBEDDINGS_MODEL_NAME, feature_set=REPORT_EMBEDDINGS_FEATURE_SET),ModelFamily(name=NN_MODEL_NAME, feature_set=REPORT_EMBEDDINGS_FEATURE_SET, learner=ONNX_MODEL_KIND, location=ML_LOCATION),+ ModelFamily(+ name=NN_WITH_PRIO_MODEL_NAME,+ feature_set=REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,+ learner=ONNX_MODEL_KIND,+ location=ML_LOCATION,+ ),)NN_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_MODEL_NAME)+NN_WITH_PRIO_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_WITH_PRIO_MODEL_NAME)def families_trained_in(location: str) -> tuple[ModelFamily, ...]: