:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): mirror inbox ranking manifests with the mirror region's own project vectors (#121431)
posthog[bot]b9ea2ea47e88
diff --git a/products/signals/backend/ranking/serving_manifest.py b/products/signals/backend/ranking/serving_manifest.pyindex 8531bf325ec0..8add20bc0eb0 100644--- a/products/signals/backend/ranking/serving_manifest.py+++ b/products/signals/backend/ranking/serving_manifest.py@@ -14,6 +14,8 @@ not act on is refused by the dag that composed it.from __future__ import annotations+import datetime+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validatorfrom products.signals.backend.artefact_schemas import MAX_RANKING_MODEL_RESULTS, RANKING_SERVED_ROLE@@ -55,6 +57,12 @@ ENGAGEMENT_FILE = "engagement.parquet"_SERVING_SEGMENT = "serving"+# The object metadata a mirror region stamps on its own project vectors day. The region that trains+# reads only these, through `head_object`, before its mirror manifest names the day. The schema+# version key is the one `write_parquet` stamps.+PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY = "feature-schema-version"+PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY = "engagement-export-date"+def serving_manifest_key(prefix: str) -> str:return f"{prefix}/{_SERVING_SEGMENT}/{MANIFEST_FILE}"@@ -78,6 +86,17 @@ def serving_engagement_key(prefix: str, partition_key: str) -> str:return f"{prefix}/{_SERVING_SEGMENT}/engagement/{partition_key}/{ENGAGEMENT_FILE}"+def serving_project_vectors_date(key: str) -> datetime.date | None:+ """The `dt` of a `serving_project_vectors_key`, or None when the key has another shape."""+ parts = key.rsplit("/", 2)+ if len(parts) != 3 or parts[2] != PROJECT_VECTORS_FILE:+ return None+ try:+ return datetime.date.fromisoformat(parts[1])+ except ValueError:+ return None++def model_key(model_name: str, model_version: str) -> str:return f"{model_name}@{model_version}"diff --git a/products/signals/backend/ranking/sweep.py b/products/signals/backend/ranking/sweep.pyindex 71736a14741c..1db1e7639911 100644--- a/products/signals/backend/ranking/sweep.py+++ b/products/signals/backend/ranking/sweep.py@@ -40,6 +40,7 @@ from products.signals.backend.ranking import scorerfrom products.signals.backend.ranking.inventory import SCORABLE_STATUSES, spine_report_filterfrom products.signals.backend.ranking.model_store import ModelLoadError, ServingSet, load_serving_setfrom products.signals.backend.ranking.overrides import RankingOverrides, read_ranking_overrides+from products.signals.backend.ranking.serving_manifest import serving_project_vectors_datefrom products.signals.backend.report_embedding_reader import REPORT_EMBEDDINGS_TABLEfrom products.signals.backend.report_embeddings import EMBEDDING_DOCUMENT_TYPE, EMBEDDING_PRODUCT@@ -344,6 +345,12 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult:scored += sum(1 for outcome in outcomes if outcome.score is not None)no_vector += sum(1 for outcome in outcomes if outcome.reason == scorer.NO_VECTOR)+ # The day the pass read, so a stale mirror region day shows in the log.+ vectors_date = (+ serving_project_vectors_date(serving.manifest.project_vectors)+ if project_vectors is not None and project_vectors.available and serving.manifest.project_vectors+ else None+ )result = ScoreInboxReportsResult(candidates=len(candidates),scored=scored,@@ -364,6 +371,8 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult:deferred_teams=result.deferred_teams,deferred_reports=result.deferred_reports,manifest_version=result.manifest_version,+ project_vectors_date=vectors_date.isoformat() if vectors_date else None,+ project_vectors_age_days=(now.date() - vectors_date).days if vectors_date else None,override_served=override.served if override else None,override_expires_at=override.expires_at.isoformat() if override and override.expires_at else None,peak_rss_mb=_peak_rss_mb(),diff --git a/products/signals/backend/test/test_ranking_sweep.py b/products/signals/backend/test/test_ranking_sweep.pyindex 25edf3601436..fbf8535d50a8 100644--- a/products/signals/backend/test/test_ranking_sweep.py+++ b/products/signals/backend/test/test_ranking_sweep.py@@ -203,7 +203,9 @@ class TestScoreInboxReports(SimpleTestCase):def setUp(self) -> None:super().setUp()self.serving = SimpleNamespace(- manifest=SimpleNamespace(manifest_version=MANIFEST, served=SimpleNamespace(key=SERVED_KEY)),+ manifest=SimpleNamespace(+ manifest_version=MANIFEST, served=SimpleNamespace(key=SERVED_KEY), project_vectors=None+ ),served_override=None,)self._patch(sweep, "read_ranking_overrides", return_value=NO_OVERRIDES)diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.mdindex d3ed5496677f..e4d99ee978ba 100644--- a/products/signals/dags/inbox_ranking/AGENTS.md+++ b/products/signals/dags/inbox_ranking/AGENTS.md@@ -38,7 +38,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f- The manifest schema and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, next to the feature contract, because the dag writes them and the sweep reads them. Its validators run at write time, so the dag refuses a manifest the sweep could not act on; the entry cap is the `ranking_score` artefact's, so a manifest can never produce a row the artefact refuses.- The `recent` role keeps the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` versions of the served `model_name`, read from the last 2N partitions, so each version gets several days of online scores. The `challenger` role keeps the newest readable version of each other serving family (`serving_families()`), read from its last 2 partitions. Each one passes the pin check before composition, because one missing file fails the publish. Keep the order under the cap as served, recent, challenger, pinned, cross-family. Each recent entry is one more model per sweep pass, so measure the sweep's duration, peak RSS and scored and deferred counts on the `inbox_ranking_sweep_finished` log before and after a change to N.- Publishing order is load-bearing. Every model copy finishes before `manifest.json` is written, so a failed copy leaves the previous manifest and the previous models serving. A model version is immutable, so a version whose `metadata.json` is already at the target is not copied again. A run with nothing to publish, such as a run without `served.json`, writes nothing rather than an empty manifest. The served model passes the same check as a pin (`_pin_metadata`) before composition.-- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror: they are per-team data of this deployment, so the mirror's manifest names none. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.+- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror, and never read the body of a mirror object: they are per-team data of the deployment that owns the teams. The mirror's manifest names the mirror region's own day only after `_mirror_project_vectors` checks its object metadata (`PROJECT_VECTORS_*_METADATA_KEY` in `serving_manifest.py`) within `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS`. Without such a day it names none, and a served model that reads them keeps the mirror on its previous manifest. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.- `HEADS` holds only the heads the inbox reads (`HeadRanker` in `backend/ranking/inbox_sort.py`). Add a head there only together with a surface that reads it. The promotion gate skips a champion head that is not in `HEADS`, so a trim does not block promotion.- `nn_v3` and `nn_with_prio_v2` are trained by the `signals_ml` location (`nn/dag.py`), never by the main one. They share one `inbox_ranking_nn_examples` object, and each has its own candidate and champion asset from `family_assets`. The `nn_v3` assets keep the names `inbox_ranking_nn_model_{candidate,champion}`, so their Dagster history continues. A later family's candidate depends on the previous family's champion, for memory and not for data: two candidates at once can exceed the job's memory limit. A family whose set reads a state column the examples object lacks skips with a warning rather than training on zeros. torch is imported only inside the candidate asset's body (through `nn/train.py` and `multihead/network.py`). `signals_ml` must import without torch, and `test_multihead_network.py` pins that. Its decision goes through `decide_family_champion` in `training/dag.py`, graded through the incumbent's `model.holdout.onnx`. Keep `signals_ml` commented out in `.dagster_home/workspace.yaml`, and keep its schedule (04:30 UTC) running by default on prod US only.diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.mdindex c13ef44aed63..2f9a41359ddb 100644--- a/products/signals/dags/inbox_ranking/README.md+++ b/products/signals/dags/inbox_ranking/README.md@@ -223,6 +223,14 @@ The manifest holds one entry per model with its `roles`: `served` for the modelA region that does not train serves the same models through a mirror. When `INBOX_RANKING_SERVING_MIRROR_BUCKET` is set, the asset repeats the publish into that bucket after the primary publish succeeds, in the same order and with the same skip of versions already there. A mirror failure does not fail the asset: it logs, and the `inbox_ranking_serving_manifest_published` event carries `mirror_published=false`. The property is absent when no mirror is set.+Project vectors are per-team data, so this region never copies its own day to the mirror. When a manifest entry reads them, the asset looks for the mirror region's own day in the mirror bucket instead:++- It calls `head_object` on `serving_project_vectors_key(prefix, D')` for each day D' from D back to D − `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS` (3), newest first. The mirror grant has no list permission, so a missing key answers 403, and both read as absent. It reads only the object metadata, never the body.+- A day passes when its `feature-schema-version` metadata is this build's `PROJECT_VECTORS_SCHEMA_VERSION` and its `engagement-export-date` metadata is no older than D − 3. The key names live in `serving_manifest.py`, and the mirror region's writer uses the same ones.+- When a day passes, the mirror manifest names it in `project_vectors`, and the publish copies no project vectors object. The mirror region builds day D after the 05:00 UTC publish, so a mirror that keeps up names D−1, one day older than the primary.+- When no day passes and the served model reads project vectors, the mirror keeps its previous manifest. When only other entries read them, the mirror manifest names none, and the mirror sweep skips those entries.+- The event carries `mirror_project_vectors_key`, `mirror_project_vectors_date` and `mirror_engagement_export_date`, and `mirror_skip_reason` when the mirror is not published: `no_fresh_project_vectors`, `unknown_schema_version`, `stale_engagement_export` or `publish_failed`. The `inbox_ranking_sweep_finished` log carries the `project_vectors_date` the pass read and its `project_vectors_age_days`.+The reader is `score_reports` in `products/signals/backend/ranking/scorer.py`. It loads every model the manifest names through `model_store.py`, reads each report's latest vector once per rendering, builds each model's matrix with its own `FeatureSet.build_matrix`, and writes one `ranking_score` artefact per report plus one `inbox_ranking_report_scored` event per report and model. The dag and the reader decide whether a model can be scored with the same functions, in `products/signals/backend/ranking/model_contract.py`. A served model that cannot load stops the pass. Any other model that cannot load, or has no vector for a report, is a `skipped` result. No scheduled caller runs the reader yet.### Overridesdiff --git a/products/signals/dags/inbox_ranking/mirror/dag.py b/products/signals/dags/inbox_ranking/mirror/dag.pyindex 2f9c76a01353..d447ec1fadf4 100644--- a/products/signals/dags/inbox_ranking/mirror/dag.py+++ b/products/signals/dags/inbox_ranking/mirror/dag.py@@ -24,7 +24,12 @@ from posthog.dataclasses import frozenfrom posthog.storage import object_storagefrom posthog.storage.object_storage import ObjectStorageClient-from products.signals.backend.ranking.serving_manifest import serving_engagement_key, serving_project_vectors_key+from products.signals.backend.ranking.serving_manifest import (+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY,+ serving_engagement_key,+ serving_project_vectors_key,+)from products.signals.backend.report_embeddings import EMBEDDING_RENDERING_TITLE_SUMMARYfrom products.signals.dags.inbox_ranking.common import (ROW_COUNT_METADATA_KEY,@@ -52,7 +57,6 @@ MIRROR_PROJECT_VECTORS_ASSET = "inbox_mirror_project_vectors"# An older export only misses the newest engagement, so a late training-region run costs a little# history. Past this age the vectors drift too far from what training read, so the asset fails.ENGAGEMENT_EXPORT_MAX_AGE_DAYS = 2-ENGAGEMENT_EXPORT_DATE_METADATA_KEY = "engagement-export-date"@frozen@@ -131,8 +135,8 @@ def inbox_mirror_project_vectors(context: dagster.AssetExecutionContext) -> None"Metadata": {ROW_COUNT_METADATA_KEY: str(table.num_rows),SNAPSHOT_DATE_METADATA_KEY: partition_key,- SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),- ENGAGEMENT_EXPORT_DATE_METADATA_KEY: export.export_date,+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY: export.export_date,}},)diff --git a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py b/products/signals/dags/inbox_ranking/tests/test_project_vectors.pyindex 830e93fb6724..6f776cee9ca1 100644--- a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py+++ b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py@@ -14,6 +14,7 @@ from parameterized import parameterizedfrom products.signals.backend.ranking import model_storefrom products.signals.backend.ranking.features import EMBEDDING_DIMENSIONSfrom products.signals.backend.ranking.serving_manifest import (+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,SERVED_ROLE,ServingManifest,ServingManifestEntry,@@ -33,6 +34,7 @@ from products.signals.dags.inbox_ranking.dataset.project_vectors import ()from products.signals.dags.inbox_ranking.mirror import dag as mirror_dagfrom products.signals.dags.inbox_ranking.multihead.prototype import project_context+from products.signals.dags.inbox_ranking.training import dag as training_dagAS_OF = datetime.datetime(2026, 7, 30, tzinfo=datetime.UTC)BEFORE = AS_OF - datetime.timedelta(days=3)@@ -357,8 +359,11 @@ def test_the_mirror_region_builds_its_vectors_from_its_embeddings_and_the_exportcommon.ROW_COUNT_METADATA_KEY: str(written.num_rows),common.SNAPSHOT_DATE_METADATA_KEY: partition_day.isoformat(),common.SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),- mirror_dag.ENGAGEMENT_EXPORT_DATE_METADATA_KEY: export_date,+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY: export_date,}+ # The training region's mirror lookup reads only this metadata, so a key drift drops every day.+ oldest = partition_day - datetime.timedelta(days=training_dag.MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS)+ assert training_dag._project_vectors_refusal(write["extras"]["Metadata"], oldest) is Noneassert written.schema == PROJECT_VECTORS_SCHEMArows = _rows_by_key(written)assert {team_id for team_id, _ in rows} == {5}diff --git a/products/signals/dags/inbox_ranking/tests/test_training.py b/products/signals/dags/inbox_ranking/tests/test_training.pyindex ea0f3cf30917..f30b3bcc0b35 100644--- a/products/signals/dags/inbox_ranking/tests/test_training.py+++ b/products/signals/dags/inbox_ranking/tests/test_training.py@@ -45,6 +45,8 @@ from products.signals.backend.ranking.serving_manifest import (ONNX_MODEL_FILE,ONNX_MODEL_KIND,PINNED_ROLE,+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY,RECENT_ROLE,SERVED_ROLE,ServingManifest,@@ -58,6 +60,7 @@ from products.signals.backend.ranking.serving_manifest import (from products.signals.backend.ranking.sinks import REPORT_SCORED_EVENTfrom products.signals.dags.inbox_ranking.common import partition_object_keyfrom products.signals.dags.inbox_ranking.dataset.dag import LABELS_TABLE, PROJECT_VECTORS_TABLE, STATE_TABLE+from products.signals.dags.inbox_ranking.dataset.project_vectors import PROJECT_VECTORS_SCHEMA_VERSIONfrom products.signals.dags.inbox_ranking.training.calibration import (BUCKETS,calibration_buckets,@@ -67,6 +70,9 @@ from products.signals.dags.inbox_ranking.training.classification import classififrom products.signals.dags.inbox_ranking.training.dag import (_STATE_READ_COLUMNS,METADATA_FILE,+ NO_FRESH_PROJECT_VECTORS,+ STALE_ENGAGEMENT_EXPORT,+ UNKNOWN_SCHEMA_VERSION,_delete_other_objects,_pinned_models,_publish_manifest,@@ -2157,14 +2163,22 @@ def test_a_model_published_before_model_kind_existed_reads_as_xgboost():class _AppObjectStore:"""The deployment's object store, the one the Temporal workers can reach."""- def __init__(self, existing: dict[str, bytes] | None = None, fail_on: str | None = None):+ def __init__(+ self,+ existing: dict[str, bytes] | None = None,+ fail_on: str | None = None,+ metadata: dict[str, dict[str, str]] | None = None,+ ):self.objects = dict(existing or {})+ self.metadata = dict(metadata or {})self.fail_on = fail_onself.written: list[str] = []self.extras: dict[str, dict[str, str] | None] = {}- def head_object(self, bucket: str, file_key: str) -> dict[str, int] | None:- return {"ContentLength": len(self.objects[file_key])} if file_key in self.objects else None+ def head_object(self, bucket: str, file_key: str) -> dict[str, Any] | None:+ if file_key not in self.objects:+ return None+ return {"ContentLength": len(self.objects[file_key]), "Metadata": self.metadata.get(file_key, {})}def write(self, bucket: str, key: str, content: str | bytes, extras: dict[str, str] | None) -> None:if self.fail_on is not None and self.fail_on in key:@@ -2336,6 +2350,91 @@ def test_project_vectors_are_published_before_the_manifest_that_names_them(assert mirror.objects == {}+_MIRROR_PARTITION = "2026-08-19"+++def _mirror_day(offset: int) -> str:+ return (datetime.date.fromisoformat(_MIRROR_PARTITION) - datetime.timedelta(days=offset)).isoformat()+++def _mirror_vectors_metadata(engagement_offset: int, schema_version: int = PROJECT_VECTORS_SCHEMA_VERSION):+ return {+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(schema_version),+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY: _mirror_day(engagement_offset),+ }+++ "mirror_days,expected_offset,skip_reason",+ [+ ({0: _mirror_vectors_metadata(1)}, 0, None),+ ({1: _mirror_vectors_metadata(2)}, 1, None),+ (+ {+ 0: _mirror_vectors_metadata(1, schema_version=PROJECT_VECTORS_SCHEMA_VERSION + 1),+ 1: _mirror_vectors_metadata(2),+ },+ 1,+ None,+ ),+ ({}, None, NO_FRESH_PROJECT_VECTORS),+ ({4: _mirror_vectors_metadata(5)}, None, NO_FRESH_PROJECT_VECTORS),+ (+ {0: _mirror_vectors_metadata(1, schema_version=PROJECT_VECTORS_SCHEMA_VERSION + 1)},+ None,+ UNKNOWN_SCHEMA_VERSION,+ ),+ ({0: _mirror_vectors_metadata(4)}, None, STALE_ENGAGEMENT_EXPORT),+ (+ {0: {PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION)}},+ None,+ STALE_ENGAGEMENT_EXPORT,+ ),+ ],+)+def test_the_mirror_manifest_names_the_mirror_regions_own_fresh_project_vectors(+ monkeypatch: pytest.MonkeyPatch,+ mirror_days: dict[int, dict[str, str]],+ expected_offset: int | None,+ skip_reason: str | None,+) -> None:+ prefix = settings.INBOX_RANKING_DATASET_S3_PREFIX+ mirror_keys = {offset: serving_project_vectors_key(prefix, _mirror_day(offset)) for offset in mirror_days}+ # The fake has no read method, so a publish that read a mirror object body would fail the mirror.+ mirror = _AppObjectStore(+ dict.fromkeys(mirror_keys.values(), b"mirror region vectors"),+ metadata={key: mirror_days[offset] for offset, key in mirror_keys.items()},+ )+ before = dict(mirror.objects)+ dataset = _serving_dataset_s3(+ prefix,+ _MIRROR_PARTITION,+ model_kind=ONNX_MODEL_KIND,+ project_vector_kinds=("typical", "open", "action", "dismiss"),+ )+ dataset.objects[partition_object_key(prefix, PROJECT_VECTORS_TABLE, _MIRROR_PARTITION)] = b"this region vectors"+ capture = _run_serving_manifest(monkeypatch, _AppObjectStore(), mirror=mirror, dataset_client=dataset)++ event = _manifest_event(capture)+ assert b"this region vectors" not in mirror.objects.values()+ assert not any("/project_vectors/" in key for key in mirror.written)+ if expected_offset is None:+ assert mirror.objects == before+ assert event["mirror_published"] is False+ assert event["mirror_skip_reason"] == skip_reason+ assert event["mirror_project_vectors_key"] is None+ return+ manifest = ServingManifest.model_validate_json(mirror.objects[serving_manifest_key(prefix)])+ assert manifest.served.reads_project_vectors+ assert manifest.project_vectors == mirror_keys[expected_offset]+ assert mirror.written[-1] == serving_manifest_key(prefix)+ assert event["mirror_published"] is True+ assert "mirror_skip_reason" not in event+ assert event["mirror_project_vectors_key"] == mirror_keys[expected_offset]+ assert event["mirror_project_vectors_date"] == _mirror_day(expected_offset)+ assert event["mirror_engagement_export_date"] == _mirror_day(expected_offset + 1)++def test_recent_versions_and_challengers_come_from_the_window_and_skip_a_version_missing_its_booster(monkeypatch):# A booster deleted by a re-run would fail the whole publish, and a version older than the# window would keep a stale model in every pass. A challenger is one version per other family,diff --git a/products/signals/dags/inbox_ranking/training/dag.py b/products/signals/dags/inbox_ranking/training/dag.pyindex e94b8ec795c5..6cd8e3bcb438 100644--- a/products/signals/dags/inbox_ranking/training/dag.py+++ b/products/signals/dags/inbox_ranking/training/dag.py@@ -46,6 +46,8 @@ from products.signals.backend.ranking.features import FEATURE_SETSfrom products.signals.backend.ranking.model_contract import model_mismatch, trained_head_filesfrom products.signals.backend.ranking.overrides import read_ranking_overridesfrom products.signals.backend.ranking.serving_manifest import (+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY,ServingManifest,model_key,serving_manifest_key,@@ -66,6 +68,7 @@ from products.signals.dags.inbox_ranking.common import (skip_unconfigured,)from products.signals.dags.inbox_ranking.dataset.dag import LABELS_TABLE, PROJECT_VECTORS_TABLE, STATE_TABLE+from products.signals.dags.inbox_ranking.dataset.project_vectors import PROJECT_VECTORS_SCHEMA_VERSIONfrom products.signals.dags.inbox_ranking.training.examples import (BASE_STATE_COLUMNS,PROVENANCE_LABEL_COLUMNS,@@ -89,6 +92,7 @@ from products.signals.dags.inbox_ranking.training.promotion import decide_promotfrom products.signals.dags.inbox_ranking.training.recipe import RECIPE_HASH_KEY, recipe_mismatchfrom products.signals.dags.inbox_ranking.training.serving import FamilyModels, ManifestDecision, compose_manifestfrom products.signals.dags.inbox_ranking.training.telemetry import (+ MirrorPublication,capture_training_events,promotion_event,serving_manifest_event,@@ -511,7 +515,7 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> dcontext, client, bucket, prefix, manifest, primary, project_vectors_source=project_vectors_source)context.log.info(f"published serving manifest {manifest.manifest_version} serving {manifest.served.key}")- mirror_published = _publish_mirror(context, client, bucket, prefix, manifest)+ mirror = _publish_mirror(context, client, bucket, prefix, partition_key, manifest)capture_training_events(context,partition_key,@@ -525,7 +529,7 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> dcopied_keys=publication.copied,present_keys=publication.present,bytes_copied=publication.bytes_copied,- mirror_published=mirror_published,+ mirror=mirror,recipe_hashes={model_key(metadata["model_name"], metadata["model_version"]): metadata.get(RECIPE_HASH_KEY)for metadata in (@@ -540,7 +544,13 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d],)mirror_metadata = (- {} if mirror_published is None else {"mirror_published": dagster.MetadataValue.bool(mirror_published)}+ {}+ if mirror is None+ else {+ "mirror_published": dagster.MetadataValue.bool(mirror.published),+ "mirror_project_vectors_key": dagster.MetadataValue.text(mirror.project_vectors_key or ""),+ "mirror_skip_reason": dagster.MetadataValue.text(mirror.skip_reason or ""),+ })return {"published": dagster.MetadataValue.bool(True),@@ -675,38 +685,131 @@ def _pin_metadata(client, bucket: str, prefix: str, key: str) -> tuple[dict[str,return metadata, None-def _publish_mirror(context: RunContext, client, bucket: str, prefix: str, manifest: ServingManifest) -> bool | None:+# How many days before the publish partition the mirror region's project vectors day, and the+# engagement export it was built from, may be. The mirror region builds day D after this region's+# 05:00 UTC publish, so a mirror that keeps up names D-1.+MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS = 3++NO_FRESH_PROJECT_VECTORS = "no_fresh_project_vectors"+UNKNOWN_SCHEMA_VERSION = "unknown_schema_version"+STALE_ENGAGEMENT_EXPORT = "stale_engagement_export"+MIRROR_PUBLISH_FAILED = "publish_failed"+++@frozen+class _MirrorProjectVectors:+ """The mirror region's project vectors day the mirror manifest names, or the newest day refused."""++ key: str | None+ date: str | None+ engagement_export_date: str | None+ # Why no day passes, or None when `key` names one.+ refusal: str | None+++def _project_vectors_refusal(metadata: Mapping[str, str], oldest: datetime.date) -> str | None:+ if metadata.get(PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY) != str(PROJECT_VECTORS_SCHEMA_VERSION):+ return UNKNOWN_SCHEMA_VERSION+ try:+ engagement_date = datetime.date.fromisoformat(metadata.get(PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY, ""))+ except ValueError:+ return STALE_ENGAGEMENT_EXPORT+ return STALE_ENGAGEMENT_EXPORT if engagement_date < oldest else None+++def _mirror_project_vectors(mirror: _ServingStore, prefix: str, partition_key: str) -> _MirrorProjectVectors:+ """The newest day within `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS` of the partition that the mirror+ region wrote its own project vectors for, and that this build can name.++ Only object metadata is read, never the body: this region must not hold the mirror region's+ per-team vectors. The mirror grant has no list permission, so a missing key answers 403, and+ `head_object` reads both as absent.+ """+ end = datetime.date.fromisoformat(partition_key)+ oldest = end - datetime.timedelta(days=MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS)+ newest_refused: _MirrorProjectVectors | None = None+ for offset in range(MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS + 1):+ day = (end - datetime.timedelta(days=offset)).isoformat()+ key = serving_project_vectors_key(prefix, day)+ head = mirror.storage.head_object(bucket=mirror.bucket, file_key=key)+ if head is None:+ continue+ metadata = head.get("Metadata") or {}+ refusal = _project_vectors_refusal(metadata, oldest)+ found = _MirrorProjectVectors(+ key=None if refusal else key,+ date=day,+ engagement_export_date=metadata.get(PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY),+ refusal=refusal,+ )+ if refusal is None:+ return found+ newest_refused = newest_refused or found+ return newest_refused or _MirrorProjectVectors(+ key=None, date=None, engagement_export_date=None, refusal=NO_FRESH_PROJECT_VECTORS+ )+++def _publish_mirror(+ context: RunContext, client, bucket: str, prefix: str, partition_key: str, manifest: ServingManifest+) -> MirrorPublication | None:"""Copy the same publish into the mirror store, if one is set. Returns None when no mirror is- set, otherwise whether the mirror now serves this manifest.+ set, otherwise what the mirror now serves.The primary publish has already succeeded, so a mirror failure only logs: the region thattrains keeps serving, and the mirror keeps its previous manifest and models.Project vectors are per-team data of this deployment's teams, so they never go to another- deployment's store. The mirror's manifest names none, and its sweep skips the models that read them.- A served model that reads them could not score there, so the mirror keeps its previous manifest.+ deployment's store. When an entry reads them, the mirror's manifest names the mirror region's+ own day instead (`_mirror_project_vectors`). That day is already in the mirror store, so+ `publish_serving_models` gets no source and copies none. Without such a day the mirror's+ manifest names none, and its sweep skips the models that read them. A served model that reads+ them could not score there, so then the mirror keeps its previous manifest."""if not settings.INBOX_RANKING_SERVING_MIRROR_BUCKET:return None- if manifest.served.reads_project_vectors:- context.log.warning(- f"serving manifest {manifest.manifest_version} not mirrored: its served model reads project vectors"- )- return False+ vectors: _MirrorProjectVectors | None = Nonetry:mirror = _ServingStore(storage=serving_mirror_storage(),bucket=settings.INBOX_RANKING_SERVING_MIRROR_BUCKET,acl="bucket-owner-full-control",)+ if manifest.project_vectors is not None:+ vectors = _mirror_project_vectors(mirror, prefix, partition_key)+ project_vectors_key = vectors.key if vectors else None+ if manifest.served.reads_project_vectors and vectors is not None and project_vectors_key is None:+ context.log.warning(+ f"serving manifest {manifest.manifest_version} not mirrored: its served model reads project "+ f"vectors and the mirror has no usable day ({vectors.refusal}, newest day {vectors.date})"+ )+ return _mirror_publication(vectors, skip_reason=vectors.refusal)publish_serving_models(- context, client, bucket, prefix, manifest.model_copy(update={"project_vectors": None}), mirror+ context,+ client,+ bucket,+ prefix,+ manifest.model_copy(update={"project_vectors": project_vectors_key}),+ mirror,)except Exception as error:context.log.exception(f"serving manifest {manifest.manifest_version} not mirrored: {error!r}")- return False- context.log.info(f"mirrored serving manifest {manifest.manifest_version} to {mirror.bucket}")- return True+ return _mirror_publication(vectors, skip_reason=MIRROR_PUBLISH_FAILED)+ context.log.info(+ f"mirrored serving manifest {manifest.manifest_version} to {mirror.bucket}"+ f" with project vectors {project_vectors_key}"+ )+ return _mirror_publication(vectors, skip_reason=None)+++def _mirror_publication(vectors: _MirrorProjectVectors | None, skip_reason: str | None) -> MirrorPublication:+ return MirrorPublication(+ published=skip_reason is None,+ project_vectors_key=vectors.key if vectors and skip_reason is None else None,+ project_vectors_date=vectors.date if vectors else None,+ engagement_export_date=vectors.engagement_export_date if vectors else None,+ skip_reason=skip_reason,+ )@frozendiff --git a/products/signals/dags/inbox_ranking/training/telemetry.py b/products/signals/dags/inbox_ranking/training/telemetry.pyindex 576e5072c48b..688ffc2248d9 100644--- a/products/signals/dags/inbox_ranking/training/telemetry.py+++ b/products/signals/dags/inbox_ranking/training/telemetry.py@@ -60,6 +60,19 @@ class TrainingEvent:properties: dict[str, object]+@frozen+class MirrorPublication:+ """What one serving publish did in the mirror store."""++ published: bool+ # The mirror region's own project vectors day that the mirror manifest names, or None.+ project_vectors_key: str | None = None+ project_vectors_date: str | None = None+ engagement_export_date: str | None = None+ # Why the mirror keeps its previous manifest, or None when it published.+ skip_reason: str | None = None++def candidate_events(metadata: Mapping[str, Any]) -> list[TrainingEvent]:"""One event per head: the head's metrics plus the candidate context. A head the candidatecould not fit still gets an event (`trained` false, `readable` false), so a per-head alert sees@@ -169,7 +182,7 @@ def serving_manifest_event(copied_keys: Sequence[str] = (),present_keys: Sequence[str] = (),bytes_copied: int = 0,- mirror_published: bool | None = None,+ mirror: MirrorPublication | None = None,recipe_hashes: Mapping[str, str | None] | None = None,) -> TrainingEvent:"""What the day's manifest published. A run that wrote nothing still reports, with `reason`:@@ -194,8 +207,15 @@ def serving_manifest_event("model_roles": {entry.key: entry.roles for entry in manifest.models},"model_recipe_hashes": {entry.key: (recipe_hashes or {}).get(entry.key) for entry in manifest.models},}- if mirror_published is not None:- properties["mirror_published"] = mirror_published+ if mirror is not None:+ properties |= {+ "mirror_published": mirror.published,+ "mirror_project_vectors_key": mirror.project_vectors_key,+ "mirror_project_vectors_date": mirror.project_vectors_date,+ "mirror_engagement_export_date": mirror.engagement_export_date,+ }+ if mirror.skip_reason is not None:+ properties["mirror_skip_reason"] = mirror.skip_reasonreturn TrainingEvent(event=SERVING_MANIFEST_PUBLISHED_EVENT, properties=properties)