Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): mirror inbox ranking manifests with the mirror region's own project vectors (#121431)

posthog[bot]b9ea2ea47e88
diff --git a/products/signals/backend/ranking/serving_manifest.py b/products/signals/backend/ranking/serving_manifest.py
index 8531bf325ec0..8add20bc0eb0 100644
--- a/products/signals/backend/ranking/serving_manifest.py
+++ b/products/signals/backend/ranking/serving_manifest.py
@@ -14,6 +14,8 @@ not act on is refused by the dag that composed it.
from __future__ import annotations
+import datetime
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from products.signals.backend.artefact_schemas import MAX_RANKING_MODEL_RESULTS, RANKING_SERVED_ROLE
@@ -55,6 +57,12 @@ ENGAGEMENT_FILE = "engagement.parquet"
_SERVING_SEGMENT = "serving"
+# The object metadata a mirror region stamps on its own project vectors day. The region that trains
+# reads only these, through `head_object`, before its mirror manifest names the day. The schema
+# version key is the one `write_parquet` stamps.
+PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY = "feature-schema-version"
+PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY = "engagement-export-date"
+
def serving_manifest_key(prefix: str) -> str:
return f"{prefix}/{_SERVING_SEGMENT}/{MANIFEST_FILE}"
@@ -78,6 +86,17 @@ def serving_engagement_key(prefix: str, partition_key: str) -> str:
return f"{prefix}/{_SERVING_SEGMENT}/engagement/{partition_key}/{ENGAGEMENT_FILE}"
+def serving_project_vectors_date(key: str) -> datetime.date | None:
+ """The `dt` of a `serving_project_vectors_key`, or None when the key has another shape."""
+ parts = key.rsplit("/", 2)
+ if len(parts) != 3 or parts[2] != PROJECT_VECTORS_FILE:
+ return None
+ try:
+ return datetime.date.fromisoformat(parts[1])
+ except ValueError:
+ return None
+
+
def model_key(model_name: str, model_version: str) -> str:
return f"{model_name}@{model_version}"
diff --git a/products/signals/backend/ranking/sweep.py b/products/signals/backend/ranking/sweep.py
index 71736a14741c..1db1e7639911 100644
--- a/products/signals/backend/ranking/sweep.py
+++ b/products/signals/backend/ranking/sweep.py
@@ -40,6 +40,7 @@ from products.signals.backend.ranking import scorer
from products.signals.backend.ranking.inventory import SCORABLE_STATUSES, spine_report_filter
from products.signals.backend.ranking.model_store import ModelLoadError, ServingSet, load_serving_set
from products.signals.backend.ranking.overrides import RankingOverrides, read_ranking_overrides
+from products.signals.backend.ranking.serving_manifest import serving_project_vectors_date
from products.signals.backend.report_embedding_reader import REPORT_EMBEDDINGS_TABLE
from products.signals.backend.report_embeddings import EMBEDDING_DOCUMENT_TYPE, EMBEDDING_PRODUCT
@@ -344,6 +345,12 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult:
scored += sum(1 for outcome in outcomes if outcome.score is not None)
no_vector += sum(1 for outcome in outcomes if outcome.reason == scorer.NO_VECTOR)
+ # The day the pass read, so a stale mirror region day shows in the log.
+ vectors_date = (
+ serving_project_vectors_date(serving.manifest.project_vectors)
+ if project_vectors is not None and project_vectors.available and serving.manifest.project_vectors
+ else None
+ )
result = ScoreInboxReportsResult(
candidates=len(candidates),
scored=scored,
@@ -364,6 +371,8 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult:
deferred_teams=result.deferred_teams,
deferred_reports=result.deferred_reports,
manifest_version=result.manifest_version,
+ project_vectors_date=vectors_date.isoformat() if vectors_date else None,
+ project_vectors_age_days=(now.date() - vectors_date).days if vectors_date else None,
override_served=override.served if override else None,
override_expires_at=override.expires_at.isoformat() if override and override.expires_at else None,
peak_rss_mb=_peak_rss_mb(),
diff --git a/products/signals/backend/test/test_ranking_sweep.py b/products/signals/backend/test/test_ranking_sweep.py
index 25edf3601436..fbf8535d50a8 100644
--- a/products/signals/backend/test/test_ranking_sweep.py
+++ b/products/signals/backend/test/test_ranking_sweep.py
@@ -203,7 +203,9 @@ class TestScoreInboxReports(SimpleTestCase):
def setUp(self) -> None:
super().setUp()
self.serving = SimpleNamespace(
- manifest=SimpleNamespace(manifest_version=MANIFEST, served=SimpleNamespace(key=SERVED_KEY)),
+ manifest=SimpleNamespace(
+ manifest_version=MANIFEST, served=SimpleNamespace(key=SERVED_KEY), project_vectors=None
+ ),
served_override=None,
)
self._patch(sweep, "read_ranking_overrides", return_value=NO_OVERRIDES)
diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.md
index d3ed5496677f..e4d99ee978ba 100644
--- a/products/signals/dags/inbox_ranking/AGENTS.md
+++ b/products/signals/dags/inbox_ranking/AGENTS.md
@@ -38,7 +38,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- The manifest schema and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, next to the feature contract, because the dag writes them and the sweep reads them. Its validators run at write time, so the dag refuses a manifest the sweep could not act on; the entry cap is the `ranking_score` artefact's, so a manifest can never produce a row the artefact refuses.
- The `recent` role keeps the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` versions of the served `model_name`, read from the last 2N partitions, so each version gets several days of online scores. The `challenger` role keeps the newest readable version of each other serving family (`serving_families()`), read from its last 2 partitions. Each one passes the pin check before composition, because one missing file fails the publish. Keep the order under the cap as served, recent, challenger, pinned, cross-family. Each recent entry is one more model per sweep pass, so measure the sweep's duration, peak RSS and scored and deferred counts on the `inbox_ranking_sweep_finished` log before and after a change to N.
- Publishing order is load-bearing. Every model copy finishes before `manifest.json` is written, so a failed copy leaves the previous manifest and the previous models serving. A model version is immutable, so a version whose `metadata.json` is already at the target is not copied again. A run with nothing to publish, such as a run without `served.json`, writes nothing rather than an empty manifest. The served model passes the same check as a pin (`_pin_metadata`) before composition.
-- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror: they are per-team data of this deployment, so the mirror's manifest names none. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.
+- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror, and never read the body of a mirror object: they are per-team data of the deployment that owns the teams. The mirror's manifest names the mirror region's own day only after `_mirror_project_vectors` checks its object metadata (`PROJECT_VECTORS_*_METADATA_KEY` in `serving_manifest.py`) within `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS`. Without such a day it names none, and a served model that reads them keeps the mirror on its previous manifest. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.
- `HEADS` holds only the heads the inbox reads (`HeadRanker` in `backend/ranking/inbox_sort.py`). Add a head there only together with a surface that reads it. The promotion gate skips a champion head that is not in `HEADS`, so a trim does not block promotion.
- `nn_v3` and `nn_with_prio_v2` are trained by the `signals_ml` location (`nn/dag.py`), never by the main one. They share one `inbox_ranking_nn_examples` object, and each has its own candidate and champion asset from `family_assets`. The `nn_v3` assets keep the names `inbox_ranking_nn_model_{candidate,champion}`, so their Dagster history continues. A later family's candidate depends on the previous family's champion, for memory and not for data: two candidates at once can exceed the job's memory limit. A family whose set reads a state column the examples object lacks skips with a warning rather than training on zeros. torch is imported only inside the candidate asset's body (through `nn/train.py` and `multihead/network.py`). `signals_ml` must import without torch, and `test_multihead_network.py` pins that. Its decision goes through `decide_family_champion` in `training/dag.py`, graded through the incumbent's `model.holdout.onnx`. Keep `signals_ml` commented out in `.dagster_home/workspace.yaml`, and keep its schedule (04:30 UTC) running by default on prod US only.
diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md
index c13ef44aed63..2f9a41359ddb 100644
--- a/products/signals/dags/inbox_ranking/README.md
+++ b/products/signals/dags/inbox_ranking/README.md
@@ -223,6 +223,14 @@ The manifest holds one entry per model with its `roles`: `served` for the model
A region that does not train serves the same models through a mirror. When `INBOX_RANKING_SERVING_MIRROR_BUCKET` is set, the asset repeats the publish into that bucket after the primary publish succeeds, in the same order and with the same skip of versions already there. A mirror failure does not fail the asset: it logs, and the `inbox_ranking_serving_manifest_published` event carries `mirror_published=false`. The property is absent when no mirror is set.
+Project vectors are per-team data, so this region never copies its own day to the mirror. When a manifest entry reads them, the asset looks for the mirror region's own day in the mirror bucket instead:
+
+- It calls `head_object` on `serving_project_vectors_key(prefix, D')` for each day D' from D back to D − `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS` (3), newest first. The mirror grant has no list permission, so a missing key answers 403, and both read as absent. It reads only the object metadata, never the body.
+- A day passes when its `feature-schema-version` metadata is this build's `PROJECT_VECTORS_SCHEMA_VERSION` and its `engagement-export-date` metadata is no older than D − 3. The key names live in `serving_manifest.py`, and the mirror region's writer uses the same ones.
+- When a day passes, the mirror manifest names it in `project_vectors`, and the publish copies no project vectors object. The mirror region builds day D after the 05:00 UTC publish, so a mirror that keeps up names D−1, one day older than the primary.
+- When no day passes and the served model reads project vectors, the mirror keeps its previous manifest. When only other entries read them, the mirror manifest names none, and the mirror sweep skips those entries.
+- The event carries `mirror_project_vectors_key`, `mirror_project_vectors_date` and `mirror_engagement_export_date`, and `mirror_skip_reason` when the mirror is not published: `no_fresh_project_vectors`, `unknown_schema_version`, `stale_engagement_export` or `publish_failed`. The `inbox_ranking_sweep_finished` log carries the `project_vectors_date` the pass read and its `project_vectors_age_days`.
+
The reader is `score_reports` in `products/signals/backend/ranking/scorer.py`. It loads every model the manifest names through `model_store.py`, reads each report's latest vector once per rendering, builds each model's matrix with its own `FeatureSet.build_matrix`, and writes one `ranking_score` artefact per report plus one `inbox_ranking_report_scored` event per report and model. The dag and the reader decide whether a model can be scored with the same functions, in `products/signals/backend/ranking/model_contract.py`. A served model that cannot load stops the pass. Any other model that cannot load, or has no vector for a report, is a `skipped` result. No scheduled caller runs the reader yet.
### Overrides
diff --git a/products/signals/dags/inbox_ranking/mirror/dag.py b/products/signals/dags/inbox_ranking/mirror/dag.py
index 2f9c76a01353..d447ec1fadf4 100644
--- a/products/signals/dags/inbox_ranking/mirror/dag.py
+++ b/products/signals/dags/inbox_ranking/mirror/dag.py
@@ -24,7 +24,12 @@ from posthog.dataclasses import frozen
from posthog.storage import object_storage
from posthog.storage.object_storage import ObjectStorageClient
-from products.signals.backend.ranking.serving_manifest import serving_engagement_key, serving_project_vectors_key
+from products.signals.backend.ranking.serving_manifest import (
+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,
+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY,
+ serving_engagement_key,
+ serving_project_vectors_key,
+)
from products.signals.backend.report_embeddings import EMBEDDING_RENDERING_TITLE_SUMMARY
from products.signals.dags.inbox_ranking.common import (
ROW_COUNT_METADATA_KEY,
@@ -52,7 +57,6 @@ MIRROR_PROJECT_VECTORS_ASSET = "inbox_mirror_project_vectors"
# An older export only misses the newest engagement, so a late training-region run costs a little
# history. Past this age the vectors drift too far from what training read, so the asset fails.
ENGAGEMENT_EXPORT_MAX_AGE_DAYS = 2
-ENGAGEMENT_EXPORT_DATE_METADATA_KEY = "engagement-export-date"
@frozen
@@ -131,8 +135,8 @@ def inbox_mirror_project_vectors(context: dagster.AssetExecutionContext) -> None
"Metadata": {
ROW_COUNT_METADATA_KEY: str(table.num_rows),
SNAPSHOT_DATE_METADATA_KEY: partition_key,
- SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),
- ENGAGEMENT_EXPORT_DATE_METADATA_KEY: export.export_date,
+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),
+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY: export.export_date,
}
},
)
diff --git a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
index 830e93fb6724..6f776cee9ca1 100644
--- a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
+++ b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
@@ -14,6 +14,7 @@ from parameterized import parameterized
from products.signals.backend.ranking import model_store
from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS
from products.signals.backend.ranking.serving_manifest import (
+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,
SERVED_ROLE,
ServingManifest,
ServingManifestEntry,
@@ -33,6 +34,7 @@ from products.signals.dags.inbox_ranking.dataset.project_vectors import (
)
from products.signals.dags.inbox_ranking.mirror import dag as mirror_dag
from products.signals.dags.inbox_ranking.multihead.prototype import project_context
+from products.signals.dags.inbox_ranking.training import dag as training_dag
AS_OF = datetime.datetime(2026, 7, 30, tzinfo=datetime.UTC)
BEFORE = AS_OF - datetime.timedelta(days=3)
@@ -357,8 +359,11 @@ def test_the_mirror_region_builds_its_vectors_from_its_embeddings_and_the_export
common.ROW_COUNT_METADATA_KEY: str(written.num_rows),
common.SNAPSHOT_DATE_METADATA_KEY: partition_day.isoformat(),
common.SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),
- mirror_dag.ENGAGEMENT_EXPORT_DATE_METADATA_KEY: export_date,
+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY: export_date,
}
+ # The training region's mirror lookup reads only this metadata, so a key drift drops every day.
+ oldest = partition_day - datetime.timedelta(days=training_dag.MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS)
+ assert training_dag._project_vectors_refusal(write["extras"]["Metadata"], oldest) is None
assert written.schema == PROJECT_VECTORS_SCHEMA
rows = _rows_by_key(written)
assert {team_id for team_id, _ in rows} == {5}
diff --git a/products/signals/dags/inbox_ranking/tests/test_training.py b/products/signals/dags/inbox_ranking/tests/test_training.py
index ea0f3cf30917..f30b3bcc0b35 100644
--- a/products/signals/dags/inbox_ranking/tests/test_training.py
+++ b/products/signals/dags/inbox_ranking/tests/test_training.py
@@ -45,6 +45,8 @@ from products.signals.backend.ranking.serving_manifest import (
ONNX_MODEL_FILE,
ONNX_MODEL_KIND,
PINNED_ROLE,
+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,
+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY,
RECENT_ROLE,
SERVED_ROLE,
ServingManifest,
@@ -58,6 +60,7 @@ from products.signals.backend.ranking.serving_manifest import (
from products.signals.backend.ranking.sinks import REPORT_SCORED_EVENT
from products.signals.dags.inbox_ranking.common import partition_object_key
from products.signals.dags.inbox_ranking.dataset.dag import LABELS_TABLE, PROJECT_VECTORS_TABLE, STATE_TABLE
+from products.signals.dags.inbox_ranking.dataset.project_vectors import PROJECT_VECTORS_SCHEMA_VERSION
from products.signals.dags.inbox_ranking.training.calibration import (
BUCKETS,
calibration_buckets,
@@ -67,6 +70,9 @@ from products.signals.dags.inbox_ranking.training.classification import classifi
from products.signals.dags.inbox_ranking.training.dag import (
_STATE_READ_COLUMNS,
METADATA_FILE,
+ NO_FRESH_PROJECT_VECTORS,
+ STALE_ENGAGEMENT_EXPORT,
+ UNKNOWN_SCHEMA_VERSION,
_delete_other_objects,
_pinned_models,
_publish_manifest,
@@ -2157,14 +2163,22 @@ def test_a_model_published_before_model_kind_existed_reads_as_xgboost():
class _AppObjectStore:
"""The deployment's object store, the one the Temporal workers can reach."""
- def __init__(self, existing: dict[str, bytes] | None = None, fail_on: str | None = None):
+ def __init__(
+ self,
+ existing: dict[str, bytes] | None = None,
+ fail_on: str | None = None,
+ metadata: dict[str, dict[str, str]] | None = None,
+ ):
self.objects = dict(existing or {})
+ self.metadata = dict(metadata or {})
self.fail_on = fail_on
self.written: list[str] = []
self.extras: dict[str, dict[str, str] | None] = {}
- def head_object(self, bucket: str, file_key: str) -> dict[str, int] | None:
- return {"ContentLength": len(self.objects[file_key])} if file_key in self.objects else None
+ def head_object(self, bucket: str, file_key: str) -> dict[str, Any] | None:
+ if file_key not in self.objects:
+ return None
+ return {"ContentLength": len(self.objects[file_key]), "Metadata": self.metadata.get(file_key, {})}
def write(self, bucket: str, key: str, content: str | bytes, extras: dict[str, str] | None) -> None:
if self.fail_on is not None and self.fail_on in key:
@@ -2336,6 +2350,91 @@ def test_project_vectors_are_published_before_the_manifest_that_names_them(
assert mirror.objects == {}
+_MIRROR_PARTITION = "2026-08-19"
+
+
+def _mirror_day(offset: int) -> str:
+ return (datetime.date.fromisoformat(_MIRROR_PARTITION) - datetime.timedelta(days=offset)).isoformat()
+
+
+def _mirror_vectors_metadata(engagement_offset: int, schema_version: int = PROJECT_VECTORS_SCHEMA_VERSION):
+ return {
+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(schema_version),
+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY: _mirror_day(engagement_offset),
+ }
+
+
+ "mirror_days,expected_offset,skip_reason",
+ [
+ ({0: _mirror_vectors_metadata(1)}, 0, None),
+ ({1: _mirror_vectors_metadata(2)}, 1, None),
+ (
+ {
+ 0: _mirror_vectors_metadata(1, schema_version=PROJECT_VECTORS_SCHEMA_VERSION + 1),
+ 1: _mirror_vectors_metadata(2),
+ },
+ 1,
+ None,
+ ),
+ ({}, None, NO_FRESH_PROJECT_VECTORS),
+ ({4: _mirror_vectors_metadata(5)}, None, NO_FRESH_PROJECT_VECTORS),
+ (
+ {0: _mirror_vectors_metadata(1, schema_version=PROJECT_VECTORS_SCHEMA_VERSION + 1)},
+ None,
+ UNKNOWN_SCHEMA_VERSION,
+ ),
+ ({0: _mirror_vectors_metadata(4)}, None, STALE_ENGAGEMENT_EXPORT),
+ (
+ {0: {PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION)}},
+ None,
+ STALE_ENGAGEMENT_EXPORT,
+ ),
+ ],
+)
+def test_the_mirror_manifest_names_the_mirror_regions_own_fresh_project_vectors(
+ monkeypatch: pytest.MonkeyPatch,
+ mirror_days: dict[int, dict[str, str]],
+ expected_offset: int | None,
+ skip_reason: str | None,
+) -> None:
+ prefix = settings.INBOX_RANKING_DATASET_S3_PREFIX
+ mirror_keys = {offset: serving_project_vectors_key(prefix, _mirror_day(offset)) for offset in mirror_days}
+ # The fake has no read method, so a publish that read a mirror object body would fail the mirror.
+ mirror = _AppObjectStore(
+ dict.fromkeys(mirror_keys.values(), b"mirror region vectors"),
+ metadata={key: mirror_days[offset] for offset, key in mirror_keys.items()},
+ )
+ before = dict(mirror.objects)
+ dataset = _serving_dataset_s3(
+ prefix,
+ _MIRROR_PARTITION,
+ model_kind=ONNX_MODEL_KIND,
+ project_vector_kinds=("typical", "open", "action", "dismiss"),
+ )
+ dataset.objects[partition_object_key(prefix, PROJECT_VECTORS_TABLE, _MIRROR_PARTITION)] = b"this region vectors"
+ capture = _run_serving_manifest(monkeypatch, _AppObjectStore(), mirror=mirror, dataset_client=dataset)
+
+ event = _manifest_event(capture)
+ assert b"this region vectors" not in mirror.objects.values()
+ assert not any("/project_vectors/" in key for key in mirror.written)
+ if expected_offset is None:
+ assert mirror.objects == before
+ assert event["mirror_published"] is False
+ assert event["mirror_skip_reason"] == skip_reason
+ assert event["mirror_project_vectors_key"] is None
+ return
+ manifest = ServingManifest.model_validate_json(mirror.objects[serving_manifest_key(prefix)])
+ assert manifest.served.reads_project_vectors
+ assert manifest.project_vectors == mirror_keys[expected_offset]
+ assert mirror.written[-1] == serving_manifest_key(prefix)
+ assert event["mirror_published"] is True
+ assert "mirror_skip_reason" not in event
+ assert event["mirror_project_vectors_key"] == mirror_keys[expected_offset]
+ assert event["mirror_project_vectors_date"] == _mirror_day(expected_offset)
+ assert event["mirror_engagement_export_date"] == _mirror_day(expected_offset + 1)
+
+
def test_recent_versions_and_challengers_come_from_the_window_and_skip_a_version_missing_its_booster(monkeypatch):
# A booster deleted by a re-run would fail the whole publish, and a version older than the
# window would keep a stale model in every pass. A challenger is one version per other family,
diff --git a/products/signals/dags/inbox_ranking/training/dag.py b/products/signals/dags/inbox_ranking/training/dag.py
index e94b8ec795c5..6cd8e3bcb438 100644
--- a/products/signals/dags/inbox_ranking/training/dag.py
+++ b/products/signals/dags/inbox_ranking/training/dag.py
@@ -46,6 +46,8 @@ from products.signals.backend.ranking.features import FEATURE_SETS
from products.signals.backend.ranking.model_contract import model_mismatch, trained_head_files
from products.signals.backend.ranking.overrides import read_ranking_overrides
from products.signals.backend.ranking.serving_manifest import (
+ PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,
+ PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY,
ServingManifest,
model_key,
serving_manifest_key,
@@ -66,6 +68,7 @@ from products.signals.dags.inbox_ranking.common import (
skip_unconfigured,
)
from products.signals.dags.inbox_ranking.dataset.dag import LABELS_TABLE, PROJECT_VECTORS_TABLE, STATE_TABLE
+from products.signals.dags.inbox_ranking.dataset.project_vectors import PROJECT_VECTORS_SCHEMA_VERSION
from products.signals.dags.inbox_ranking.training.examples import (
BASE_STATE_COLUMNS,
PROVENANCE_LABEL_COLUMNS,
@@ -89,6 +92,7 @@ from products.signals.dags.inbox_ranking.training.promotion import decide_promot
from products.signals.dags.inbox_ranking.training.recipe import RECIPE_HASH_KEY, recipe_mismatch
from products.signals.dags.inbox_ranking.training.serving import FamilyModels, ManifestDecision, compose_manifest
from products.signals.dags.inbox_ranking.training.telemetry import (
+ MirrorPublication,
capture_training_events,
promotion_event,
serving_manifest_event,
@@ -511,7 +515,7 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
context, client, bucket, prefix, manifest, primary, project_vectors_source=project_vectors_source
)
context.log.info(f"published serving manifest {manifest.manifest_version} serving {manifest.served.key}")
- mirror_published = _publish_mirror(context, client, bucket, prefix, manifest)
+ mirror = _publish_mirror(context, client, bucket, prefix, partition_key, manifest)
capture_training_events(
context,
partition_key,
@@ -525,7 +529,7 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
copied_keys=publication.copied,
present_keys=publication.present,
bytes_copied=publication.bytes_copied,
- mirror_published=mirror_published,
+ mirror=mirror,
recipe_hashes={
model_key(metadata["model_name"], metadata["model_version"]): metadata.get(RECIPE_HASH_KEY)
for metadata in (
@@ -540,7 +544,13 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
],
)
mirror_metadata = (
- {} if mirror_published is None else {"mirror_published": dagster.MetadataValue.bool(mirror_published)}
+ {}
+ if mirror is None
+ else {
+ "mirror_published": dagster.MetadataValue.bool(mirror.published),
+ "mirror_project_vectors_key": dagster.MetadataValue.text(mirror.project_vectors_key or ""),
+ "mirror_skip_reason": dagster.MetadataValue.text(mirror.skip_reason or ""),
+ }
)
return {
"published": dagster.MetadataValue.bool(True),
@@ -675,38 +685,131 @@ def _pin_metadata(client, bucket: str, prefix: str, key: str) -> tuple[dict[str,
return metadata, None
-def _publish_mirror(context: RunContext, client, bucket: str, prefix: str, manifest: ServingManifest) -> bool | None:
+# How many days before the publish partition the mirror region's project vectors day, and the
+# engagement export it was built from, may be. The mirror region builds day D after this region's
+# 05:00 UTC publish, so a mirror that keeps up names D-1.
+MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS = 3
+
+NO_FRESH_PROJECT_VECTORS = "no_fresh_project_vectors"
+UNKNOWN_SCHEMA_VERSION = "unknown_schema_version"
+STALE_ENGAGEMENT_EXPORT = "stale_engagement_export"
+MIRROR_PUBLISH_FAILED = "publish_failed"
+
+
+@frozen
+class _MirrorProjectVectors:
+ """The mirror region's project vectors day the mirror manifest names, or the newest day refused."""
+
+ key: str | None
+ date: str | None
+ engagement_export_date: str | None
+ # Why no day passes, or None when `key` names one.
+ refusal: str | None
+
+
+def _project_vectors_refusal(metadata: Mapping[str, str], oldest: datetime.date) -> str | None:
+ if metadata.get(PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY) != str(PROJECT_VECTORS_SCHEMA_VERSION):
+ return UNKNOWN_SCHEMA_VERSION
+ try:
+ engagement_date = datetime.date.fromisoformat(metadata.get(PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY, ""))
+ except ValueError:
+ return STALE_ENGAGEMENT_EXPORT
+ return STALE_ENGAGEMENT_EXPORT if engagement_date < oldest else None
+
+
+def _mirror_project_vectors(mirror: _ServingStore, prefix: str, partition_key: str) -> _MirrorProjectVectors:
+ """The newest day within `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS` of the partition that the mirror
+ region wrote its own project vectors for, and that this build can name.
+
+ Only object metadata is read, never the body: this region must not hold the mirror region's
+ per-team vectors. The mirror grant has no list permission, so a missing key answers 403, and
+ `head_object` reads both as absent.
+ """
+ end = datetime.date.fromisoformat(partition_key)
+ oldest = end - datetime.timedelta(days=MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS)
+ newest_refused: _MirrorProjectVectors | None = None
+ for offset in range(MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS + 1):
+ day = (end - datetime.timedelta(days=offset)).isoformat()
+ key = serving_project_vectors_key(prefix, day)
+ head = mirror.storage.head_object(bucket=mirror.bucket, file_key=key)
+ if head is None:
+ continue
+ metadata = head.get("Metadata") or {}
+ refusal = _project_vectors_refusal(metadata, oldest)
+ found = _MirrorProjectVectors(
+ key=None if refusal else key,
+ date=day,
+ engagement_export_date=metadata.get(PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY),
+ refusal=refusal,
+ )
+ if refusal is None:
+ return found
+ newest_refused = newest_refused or found
+ return newest_refused or _MirrorProjectVectors(
+ key=None, date=None, engagement_export_date=None, refusal=NO_FRESH_PROJECT_VECTORS
+ )
+
+
+def _publish_mirror(
+ context: RunContext, client, bucket: str, prefix: str, partition_key: str, manifest: ServingManifest
+) -> MirrorPublication | None:
"""Copy the same publish into the mirror store, if one is set. Returns None when no mirror is
- set, otherwise whether the mirror now serves this manifest.
+ set, otherwise what the mirror now serves.
The primary publish has already succeeded, so a mirror failure only logs: the region that
trains keeps serving, and the mirror keeps its previous manifest and models.
Project vectors are per-team data of this deployment's teams, so they never go to another
- deployment's store. The mirror's manifest names none, and its sweep skips the models that read them.
- A served model that reads them could not score there, so the mirror keeps its previous manifest.
+ deployment's store. When an entry reads them, the mirror's manifest names the mirror region's
+ own day instead (`_mirror_project_vectors`). That day is already in the mirror store, so
+ `publish_serving_models` gets no source and copies none. Without such a day the mirror's
+ manifest names none, and its sweep skips the models that read them. A served model that reads
+ them could not score there, so then the mirror keeps its previous manifest.
"""
if not settings.INBOX_RANKING_SERVING_MIRROR_BUCKET:
return None
- if manifest.served.reads_project_vectors:
- context.log.warning(
- f"serving manifest {manifest.manifest_version} not mirrored: its served model reads project vectors"
- )
- return False
+ vectors: _MirrorProjectVectors | None = None
try:
mirror = _ServingStore(
storage=serving_mirror_storage(),
bucket=settings.INBOX_RANKING_SERVING_MIRROR_BUCKET,
acl="bucket-owner-full-control",
)
+ if manifest.project_vectors is not None:
+ vectors = _mirror_project_vectors(mirror, prefix, partition_key)
+ project_vectors_key = vectors.key if vectors else None
+ if manifest.served.reads_project_vectors and vectors is not None and project_vectors_key is None:
+ context.log.warning(
+ f"serving manifest {manifest.manifest_version} not mirrored: its served model reads project "
+ f"vectors and the mirror has no usable day ({vectors.refusal}, newest day {vectors.date})"
+ )
+ return _mirror_publication(vectors, skip_reason=vectors.refusal)
publish_serving_models(
- context, client, bucket, prefix, manifest.model_copy(update={"project_vectors": None}), mirror
+ context,
+ client,
+ bucket,
+ prefix,
+ manifest.model_copy(update={"project_vectors": project_vectors_key}),
+ mirror,
)
except Exception as error:
context.log.exception(f"serving manifest {manifest.manifest_version} not mirrored: {error!r}")
- return False
- context.log.info(f"mirrored serving manifest {manifest.manifest_version} to {mirror.bucket}")
- return True
+ return _mirror_publication(vectors, skip_reason=MIRROR_PUBLISH_FAILED)
+ context.log.info(
+ f"mirrored serving manifest {manifest.manifest_version} to {mirror.bucket}"
+ f" with project vectors {project_vectors_key}"
+ )
+ return _mirror_publication(vectors, skip_reason=None)
+
+
+def _mirror_publication(vectors: _MirrorProjectVectors | None, skip_reason: str | None) -> MirrorPublication:
+ return MirrorPublication(
+ published=skip_reason is None,
+ project_vectors_key=vectors.key if vectors and skip_reason is None else None,
+ project_vectors_date=vectors.date if vectors else None,
+ engagement_export_date=vectors.engagement_export_date if vectors else None,
+ skip_reason=skip_reason,
+ )
@frozen
diff --git a/products/signals/dags/inbox_ranking/training/telemetry.py b/products/signals/dags/inbox_ranking/training/telemetry.py
index 576e5072c48b..688ffc2248d9 100644
--- a/products/signals/dags/inbox_ranking/training/telemetry.py
+++ b/products/signals/dags/inbox_ranking/training/telemetry.py
@@ -60,6 +60,19 @@ class TrainingEvent:
properties: dict[str, object]
+@frozen
+class MirrorPublication:
+ """What one serving publish did in the mirror store."""
+
+ published: bool
+ # The mirror region's own project vectors day that the mirror manifest names, or None.
+ project_vectors_key: str | None = None
+ project_vectors_date: str | None = None
+ engagement_export_date: str | None = None
+ # Why the mirror keeps its previous manifest, or None when it published.
+ skip_reason: str | None = None
+
+
def candidate_events(metadata: Mapping[str, Any]) -> list[TrainingEvent]:
"""One event per head: the head's metrics plus the candidate context. A head the candidate
could not fit still gets an event (`trained` false, `readable` false), so a per-head alert sees
@@ -169,7 +182,7 @@ def serving_manifest_event(
copied_keys: Sequence[str] = (),
present_keys: Sequence[str] = (),
bytes_copied: int = 0,
- mirror_published: bool | None = None,
+ mirror: MirrorPublication | None = None,
recipe_hashes: Mapping[str, str | None] | None = None,
) -> TrainingEvent:
"""What the day's manifest published. A run that wrote nothing still reports, with `reason`:
@@ -194,8 +207,15 @@ def serving_manifest_event(
"model_roles": {entry.key: entry.roles for entry in manifest.models},
"model_recipe_hashes": {entry.key: (recipe_hashes or {}).get(entry.key) for entry in manifest.models},
}
- if mirror_published is not None:
- properties["mirror_published"] = mirror_published
+ if mirror is not None:
+ properties |= {
+ "mirror_published": mirror.published,
+ "mirror_project_vectors_key": mirror.project_vectors_key,
+ "mirror_project_vectors_date": mirror.project_vectors_date,
+ "mirror_engagement_export_date": mirror.engagement_export_date,
+ }
+ if mirror.skip_reason is not None:
+ properties["mirror_skip_reason"] = mirror.skip_reason
return TrainingEvent(event=SERVING_MANIFEST_PUBLISHED_EVENT, properties=properties)