Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): compute inbox ranking project vectors in eu (#121440)

posthog[bot]ef3873faef97
diff --git a/posthog/dags/locations/signals.py b/posthog/dags/locations/signals.py
index 2df2c300d787..1f4f4b13ed5d 100644
--- a/posthog/dags/locations/signals.py
+++ b/posthog/dags/locations/signals.py
@@ -1,7 +1,8 @@
import dagster
-from products.signals.dags.inbox_ranking.common import is_inbox_ranking_registered
+from products.signals.dags.inbox_ranking.common import is_inbox_ranking_mirror_registered, is_inbox_ranking_registered
from products.signals.dags.inbox_ranking.dataset import dag as inbox_ranking_dataset
+from products.signals.dags.inbox_ranking.mirror import dag as inbox_ranking_mirror
from products.signals.dags.inbox_ranking.training import (
dag as inbox_ranking_training,
promote as inbox_ranking_promote,
@@ -43,5 +44,14 @@ if is_inbox_ranking_registered():
loggers=loggers,
resources=resources,
)
+# EU trains nothing, but it builds its own project vectors from its own report embeddings.
+elif is_inbox_ranking_mirror_registered():
+ defs = dagster.Definitions(
+ assets=[inbox_ranking_mirror.inbox_mirror_project_vectors],
+ jobs=[inbox_ranking_mirror.inbox_ranking_mirror_job],
+ schedules=[inbox_ranking_mirror.inbox_ranking_mirror_schedule],
+ loggers=loggers,
+ resources=resources,
+ )
else:
defs = dagster.Definitions(loggers=loggers, resources=resources)
diff --git a/posthog/settings/object_storage.py b/posthog/settings/object_storage.py
index 2c1298b66c6e..adb2a5b0d661 100644
--- a/posthog/settings/object_storage.py
+++ b/posthog/settings/object_storage.py
@@ -115,6 +115,12 @@ INBOX_RANKING_LABELS_REFRESH_MAX_RUNS = get_from_env("INBOX_RANKING_LABELS_REFRE
# no mirror.
INBOX_RANKING_SERVING_MIRROR_BUCKET = os.getenv("INBOX_RANKING_SERVING_MIRROR_BUCKET", "")
INBOX_RANKING_SERVING_MIRROR_REGION = os.getenv("INBOX_RANKING_SERVING_MIRROR_REGION", "")
+# Project vectors in the mirror region (products/signals/dags/inbox_ranking/mirror). The region builds
+# them from its own report embeddings and the engagement export the training region writes to its
+# object store. Off by default: the schedule skips its ticks until this is set.
+INBOX_RANKING_MIRROR_PROJECT_VECTORS_ENABLED = get_from_env(
+ "INBOX_RANKING_MIRROR_PROJECT_VECTORS_ENABLED", False, type_cast=str_to_bool
+)
# Serving manifest (products/signals/dags/inbox_ranking/training/serving.py): how many of the served
# family's newest versions the sweep also scores, read from the last 2N days. Each one costs the
# sweep one more model per pass. The served entry, these versions and one challenger per other family
diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.md
index edefdcf77ce3..d3ed5496677f 100644
--- a/products/signals/dags/inbox_ranking/AGENTS.md
+++ b/products/signals/dags/inbox_ranking/AGENTS.md
@@ -9,6 +9,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- `inbox_user_report_interactions` is a **leaf** with its own `USER_INTERACTIONS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`, and never stamp it with `FEATURE_SCHEMA_VERSION`, or a user-grain change sets off the labels refresh sensor. It writes a row only for a report in this region's Postgres whose team is in `training_consent_team_ids()`.
- `inbox_project_vectors` is a **leaf** with its own `PROJECT_VECTORS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`. Its `deps=[EMBEDDINGS_TABLE, LABELS_TABLE]` keep it after the embeddings snapshot, so the two vector-heavy steps do not share the run pod's memory. `project_vectors` takes a report into a vector only when its vector landed and its UI engagement happened strictly before `as_of`; never widen that to the latest snapshot value. A report born on day D reads the dt=D−1 vectors (see the README).
- `inbox_engagement_export` is a **leaf** with its own `ENGAGEMENT_EXPORT_SCHEMA_VERSION`, read by another region. It writes to the mirror store only under `serving_engagement_key`, because the mirror's write grant covers only `<prefix>/serving/`. Keep its columns exactly `LABEL_COLUMNS` and its rows limited to reports with no `SignalReport` in this region: no team ids, counts, text or vectors leave the region. A mirror failure logs and must not fail the asset.
+- `inbox_mirror_project_vectors` (`mirror/dag.py`) is the only inbox-ranking asset that registers on EU (`is_inbox_ranking_mirror_registered()`). It reads this region's report vectors through `report_embeddings_table` and the engagement export through `serving_engagement_key`, and writes only to this region's object store at `serving_project_vectors_key`. Keep it on the same `project_vectors()` and the same consent rule as `inbox_project_vectors`, so the two regions build their vectors the same way. A missing, too old or unknown-schema export fails the asset rather than writing vectors without engagement.
- Register new jobs/schedules in `posthog/dags/locations/signals.py` (or `signals_ml.py` for a torch-dependent dag), keep the EU gate (`is_inbox_ranking_registered()`), and remember the ci-dagster paths filter must cover every module a dag file imports (`uv run .github/scripts/check-dagster-paths.py`).
## Training dag specifics
diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md
index f75b42065b50..ca9a047ed19d 100644
--- a/products/signals/dags/inbox_ranking/README.md
+++ b/products/signals/dags/inbox_ranking/README.md
@@ -93,6 +93,19 @@ The inbox telemetry of every region lands in the US dogfood project, so the `ope
- A report deleted in this region keeps its Postgres row with the `deleted` status, so it is not exported. Only a report whose row is gone, for example after its team was deleted, looks remote. Such a row exposes a report id and timestamps, and it never matches a report vector in the other region.
- The grant has no delete, and no lifecycle rule covers the prefix, so the daily objects accumulate. Each one holds a report id and a few timestamps per row.
+### Mirror region project vectors
+
+`inbox_mirror_project_vectors` (`mirror/dag.py`) builds the project vectors of the mirror region (EU) in that region.
+The report vectors of a region exist only in its own ClickHouse, so the training region cannot build them.
+
+- It registers on EU only (`is_inbox_ranking_mirror_registered()`), in its own daily job `inbox_ranking_mirror_job` at 05:30 UTC, after the training region's dataset run.
+- The schedule skips its ticks until `INBOX_RANKING_MIRROR_PROJECT_VECTORS_ENABLED` is set. A run by hand works without it.
+- The report vectors come from `report_embeddings_table`, the same function the `inbox_report_embeddings` snapshot uses, for the teams in this region's `training_consent_team_ids()`.
+- The engagement moments come from the dt=D engagement export at `serving_engagement_key`, in this region's object store. When dt=D is missing, it uses the newest export up to `ENGAGEMENT_EXPORT_MAX_AGE_DAYS` older and logs a warning. When none is in that window, or the export has another schema version, the asset fails.
+- It calls `project_vectors()` in memory with `as_of` at the end of dt=D, and writes no dataset object.
+- The output is `serving_project_vectors_key(prefix, dt)` in this region's object store, stamped with the row count, the snapshot date, `PROJECT_VECTORS_SCHEMA_VERSION` and the export date it read.
+- The output metadata carries the rows, the teams, the teams with enough history for each kind, the export date and its age.
+
### Partition semantics
- Partition `dt=D` snapshots the eligible report inventory (promoted before `D+1 00:00 UTC`, or referenced by any label event before it). Every label aggregate is bounded `event_time < D+1 00:00 UTC`.
diff --git a/products/signals/dags/inbox_ranking/common.py b/products/signals/dags/inbox_ranking/common.py
index 05200c6295d3..f66366b40be6 100644
--- a/products/signals/dags/inbox_ranking/common.py
+++ b/products/signals/dags/inbox_ranking/common.py
@@ -74,6 +74,11 @@ def is_inbox_ranking_registered() -> bool:
return settings.CLOUD_DEPLOYMENT != "EU"
+def is_inbox_ranking_mirror_registered() -> bool:
+ """EU trains nothing, but it builds its own project vectors from its own report embeddings."""
+ return settings.CLOUD_DEPLOYMENT == "EU"
+
+
def dataset_unconfigured() -> bool:
# On Cloud the dataset must never fall back to the shared object-storage bucket: assets skip
# until the dedicated bucket is provisioned. Local dev falls back to the object-storage service.
diff --git a/products/signals/dags/inbox_ranking/dataset/dag.py b/products/signals/dags/inbox_ranking/dataset/dag.py
index 7d8ab538e3e1..55327457b202 100644
--- a/products/signals/dags/inbox_ranking/dataset/dag.py
+++ b/products/signals/dags/inbox_ranking/dataset/dag.py
@@ -134,10 +134,9 @@ from products.signals.dags.inbox_ranking.consent import training_consent_team_id
from products.signals.dags.inbox_ranking.dataset.project_vectors import (
EMBEDDING_COLUMNS,
LABEL_COLUMNS,
- MIN_HISTORY_REPORTS,
PROJECT_VECTORS_SCHEMA_VERSION,
- VECTOR_KINDS,
project_vectors,
+ teams_with_history,
)
from products.signals.dags.inbox_ranking.dataset.queries import (
LABEL_DEFAULTS,
@@ -202,7 +201,7 @@ COMMON_ASSET_KWARGS: dict[str, Any] = {
}
-def _tag_dagster_queries(context: dagster.AssetExecutionContext, query_type: str) -> None:
+def tag_dagster_queries(context: dagster.AssetExecutionContext, query_type: str) -> None:
"""Stamp product + feature + dagster run tags into the thread's query tags so every ClickHouse
query this asset issues (sync_execute and HogQL alike) is attributable in system.query_log.
Both product and feature are required: sync_execute refuses an untagged query in local dev.
@@ -782,7 +781,7 @@ def assemble_user_interaction_rows(
def inbox_report_state(context: dagster.AssetExecutionContext) -> None:
if skip_unconfigured(context):
return
- _tag_dagster_queries(context, query_type="inbox_ranking_report_state")
+ tag_dagster_queries(context, query_type="inbox_ranking_report_state")
partition_key = context.partition_key
_, snapshot_end = snapshot_bounds(partition_key)
snapshot_date = datetime.date.fromisoformat(partition_key)
@@ -1036,17 +1035,12 @@ def inbox_report_reviewers(context: dagster.AssetExecutionContext) -> None:
)
-def _snapshot_report_embeddings(
- context: dagster.AssetExecutionContext, table: str, rendering: str, query_type: str
-) -> None:
- """Point-in-time snapshot of one embedding rendering, shared by the report and title assets."""
- if skip_unconfigured(context):
- return
- _tag_dagster_queries(context, query_type=query_type)
- partition_key = context.partition_key
+def report_embeddings_table(partition_key: str, rendering: str, consent_team_ids: Collection[int]) -> pa.Table:
+ """Point-in-time snapshot of one embedding rendering as of the end of dt=D, for the teams in
+ `consent_team_ids`. Every region's ClickHouse holds only its own reports, so a dag in each region
+ reads its own vectors through this function."""
_, snapshot_end = snapshot_bounds(partition_key)
snapshot_date = datetime.date.fromisoformat(partition_key)
- consent_team_ids = training_consent_team_ids()
# With no consenting team there is no row to read, so the query is not sent.
results = (
@@ -1075,7 +1069,6 @@ def _snapshot_report_embeddings(
# This is the widest thing the dag holds — a 1536-float vector per live report, fleet-wide — so
# the intermediate list of row dicts that from_pylist would need is a whole extra copy of it.
row_count = len(results)
- tombstones = 0
report_ids: list[str] = []
team_ids: list[int] = []
embeddings: list[list[float] | None] = []
@@ -1084,8 +1077,6 @@ def _snapshot_report_embeddings(
for index in range(row_count):
team_id, document_id, embedding, is_tombstone, inserted_at = results[index]
results[index] = ()
- if is_tombstone:
- tombstones += 1
report_ids.append(str(document_id))
team_ids.append(int(team_id))
# Tombstoned vectors embed the fixed retraction text, not report content, so they are
@@ -1095,9 +1086,7 @@ def _snapshot_report_embeddings(
tombstone_flags.append(bool(is_tombstone))
del results
- # An empty result still writes an object: a backfilled day from before the rendering was
- # emitted must read as "present, zero rows" rather than as a missing partition.
- arrow_table = pa.Table.from_pydict(
+ return pa.Table.from_pydict(
{
"snapshot_date": [snapshot_date] * row_count,
"report_id": report_ids,
@@ -1110,13 +1099,27 @@ def _snapshot_report_embeddings(
schema=EMBEDDINGS_SCHEMA,
)
+
+def _snapshot_report_embeddings(
+ context: dagster.AssetExecutionContext, table: str, rendering: str, query_type: str
+) -> None:
+ """Write one embedding rendering's snapshot, shared by the report and title assets."""
+ if skip_unconfigured(context):
+ return
+ tag_dagster_queries(context, query_type=query_type)
+ partition_key = context.partition_key
+ consent_team_ids = training_consent_team_ids()
+ # An empty result still writes an object: a backfilled day from before the rendering was
+ # emitted must read as "present, zero rows" rather than as a missing partition.
+ arrow_table = report_embeddings_table(partition_key, rendering, consent_team_ids)
+
bucket = dataset_bucket()
key = partition_object_key(settings.INBOX_RANKING_DATASET_S3_PREFIX, table, partition_key)
write_parquet(s3_client(), bucket, key, arrow_table)
context.add_output_metadata(
{
- "rows": dagster.MetadataValue.int(row_count),
- "tombstones": dagster.MetadataValue.int(tombstones),
+ "rows": dagster.MetadataValue.int(arrow_table.num_rows),
+ "tombstones": dagster.MetadataValue.int(pc.sum(arrow_table.column("is_tombstone")).as_py() or 0),
"training_consent_teams": dagster.MetadataValue.int(len(consent_team_ids)),
"s3_key": dagster.MetadataValue.text(f"s3://{bucket}/{key}"),
}
@@ -1186,7 +1189,7 @@ def inbox_signal_embeddings(context: dagster.AssetExecutionContext) -> None:
"""
if skip_unconfigured(context):
return
- _tag_dagster_queries(context, query_type="inbox_ranking_signal_embeddings")
+ tag_dagster_queries(context, query_type="inbox_ranking_signal_embeddings")
partition_key = context.partition_key
window_start, window_end = snapshot_bounds(partition_key)
snapshot_date = datetime.date.fromisoformat(partition_key)
@@ -1299,7 +1302,7 @@ def inbox_signal_embeddings(context: dagster.AssetExecutionContext) -> None:
def inbox_report_labels(context: dagster.AssetExecutionContext) -> None:
if skip_unconfigured(context):
return
- _tag_dagster_queries(context, query_type="inbox_ranking_labels")
+ tag_dagster_queries(context, query_type="inbox_ranking_labels")
partition_key = context.partition_key
_, snapshot_end = snapshot_bounds(partition_key)
team = labels_team()
@@ -1355,7 +1358,7 @@ def inbox_user_report_interactions(context: dagster.AssetExecutionContext) -> No
"""
if skip_unconfigured(context):
return
- _tag_dagster_queries(context, query_type="inbox_ranking_user_interactions")
+ tag_dagster_queries(context, query_type="inbox_ranking_user_interactions")
partition_key = context.partition_key
_, snapshot_end = snapshot_bounds(partition_key)
team = labels_team()
@@ -1459,16 +1462,13 @@ def inbox_project_vectors(context: dagster.AssetExecutionContext) -> None:
key = partition_object_key(prefix, PROJECT_VECTORS_TABLE, partition_key)
write_parquet(client, bucket, key, table, schema_version=PROJECT_VECTORS_SCHEMA_VERSION)
- counts = table.select(["kind", "n_reports"]).to_pylist()
context.add_output_metadata(
{
"rows": dagster.MetadataValue.int(table.num_rows),
"teams": dagster.MetadataValue.int(len(set(table.column("team_id").to_pylist()))),
**{
- f"teams_with_{kind}_history": dagster.MetadataValue.int(
- sum(1 for row in counts if row["kind"] == kind and row["n_reports"] >= MIN_HISTORY_REPORTS)
- )
- for kind in VECTOR_KINDS
+ f"teams_with_{kind}_history": dagster.MetadataValue.int(count)
+ for kind, count in teams_with_history(table).items()
},
"excluded_no_training_consent_reports": dagster.MetadataValue.int(excluded_reports),
"s3_key": dagster.MetadataValue.text(f"s3://{bucket}/{key}"),
diff --git a/products/signals/dags/inbox_ranking/dataset/project_vectors.py b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
index 27f8e67a8ab0..86abcc68644a 100644
--- a/products/signals/dags/inbox_ranking/dataset/project_vectors.py
+++ b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
@@ -96,6 +96,15 @@ def project_vectors(labels: pa.Table, embeddings: pa.Table, as_of: datetime.date
return pa.Table.from_pylist(rows, schema=PROJECT_VECTORS_SCHEMA)
+def teams_with_history(table: pa.Table) -> dict[str, int]:
+ """How many teams of a `project_vectors` table have enough history for a real vector, per kind."""
+ counts = table.select(["kind", "n_reports"]).to_pylist()
+ return {
+ kind: sum(1 for row in counts if row["kind"] == kind and row["n_reports"] >= MIN_HISTORY_REPORTS)
+ for kind in VECTOR_KINDS
+ }
+
+
def _usable_embeddings(embeddings: pa.Table, as_of: datetime.datetime) -> pa.Table:
"""The rows with a full-width, live vector that landed before `as_of`.
diff --git a/products/signals/dags/inbox_ranking/mirror/dag.py b/products/signals/dags/inbox_ranking/mirror/dag.py
new file mode 100644
index 000000000000..2f9c76a01353
--- /dev/null
+++ b/products/signals/dags/inbox_ranking/mirror/dag.py
@@ -0,0 +1,195 @@
+"""Project vectors for the mirror region, built in that region.
+
+The mirror region (EU) trains nothing: the training region (US) mirrors its served models there. A
+served model can read project vectors, but those are per-team means of report vectors, and each
+region's report vectors exist only in its own ClickHouse. So this region builds its own day with the
+same `project_vectors()` the training region uses:
+
+ report vectors this region's ClickHouse, through `report_embeddings_table`
+ UI engagement the training region's `inbox_engagement_export` in this region's object store
+
+The output lands in this region's object store at `serving_project_vectors_key`, which the served
+manifest can name.
+"""
+
+import io
+import datetime
+
+import dagster
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from posthog import settings
+from posthog.dataclasses import frozen
+from posthog.storage import object_storage
+from posthog.storage.object_storage import ObjectStorageClient
+
+from products.signals.backend.ranking.serving_manifest import serving_engagement_key, serving_project_vectors_key
+from products.signals.backend.report_embeddings import EMBEDDING_RENDERING_TITLE_SUMMARY
+from products.signals.dags.inbox_ranking.common import (
+ ROW_COUNT_METADATA_KEY,
+ SCHEMA_VERSION_METADATA_KEY,
+ SNAPSHOT_DATE_METADATA_KEY,
+ owner_tags,
+ snapshot_bounds,
+)
+from products.signals.dags.inbox_ranking.consent import training_consent_team_ids
+from products.signals.dags.inbox_ranking.dataset.dag import (
+ COMMON_ASSET_KWARGS,
+ ENGAGEMENT_EXPORT_SCHEMA_VERSION,
+ report_embeddings_table,
+ tag_dagster_queries,
+)
+from products.signals.dags.inbox_ranking.dataset.project_vectors import (
+ LABEL_COLUMNS,
+ PROJECT_VECTORS_SCHEMA_VERSION,
+ project_vectors,
+ teams_with_history,
+)
+
+MIRROR_PROJECT_VECTORS_ASSET = "inbox_mirror_project_vectors"
+
+# An older export only misses the newest engagement, so a late training-region run costs a little
+# history. Past this age the vectors drift too far from what training read, so the asset fails.
+ENGAGEMENT_EXPORT_MAX_AGE_DAYS = 2
+ENGAGEMENT_EXPORT_DATE_METADATA_KEY = "engagement-export-date"
+
+
+@frozen
+class EngagementExport:
+ export_date: str
+ age_days: int
+ labels: pa.Table
+
+
+def read_engagement_export(
+ storage: ObjectStorageClient, bucket: str, prefix: str, partition_key: str
+) -> EngagementExport | None:
+ """The newest engagement export at or before dt=D and at most `ENGAGEMENT_EXPORT_MAX_AGE_DAYS` old.
+
+ The export is a cumulative snapshot, so an older day is a subset of the newer one. An export with
+ another schema version fails rather than feeding columns this reader does not know."""
+ partition_day = datetime.date.fromisoformat(partition_key)
+ for age_days in range(ENGAGEMENT_EXPORT_MAX_AGE_DAYS + 1):
+ export_date = (partition_day - datetime.timedelta(days=age_days)).isoformat()
+ key = serving_engagement_key(prefix, export_date)
+ head = storage.head_object_strict(bucket, key)
+ if head is None:
+ continue
+ schema_version = head.get("Metadata", {}).get(SCHEMA_VERSION_METADATA_KEY)
+ if schema_version != str(ENGAGEMENT_EXPORT_SCHEMA_VERSION):
+ raise dagster.Failure(
+ f"engagement export {key} has schema version {schema_version}, "
+ f"this reader knows {ENGAGEMENT_EXPORT_SCHEMA_VERSION}"
+ )
+ body = storage.read_bytes(bucket, key)
+ if body is None:
+ continue
+ labels = pq.read_table(pa.BufferReader(body), columns=list(LABEL_COLUMNS))
+ return EngagementExport(export_date=export_date, age_days=age_days, labels=labels)
+ return None
+
+
[email protected](name=MIRROR_PROJECT_VECTORS_ASSET, **COMMON_ASSET_KWARGS)
+def inbox_mirror_project_vectors(context: dagster.AssetExecutionContext) -> None:
+ """Per-team context vectors of this region as of the partition end, for a report born on the next day."""
+ partition_key = context.partition_key
+ _, snapshot_end = snapshot_bounds(partition_key)
+ prefix = settings.INBOX_RANKING_DATASET_S3_PREFIX
+ bucket = settings.OBJECT_STORAGE_BUCKET
+ storage = object_storage.object_storage_client()
+
+ export = read_engagement_export(storage, bucket, prefix, partition_key)
+ if export is None:
+ raise dagster.Failure(
+ f"no engagement export for {partition_key} or the {ENGAGEMENT_EXPORT_MAX_AGE_DAYS} days before it; "
+ "check the training region's inbox_engagement_export"
+ )
+ if export.age_days:
+ context.log.warning(
+ f"engagement export for {partition_key} is missing; using {export.export_date}, "
+ f"so engagement after that day is not in the vectors"
+ )
+
+ tag_dagster_queries(context, query_type="inbox_ranking_mirror_report_embeddings")
+ # The same consent rule as the training region, decided by this region for its own teams.
+ consent_team_ids = training_consent_team_ids()
+ embeddings = report_embeddings_table(partition_key, EMBEDDING_RENDERING_TITLE_SUMMARY, consent_team_ids)
+ embedding_rows = embeddings.num_rows
+ table = project_vectors(export.labels, embeddings, snapshot_end)
+ del embeddings
+
+ buffer = io.BytesIO()
+ pq.write_table(table, buffer, compression="zstd")
+ content = buffer.getvalue()
+ key = serving_project_vectors_key(prefix, partition_key)
+ storage.write(
+ bucket=bucket,
+ key=key,
+ content=content,
+ extras={
+ "Metadata": {
+ ROW_COUNT_METADATA_KEY: str(table.num_rows),
+ SNAPSHOT_DATE_METADATA_KEY: partition_key,
+ SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),
+ ENGAGEMENT_EXPORT_DATE_METADATA_KEY: export.export_date,
+ }
+ },
+ )
+ context.add_output_metadata(
+ {
+ "rows": dagster.MetadataValue.int(table.num_rows),
+ "teams": dagster.MetadataValue.int(len(set(table.column("team_id").to_pylist()))),
+ **{
+ f"teams_with_{kind}_history": dagster.MetadataValue.int(count)
+ for kind, count in teams_with_history(table).items()
+ },
+ "embedding_rows": dagster.MetadataValue.int(embedding_rows),
+ "engagement_rows": dagster.MetadataValue.int(export.labels.num_rows),
+ "engagement_export_date": dagster.MetadataValue.text(export.export_date),
+ "engagement_export_age_days": dagster.MetadataValue.int(export.age_days),
+ "training_consent_teams": dagster.MetadataValue.int(len(consent_team_ids)),
+ "bytes": dagster.MetadataValue.int(len(content)),
+ "s3_key": dagster.MetadataValue.text(f"s3://{bucket}/{key}"),
+ }
+ )
+
+
+inbox_ranking_mirror_job = dagster.define_asset_job(
+ name="inbox_ranking_mirror_job",
+ selection=[MIRROR_PROJECT_VECTORS_ASSET],
+ tags={
+ **owner_tags,
+ "dagster/max_runtime": str(60 * 60),
+ # The embeddings snapshot holds a 1536-float vector per live report of this region, so the
+ # pod's peak memory grows with the inventory, the same as the training region's dataset job.
+ "dagster-k8s/config": {
+ "container_config": {
+ "resources": {
+ "requests": {"memory": "8Gi"},
+ "limits": {"memory": "24Gi"},
+ }
+ }
+ },
+ },
+)
+
+
+# After the training region's dataset run, which starts at 02:30 UTC and is capped at two hours, so
+# the export of the same day is normally in place.
+ cron_schedule="30 5 * * *",
+ job=inbox_ranking_mirror_job,
+ execution_timezone="UTC",
+ default_status=dagster.DefaultScheduleStatus.RUNNING
+ if settings.CLOUD_DEPLOYMENT == "EU"
+ else dagster.DefaultScheduleStatus.STOPPED,
+ tags=owner_tags,
+)
+def inbox_ranking_mirror_schedule(
+ context: dagster.ScheduleEvaluationContext,
+) -> dagster.RunRequest | dagster.SkipReason:
+ if not settings.INBOX_RANKING_MIRROR_PROJECT_VECTORS_ENABLED:
+ return dagster.SkipReason("INBOX_RANKING_MIRROR_PROJECT_VECTORS_ENABLED is not set")
+ previous_day = context.scheduled_execution_time.date() - datetime.timedelta(days=1)
+ return dagster.RunRequest(partition_key=previous_day.isoformat(), run_key=previous_day.isoformat())
diff --git a/products/signals/dags/inbox_ranking/tests/test_dataset.py b/products/signals/dags/inbox_ranking/tests/test_dataset.py
index 9cd8e2503854..64a75f5977db 100644
--- a/products/signals/dags/inbox_ranking/tests/test_dataset.py
+++ b/products/signals/dags/inbox_ranking/tests/test_dataset.py
@@ -570,7 +570,7 @@ class TestSpineInclusion(BaseTest):
with (
patch.object(dag, "skip_unconfigured", lambda context: False),
- patch.object(dag, "_tag_dagster_queries", lambda context, query_type: None),
+ patch.object(dag, "tag_dagster_queries", lambda context, query_type: None),
# A label event that names an opted-out report must not pull it back into the spine.
patch.object(dag, "labels_team", lambda: self.team),
patch.object(dag, "hogql_rows", lambda *args, **kwargs: [(excluded[0],)]),
@@ -1207,7 +1207,7 @@ def _run_embeddings_asset(monkeypatch, asset, rows, consent_team_ids=frozenset({
monkeypatch.setattr(dag, "sync_execute", fake_sync_execute)
monkeypatch.setattr(dag, "write_parquet", fake_write_parquet)
monkeypatch.setattr(dag, "skip_unconfigured", lambda context: False)
- monkeypatch.setattr(dag, "_tag_dagster_queries", lambda context, query_type: None)
+ monkeypatch.setattr(dag, "tag_dagster_queries", lambda context, query_type: None)
monkeypatch.setattr(dag, "dataset_bucket", lambda: "test-bucket")
monkeypatch.setattr(dag, "s3_client", lambda: None)
monkeypatch.setattr(dag, "read_parquet_if_exists", lambda *args, **kwargs: None)
diff --git a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
index 2c7583f17d93..830e93fb6724 100644
--- a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
+++ b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
@@ -31,6 +31,7 @@ from products.signals.dags.inbox_ranking.dataset.project_vectors import (
PROJECT_VECTORS_SCHEMA_VERSION,
project_vectors,
)
+from products.signals.dags.inbox_ranking.mirror import dag as mirror_dag
from products.signals.dags.inbox_ranking.multihead.prototype import project_context
AS_OF = datetime.datetime(2026, 7, 30, tzinfo=datetime.UTC)
@@ -277,3 +278,102 @@ def test_the_engagement_export_sends_only_remote_reports_to_the_mirror(
assert exported.to_pylist() == [
dict.fromkeys(LABEL_COLUMNS) | {"report_id": REMOTE_REPORT_ID, "first_opened_at": BEFORE}
]
+
+
+class _RegionStorage:
+ def __init__(self, objects: dict[str, tuple[bytes, dict[str, str]]]) -> None:
+ self.objects = objects
+ self.writes: list[dict[str, Any]] = []
+
+ def head_object_strict(self, bucket: str, file_key: str) -> dict | None:
+ stored = self.objects.get(file_key)
+ return None if stored is None else {"Metadata": stored[1]}
+
+ def read_bytes(self, bucket: str, key: str, *, missing_ok: bool = False) -> bytes | None:
+ stored = self.objects.get(key)
+ return None if stored is None else stored[0]
+
+ def write(self, bucket: str, key: str, content: bytes, extras: dict | None) -> None:
+ self.writes.append({"bucket": bucket, "key": key, "content": content, "extras": extras})
+
+
+def _export_object(engagements: list[tuple[int, int, str, datetime.datetime]], schema_version: int) -> tuple:
+ buffer = io.BytesIO()
+ pq.write_table(_labels(engagements).select(list(LABEL_COLUMNS)), buffer)
+ return buffer.getvalue(), {common.SCHEMA_VERSION_METADATA_KEY: str(schema_version)}
+
+
+ "export_ages,schema_version,expected_export_age",
+ [
+ ([0, 1], dag.ENGAGEMENT_EXPORT_SCHEMA_VERSION, 0),
+ ([1], dag.ENGAGEMENT_EXPORT_SCHEMA_VERSION, 1),
+ ([mirror_dag.ENGAGEMENT_EXPORT_MAX_AGE_DAYS + 1], dag.ENGAGEMENT_EXPORT_SCHEMA_VERSION, None),
+ ([0], dag.ENGAGEMENT_EXPORT_SCHEMA_VERSION + 1, None),
+ ],
+ ids=["fresh", "one_day_late", "too_stale", "unknown_schema"],
+)
+def test_the_mirror_region_builds_its_vectors_from_its_embeddings_and_the_export(
+ monkeypatch: pytest.MonkeyPatch, export_ages: list[int], schema_version: int, expected_export_age: int | None
+) -> None:
+ partition_day = AS_OF.date() - datetime.timedelta(days=1)
+ reports = [(team_id, index, 1.0, BEFORE) for team_id in (5, 9) for index in range(MIN_HISTORY_REPORTS)]
+ engagements = [(5, index, "first_opened_at", BEFORE) for index in range(MIN_HISTORY_REPORTS)]
+ storage = _RegionStorage(
+ {
+ serving_engagement_key(
+ "inbox_ranking", (partition_day - datetime.timedelta(days=age)).isoformat()
+ ): _export_object(engagements, schema_version)
+ for age in export_ages
+ }
+ )
+ read_consent: list[frozenset[int]] = []
+
+ def fake_report_embeddings_table(partition_key: str, rendering: str, consent_team_ids: frozenset[int]) -> pa.Table:
+ read_consent.append(consent_team_ids)
+ return _embeddings([report for report in reports if report[0] in consent_team_ids])
+
+ monkeypatch.setattr(mirror_dag.object_storage, "object_storage_client", lambda: storage)
+ monkeypatch.setattr(mirror_dag, "report_embeddings_table", fake_report_embeddings_table)
+ monkeypatch.setattr(mirror_dag, "tag_dagster_queries", lambda context, query_type: None)
+ monkeypatch.setattr(mirror_dag, "training_consent_team_ids", lambda: frozenset({5}))
+ monkeypatch.setattr(mirror_dag.settings, "INBOX_RANKING_DATASET_S3_PREFIX", "inbox_ranking")
+ monkeypatch.setattr(mirror_dag.settings, "OBJECT_STORAGE_BUCKET", "region-bucket")
+ context = dagster.build_asset_context(partition_key=partition_day.isoformat())
+
+ if expected_export_age is None:
+ with pytest.raises(dagster.Failure):
+ mirror_dag.inbox_mirror_project_vectors(context)
+ assert storage.writes == []
+ return
+ mirror_dag.inbox_mirror_project_vectors(context)
+
+ assert read_consent == [frozenset({5})]
+ (write,) = storage.writes
+ assert write["key"] == serving_project_vectors_key("inbox_ranking", partition_day.isoformat())
+ export_date = (partition_day - datetime.timedelta(days=expected_export_age)).isoformat()
+ written = pq.read_table(io.BytesIO(write["content"]))
+ assert write["extras"]["Metadata"] == {
+ common.ROW_COUNT_METADATA_KEY: str(written.num_rows),
+ common.SNAPSHOT_DATE_METADATA_KEY: partition_day.isoformat(),
+ common.SCHEMA_VERSION_METADATA_KEY: str(PROJECT_VECTORS_SCHEMA_VERSION),
+ mirror_dag.ENGAGEMENT_EXPORT_DATE_METADATA_KEY: export_date,
+ }
+ assert written.schema == PROJECT_VECTORS_SCHEMA
+ rows = _rows_by_key(written)
+ assert {team_id for team_id, _ in rows} == {5}
+ assert rows[(5, "open")]["n_reports"] == MIN_HISTORY_REPORTS
+ assert set(written.column("as_of").to_pylist()) == {AS_OF}
+
+
+def test_the_mirror_schedule_skips_until_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(mirror_dag.settings, "INBOX_RANKING_MIRROR_PROJECT_VECTORS_ENABLED", False)
+ context = dagster.build_schedule_context(scheduled_execution_time=datetime.datetime(2026, 7, 30, 5, 30))
+
+ assert isinstance(mirror_dag.inbox_ranking_mirror_schedule(context), dagster.SkipReason)
+
+ monkeypatch.setattr(mirror_dag.settings, "INBOX_RANKING_MIRROR_PROJECT_VECTORS_ENABLED", True)
+ run_request = mirror_dag.inbox_ranking_mirror_schedule(context)
+
+ assert isinstance(run_request, dagster.RunRequest)
+ assert run_request.partition_key == "2026-07-29"