Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): publish daily inbox user vectors to serving storage (#125893)

posthog[bot]8a87cd5cbbff
diff --git a/posthog/dags/locations/signals.py b/posthog/dags/locations/signals.py
index f48c1cc3a5c7..d8cc2e852657 100644
--- a/posthog/dags/locations/signals.py
+++ b/posthog/dags/locations/signals.py
@@ -25,6 +25,7 @@ if is_inbox_ranking_registered():
inbox_ranking_dataset.inbox_report_reviewers,
inbox_ranking_dataset.inbox_user_report_interactions,
inbox_ranking_dataset.inbox_project_vectors,
+ inbox_ranking_dataset.inbox_user_vectors,
inbox_ranking_dataset.inbox_engagement_export,
inbox_ranking_dataset.inbox_activity_summary_vectors,
inbox_ranking_training.inbox_ranking_serving_manifest,
diff --git a/products/signals/backend/ranking/model_store.py b/products/signals/backend/ranking/model_store.py
index 5fede0dbe64b..6c4932745e94 100644
--- a/products/signals/backend/ranking/model_store.py
+++ b/products/signals/backend/ranking/model_store.py
@@ -10,13 +10,15 @@ model is cached in the process by key with no invalidation. The manifest itself
call, because a promotion changes it in place.
A model that reads its team's project vectors reads them from the one day the manifest names
-(`load_project_vectors`). The sweep reads that day once per pass, for the teams it scores.
+(`load_project_vectors`). The sweep reads that day once per pass, for the teams it scores. User
+vectors follow the same path (`load_user_vectors`), for the users a pass scores.
"""
from __future__ import annotations
import io
import json
+import datetime
import threading
from collections import OrderedDict
from collections.abc import Collection, Mapping, Sequence
@@ -77,6 +79,8 @@ ONNX_HAS_HISTORY_INPUT = "has_history"
ONNX_PROJECT_VECTOR_KINDS_KEY = "project_vector_kinds"
# The `inbox_project_vectors` columns `project_inputs` reads.
PROJECT_VECTORS_COLUMNS = ["team_id", "kind", "vector", "n_reports"]
+# The `inbox_user_vectors` columns `load_user_vectors` reads.
+USER_VECTORS_COLUMNS = ["organization_id", "user_distinct_id", "kind", "vector", "n_reports", "last_at", "user_hash"]
# Threads one ONNX session may use for one operator. The sweep shares the worker with other
# activities, so a model must not take every core.
ONNX_INTRA_OP_THREADS = 2
@@ -399,6 +403,64 @@ def load_project_vectors(manifest: ServingManifest, team_ids: Collection[int]) -
return ProjectVectors(available=True, by_team=by_team)
+@frozen
+class UserKindVector:
+ vector: np.ndarray
+ n_reports: int
+ last_at: datetime.datetime
+
+
+@frozen
+class UserVectors:
+ """The user vectors one pass reads, for the users it scores."""
+
+ # False when the manifest names no object or the object is missing, so no user has vectors.
+ available: bool
+ # (organization id, user distinct id) to the user's vector of each kind with history. A user
+ # without history has no entry.
+ by_user: Mapping[tuple[str, str], Mapping[str, UserKindVector]]
+ # The same key to the hash of the user's kinds, counts and vectors. It changes only when one of them changes.
+ hashes: Mapping[tuple[str, str], str]
+
+
+def load_user_vectors(manifest: ServingManifest, users: Collection[tuple[str, str]]) -> UserVectors:
+ """The vectors of `users`, each an (organization id, user distinct id), from the day the manifest
+ names, read once.
+
+ Only the rows of `users` are kept. A missing object is no vectors at all rather than an empty
+ day, like `load_project_vectors`.
+ """
+ key = manifest.user_vectors
+ wanted = {(str(organization_id), str(user)) for organization_id, user in users}
+ body = None if key is None or not wanted else object_storage.read_bytes(key, missing_ok=True)
+ if body is None:
+ if key is not None and wanted:
+ logger.warning("inbox_ranking_user_vectors_missing", user_vectors_key=key)
+ return UserVectors(available=False, by_user={}, hashes={})
+ # Both filters reach the Parquet reader, so only rows of the requested organizations and users are decoded.
+ rows = pd.read_parquet(
+ io.BytesIO(body),
+ columns=USER_VECTORS_COLUMNS,
+ filters=[
+ ("organization_id", "in", sorted({organization_id for organization_id, _ in wanted})),
+ ("user_distinct_id", "in", sorted({user for _, user in wanted})),
+ ],
+ )
+ by_user: dict[tuple[str, str], dict[str, UserKindVector]] = {}
+ hashes: dict[tuple[str, str], str] = {}
+ for row in rows.itertuples(index=False):
+ user = (row.organization_id, row.user_distinct_id)
+ if user not in wanted:
+ continue
+ by_user.setdefault(user, {})[row.kind] = UserKindVector(
+ vector=np.asarray(row.vector, dtype=np.float32),
+ n_reports=int(row.n_reports),
+ last_at=row.last_at.to_pydatetime(),
+ )
+ hashes[user] = row.user_hash
+ return UserVectors(available=True, by_user=by_user, hashes=hashes)
+
+
def served_override_rejection(serving: ServingSet, key: str) -> str | None:
"""Why the model `key` cannot take the served role this pass, or None when it can.
diff --git a/products/signals/backend/ranking/serving_manifest.py b/products/signals/backend/ranking/serving_manifest.py
index 8add20bc0eb0..b98445b7ae4f 100644
--- a/products/signals/backend/ranking/serving_manifest.py
+++ b/products/signals/backend/ranking/serving_manifest.py
@@ -53,6 +53,7 @@ HEAD_FILE_SUFFIX = ".ubj"
MANIFEST_FILE = "manifest.json"
METADATA_FILE = "metadata.json"
PROJECT_VECTORS_FILE = "project_vectors.parquet"
+USER_VECTORS_FILE = "user_vectors.parquet"
ENGAGEMENT_FILE = "engagement.parquet"
_SERVING_SEGMENT = "serving"
@@ -62,6 +63,8 @@ _SERVING_SEGMENT = "serving"
# version key is the one `write_parquet` stamps.
PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY = "feature-schema-version"
PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY = "engagement-export-date"
+# The same key, stamped on the serving copy of a user vectors day.
+USER_VECTORS_SCHEMA_VERSION_METADATA_KEY = PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY
def serving_manifest_key(prefix: str) -> str:
@@ -80,6 +83,11 @@ def serving_project_vectors_key(prefix: str, partition_key: str) -> str:
return f"{prefix}/{_SERVING_SEGMENT}/project_vectors/{partition_key}/{PROJECT_VECTORS_FILE}"
+def serving_user_vectors_key(prefix: str, partition_key: str) -> str:
+ """Where one day of `inbox_user_vectors` sits. Keyed by its `dt`, like the project vectors."""
+ return f"{prefix}/{_SERVING_SEGMENT}/user_vectors/{partition_key}/{USER_VECTORS_FILE}"
+
+
def serving_engagement_key(prefix: str, partition_key: str) -> str:
"""Where one day of the engagement export for another region's reports sits. The US dataset dag
writes it to the mirror store, and the mirror region reads it to build its own project vectors."""
@@ -176,6 +184,13 @@ class ServingManifest(BaseModel):
"Null when no entry reads them, or when this store has no copy, and then those entries score nothing."
),
)
+ user_vectors: str | None = Field(
+ default=None,
+ description=(
+ "Object key of the one `inbox_user_vectors` day this store holds. "
+ "Null when this store has no copy. No model reads it yet, so it never blocks a publish."
+ ),
+ )
@field_validator("manifest_version")
@classmethod
diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.md
index 67683a596f2f..cacbf478c005 100644
--- a/products/signals/dags/inbox_ranking/AGENTS.md
+++ b/products/signals/dags/inbox_ranking/AGENTS.md
@@ -8,6 +8,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- `inbox_report_title_embeddings` is a **leaf**: never add it to `inbox_report_model_data`'s `deps` or give that join a title column, or a failure in the title snapshot fails the training table. Its own `deps=[MODEL_DATA_TABLE]` exists for memory, not data — two full sets of 1536-float vectors must not be held at once in the run pod. Raising the pod memory limit is not the first resort.
- `inbox_user_report_interactions` is a **leaf** with its own `USER_INTERACTIONS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`, and never stamp it with `FEATURE_SCHEMA_VERSION`, or a user-grain change sets off the labels refresh sensor. It writes a row only for a report in this region's Postgres whose team is in `training_consent_team_ids()`.
- `inbox_project_vectors` is a **leaf** with its own `PROJECT_VECTORS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`. Its `deps=[EMBEDDINGS_TABLE, LABELS_TABLE]` keep it after the embeddings snapshot, so the two vector-heavy steps do not share the run pod's memory. `project_vectors` takes a report into a vector only when its vector landed and its UI engagement happened strictly before `as_of`; never widen that to the latest snapshot value. A report born on day D reads the dt=D−1 vectors (see the README).
+- `inbox_user_vectors` is a **leaf** with its own `USER_VECTORS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`. Its `deps=[PROJECT_VECTORS_TABLE, USER_INTERACTIONS_TABLE]` keep it after the project vectors, so two reads of the embeddings snapshot do not share the run pod's memory. It builds its rows through `user_vectors()` with the same consent re-filter and `team_organizations` mapping as the pair lane, so its dt=D rows equal what `pair_user_vectors` rebuilds for a report born on D+1; a test pins that parity. `user_hash` covers each kind's name, `n_reports` and vector only, so a day that adds no report keeps a user's hash.
- `inbox_engagement_export` is a **leaf** with its own `ENGAGEMENT_EXPORT_SCHEMA_VERSION`, read by another region. It writes to the mirror store only under `serving_engagement_key`, because the mirror's write grant covers only `<prefix>/serving/`. Keep its columns exactly `LABEL_COLUMNS` and its rows limited to reports with no `SignalReport` in this region: no team ids, counts, text or vectors leave the region. A mirror failure logs and must not fail the asset. Keep the local-share guard before the write: when the local report lookup finds no local report (`no_local_reports`) or a share below `ENGAGEMENT_EXPORT_MIN_LOCAL_SHARE` (`local_share_collapsed`), the asset writes nothing and still succeeds. Its `inbox_ranking_engagement_export_published` event carries counts only, never a report id or a team id.
- `inbox_mirror_project_vectors` (`mirror/dag.py`) is the only inbox-ranking asset that registers on EU (`is_inbox_ranking_mirror_registered()`). It reads this region's report vectors through `report_embeddings_table` and the engagement export through `serving_engagement_key`, and writes only to this region's object store at `serving_project_vectors_key`. Keep it on the same `project_vectors()` and the same consent rule as `inbox_project_vectors`, so the two regions build their vectors the same way. A missing, too old or unknown-schema export fails the asset rather than writing vectors without engagement.
- `inbox_activity_summary_vectors` is a **leaf** with its own `ACTIVITY_SUMMARY_VECTORS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`, and never select the summary text (`content`). It is the only durable copy of the activity summary vectors once the source TTL passes, so a re-run must keep the rows the source no longer returns (`merge_activity_summary_vectors`). Training reads a user's vector at a moment through `activity_vector_as_of`, never the newest vector of the user.
@@ -39,6 +40,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- The manifest schema and the object keys live in `products/signals/backend/ranking/serving_manifest.py`, next to the feature contract, because the dag writes them and the sweep reads them. Its validators run at write time, so the dag refuses a manifest the sweep could not act on; the entry cap is the `ranking_score` artefact's, so a manifest can never produce a row the artefact refuses.
- The `recent` role keeps the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` versions of the served `model_name`, read from the last 2N partitions, so each version gets several days of online scores. The `challenger` role keeps the newest readable version of each other serving family (`serving_families()`), read from its last 2 partitions. Each one passes the pin check before composition, because one missing file fails the publish. Keep the order under the cap as served, recent, challenger, pinned, cross-family. Each recent entry is one more model per sweep pass, so measure the sweep's duration, peak RSS and scored and deferred counts on the `inbox_ranking_sweep_finished` log before and after a change to N.
- Publishing order is load-bearing. Every model copy finishes before `manifest.json` is written, so a failed copy leaves the previous manifest and the previous models serving. A model version is immutable, so a version whose `metadata.json` is already at the target is not copied again. A run with nothing to publish, such as a run without `served.json`, writes nothing rather than an empty manifest. The served model passes the same check as a pin (`_pin_metadata`) before composition.
+- The manifest asset copies the dt=D `inbox_user_vectors` day to `serving/user_vectors/<D>/` before `manifest.json` and names it in `user_vectors`. No model reads it yet, so a missing day or a failed copy only logs and leaves the key out; it never blocks a publish. Never copy user vectors to the mirror: the mirror's manifest always names none.
- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror, and never read the body of a mirror object: they are per-team data of the deployment that owns the teams. The mirror's manifest names the mirror region's own day only after `_mirror_project_vectors` checks its object metadata (`PROJECT_VECTORS_*_METADATA_KEY` in `serving_manifest.py`) within `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS`. Without such a day it names none, and a served model that reads them keeps the mirror on its previous manifest. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.
- `HEADS` holds only the heads the inbox reads (`HeadRanker` in `backend/ranking/inbox_sort.py`). Add a head there only together with a surface that reads it. The promotion gate skips a champion head that is not in `HEADS`, so a trim does not block promotion.
- `nn_v3` and `nn_with_prio_v2` are trained by the `signals_ml` location (`nn/dag.py`), never by the main one. They share one `inbox_ranking_nn_examples` object, and each has its own candidate and champion asset from `family_assets`. The `nn_v3` assets keep the names `inbox_ranking_nn_model_{candidate,champion}`, so their Dagster history continues. A later family's candidate depends on the previous family's champion, for memory and not for data: two candidates at once can exceed the job's memory limit. A family whose set reads a state column the examples object lacks skips with a warning rather than training on zeros. torch is imported only inside the candidate asset's body (through `nn/train.py` and `multihead/network.py`). `signals_ml` must import without torch, and `test_multihead_network.py` pins that. Its decision goes through `decide_family_champion` in `training/dag.py`, graded through the incumbent's `model.holdout.onnx`. Keep `signals_ml` commented out in `.dagster_home/workspace.yaml`, and keep its schedule (04:30 UTC) running by default on prod US only.
diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md
index e093e232ff6e..c2a9a4e8a0fe 100644
--- a/products/signals/dags/inbox_ranking/README.md
+++ b/products/signals/dags/inbox_ranking/README.md
@@ -42,6 +42,7 @@ s3://<bucket>/<prefix>/
├── inbox_report_reviewers/v1/dt=YYYY-MM-DD/ # one row per suggested reviewer of each spine report
├── inbox_user_report_interactions/v1/dt=YYYY-MM-DD/ # cumulative interactions per (report, person)
├── inbox_project_vectors/v1/dt=YYYY-MM-DD/ # per-team mean report vectors of past engagement, by kind
+├── inbox_user_vectors/v1/dt=YYYY-MM-DD/ # per-user mean report vectors of past engagement, by kind
└── inbox_activity_summary_vectors/v1/dt=YYYY-MM-DD/ # one row per activity summary vector generated during the day
```
@@ -80,6 +81,15 @@ Point-in-time contract:
- The training side can rebuild any past day by calling the function on newer snapshots with an earlier `as_of`. No backfill is needed while the vectors are inside the embedding TTL.
- Serving: when any manifest entry reads project vectors, `inbox_ranking_serving_manifest` copies the dt=D day into the deployment's object store before it writes the manifest. The sweep reads that day once per pass, only for the teams it scores. Its `as_of` is the end of dt=D, so it is never newer than a report scored after the publish. Until the next publish, a report scored on day D+2 still reads dt=D, which is one day older than training reads.
+### User vectors
+
+`inbox_user_vectors` is an (organization, user, kind) leaf with the user vectors of `user_vectors(interactions, embeddings, as_of)` in `dataset/project_vectors.py`.
+The asset calls it with `as_of` = the end of dt=D, on the dt=D interactions and embeddings snapshots, so its rows equal what `pair_user_vectors` rebuilds for a report born on D+1.
+
+- It filters the interactions again by `training_consent_team_ids()`, and maps each team to its organization through `team_organizations`, like the pair lane.
+- Each row carries the `user_hash` of its (organization, user): a hash over each kind's name, `n_reports` and vector. A day that adds no report to a user's history keeps the hash.
+- The manifest asset copies the day to `serving_user_vectors_key(prefix, dt)` and names it in `user_vectors`. The mirror never gets a copy.
+
### Engagement export
`inbox_engagement_export` sends the UI engagement moments of the other region's reports to the mirror store, so that region can build its own project vectors from its own report embeddings.
@@ -184,7 +194,8 @@ That table drops a row 3 months after its `timestamp`, and `UserActivitySummaryV
`inbox_ranking_training_job` runs daily at 05:00 UTC on the same partition definition (gated like the dataset job).
It fits no model: the `nn` families train on the `signals_ml` location (see [The nn family](#the-nn-family)).
The job runs three assets: `inbox_ranking_serving_manifest`, `inbox_ranking_served_scores` and `inbox_ranking_unseen_graded`.
-The manifest depends on the dt=D `inbox_project_vectors` partition, which it copies.
+The manifest depends on the dt=D `inbox_project_vectors` and `inbox_user_vectors` partitions, which it copies.
+The user vectors copy is optional: a missing day leaves `user_vectors` out of the manifest and never blocks the publish.
The models live under the dataset prefix:
```text
diff --git a/products/signals/dags/inbox_ranking/dataset/dag.py b/products/signals/dags/inbox_ranking/dataset/dag.py
index eb92f31d88a1..44ff89985e08 100644
--- a/products/signals/dags/inbox_ranking/dataset/dag.py
+++ b/products/signals/dags/inbox_ranking/dataset/dag.py
@@ -1,6 +1,6 @@
"""Daily modeling dataset for the Self-driving Inbox report-ranking model.
-Ten assets on one daily partition, each writing Parquet under the configured S3 prefix:
+Eleven assets on one daily partition, each writing Parquet under the configured S3 prefix:
inbox_report_state/v1/dt=D/ Postgres spine + report-state columns
inbox_report_embeddings/v1/dt=D/ report_id -> small-1536 vector as of snapshot end
@@ -11,6 +11,7 @@ Ten assets on one daily partition, each writing Parquet under the configured S3
inbox_report_reviewers/v1/dt=D/ one row per suggested reviewer of each spine report
inbox_user_report_interactions/v1/dt=D/ cumulative interactions at (report, person) grain
inbox_project_vectors/v1/dt=D/ per-team mean report vectors of past engagement, by kind
+ inbox_user_vectors/v1/dt=D/ per-user mean report vectors of past engagement, by kind
inbox_activity_summary_vectors/v1/dt=D/ one row per activity summary vector generated during D
inbox_engagement_export writes no dataset object. It sends the dt=D engagement moments of the other
@@ -23,6 +24,8 @@ is a report-grain leaf: nothing joins it, and no model family trains on it. inbo
it to the report tables by report_id. inbox_user_report_interactions is a (report, person) grain
leaf with its own schema version, so nothing joins it either. inbox_project_vectors is a
(team, kind) leaf with its own schema version, built from the embeddings and labels snapshots.
+inbox_user_vectors is an (organization, user, kind) leaf with its own schema version, built from the
+embeddings and interactions snapshots.
inbox_activity_summary_vectors is a (team, user, day) leaf with its own schema version: the durable
copy of the per-user activity summary vectors, which the source table drops after its 3-month TTL.
@@ -142,8 +145,14 @@ from products.signals.dags.inbox_ranking.dataset.project_vectors import (
EMBEDDING_COLUMNS,
LABEL_COLUMNS,
PROJECT_VECTORS_SCHEMA_VERSION,
+ USER_INTERACTION_COLUMNS,
+ USER_VECTORS_SCHEMA_VERSION,
+ history_buckets,
+ prior_views,
project_vectors,
teams_with_history,
+ user_vectors,
+ with_user_hashes,
)
from products.signals.dags.inbox_ranking.dataset.queries import (
ACTIVITY_SUMMARY_VECTORS_QUERY_SETTINGS,
@@ -187,6 +196,7 @@ SIGNAL_EMBEDDINGS_TABLE = "inbox_signal_embeddings"
REVIEWERS_TABLE = "inbox_report_reviewers"
USER_INTERACTIONS_TABLE = "inbox_user_report_interactions"
PROJECT_VECTORS_TABLE = "inbox_project_vectors"
+USER_VECTORS_TABLE = "inbox_user_vectors"
ENGAGEMENT_EXPORT = "inbox_engagement_export"
ACTIVITY_SUMMARY_VECTORS_TABLE = "inbox_activity_summary_vectors"
@@ -1511,6 +1521,77 @@ def inbox_project_vectors(context: dagster.AssetExecutionContext) -> None:
)
+def team_organizations(team_ids: frozenset[int]) -> dict[int, str]:
+ return {
+ team_id: str(organization_id)
+ for team_id, organization_id in Team.objects.filter(id__in=team_ids).values_list("id", "organization_id")
+ }
+
+
+ name=USER_VECTORS_TABLE,
+ # After the project vectors, so the two reads of the embeddings snapshot do not share the run
+ # pod's memory. Nothing joins this asset, so its failure does not fail the training table.
+ deps=[PROJECT_VECTORS_TABLE, USER_INTERACTIONS_TABLE],
+ **COMMON_ASSET_KWARGS,
+)
+def inbox_user_vectors(context: dagster.AssetExecutionContext) -> None:
+ """Per-user context vectors as of the partition end, for a report scored on the next day.
+
+ The rows are the user vectors the pair lane rebuilds through `pair_user_vectors` for a report
+ born on D+1, so serving reads the same inputs that training reads. Each row carries the
+ `user_hash` of its (organization, user).
+ """
+ if skip_unconfigured(context):
+ return
+ partition_key = context.partition_key
+ _, snapshot_end = snapshot_bounds(partition_key)
+ bucket = dataset_bucket()
+ prefix = settings.INBOX_RANKING_DATASET_S3_PREFIX
+ client = s3_client()
+
+ interactions = read_parquet(
+ client,
+ bucket,
+ partition_object_key(prefix, USER_INTERACTIONS_TABLE, partition_key),
+ list(USER_INTERACTION_COLUMNS),
+ )
+ # The snapshot was filtered when it was written. Filter it again, so an opt-out since then counts.
+ consent_team_ids = training_consent_team_ids()
+ consented = interactions.filter(
+ pc.is_in(interactions.column("report_team_id"), value_set=pa.array(sorted(consent_team_ids), type=pa.int64()))
+ )
+ excluded_rows = interactions.num_rows - consented.num_rows
+ del interactions
+ organizations = team_organizations(consent_team_ids)
+ embeddings = read_parquet(
+ client, bucket, partition_object_key(prefix, EMBEDDINGS_TABLE, partition_key), list(EMBEDDING_COLUMNS)
+ )
+ table = with_user_hashes(user_vectors(consented, embeddings, snapshot_end, organizations=organizations))
+ del embeddings
+
+ key = partition_object_key(prefix, USER_VECTORS_TABLE, partition_key)
+ write_parquet(client, bucket, key, table, schema_version=USER_VECTORS_SCHEMA_VERSION)
+ users = set(
+ zip(table.column("organization_id").to_pylist(), table.column("user_distinct_id").to_pylist(), strict=True)
+ )
+ context.add_output_metadata(
+ {
+ "rows": dagster.MetadataValue.int(table.num_rows),
+ "users": dagster.MetadataValue.int(len(users)),
+ "organizations": dagster.MetadataValue.int(len({organization_id for organization_id, _ in users})),
+ **{
+ f"users_with_{name}_prior_views": dagster.MetadataValue.int(count)
+ for name, count in history_buckets(
+ prior_views(consented, snapshot_end, organizations=organizations)
+ ).items()
+ },
+ "excluded_no_training_consent_rows": dagster.MetadataValue.int(excluded_rows),
+ "s3_key": dagster.MetadataValue.text(f"s3://{bucket}/{key}"),
+ }
+ )
+
+
def activity_summary_vectors_table(partition_key: str, consent_team_ids: Collection[int]) -> tuple[pa.Table, int]:
"""The activity summary vectors generated during dt=D for the teams in `consent_team_ids`, one row
per document, and the number of rows skipped because their document_id names no user."""
@@ -1930,6 +2011,7 @@ inbox_ranking_dataset_job = dagster.define_asset_job(
REVIEWERS_TABLE,
USER_INTERACTIONS_TABLE,
PROJECT_VECTORS_TABLE,
+ USER_VECTORS_TABLE,
ENGAGEMENT_EXPORT,
ACTIVITY_SUMMARY_VECTORS_TABLE,
],
diff --git a/products/signals/dags/inbox_ranking/dataset/project_vectors.py b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
index 610b51bd2377..437033bc40af 100644
--- a/products/signals/dags/inbox_ranking/dataset/project_vectors.py
+++ b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
@@ -21,10 +21,14 @@ organization.
Both functions take a report into a vector through the same as-of rule (`known_before`) and the same
embedding gate (`_usable_embeddings`).
+
+`with_user_hashes` gives each (organization, user) one hash over its kinds, counts and vectors, so a
+reader can tell a changed history from an unchanged one without a comparison of the vectors.
"""
+import hashlib
import datetime
-from collections import defaultdict
+from collections import Counter, defaultdict
from collections.abc import Mapping, Sequence
from typing import TypeGuard
@@ -32,6 +36,8 @@ import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
+from posthog.dataclasses import frozen
+
from products.signals.backend.ranking.features import (
EMBEDDING_DIMENSIONS,
MIN_PROJECT_HISTORY_REPORTS,
@@ -66,6 +72,14 @@ _PROJECT_VECTORS_FIELDS: list[tuple[str, pa.DataType]] = [
PROJECT_VECTORS_SCHEMA = pa.schema(_PROJECT_VECTORS_FIELDS)
USER_HISTORY_LIMIT = 20
+
+
+@frozen
+class OrganizationUser:
+ organization_id: str
+ user_distinct_id: str
+
+
# The backend owns the user kinds too, because the pair network reads them in this order.
USER_OPEN, USER_DISMISS = USER_HISTORY_KINDS
# The durable VIEW action row dates an open, because the `Inbox report opened` event misses the
@@ -85,6 +99,18 @@ _USER_VECTORS_FIELDS: list[tuple[str, pa.DataType]] = [
]
USER_VECTORS_SCHEMA = pa.schema(_USER_VECTORS_FIELDS)
+# A leaf with its own columns too, so its schema moves apart from the project vectors'.
+USER_VECTORS_SCHEMA_VERSION = 1
+USER_VECTORS_DAY_SCHEMA = USER_VECTORS_SCHEMA.append(pa.field("user_hash", pa.string()))
+# (name, lowest, highest) prior views per bucket. The within-user grade needs enough rows in the
+# buckets with real history to be readable.
+HISTORY_BUCKETS: tuple[tuple[str, int, int | None], ...] = (
+ ("0", 0, 0),
+ ("1_9", 1, 9),
+ ("10_49", 10, 49),
+ ("50_plus", 50, None),
+)
+
def project_vectors(labels: pa.Table, embeddings: pa.Table, as_of: datetime.datetime) -> pa.Table:
"""One row per (team, kind) for every team with a report vector before `as_of`.
@@ -180,6 +206,52 @@ def user_vectors(
return pa.Table.from_pylist(rows, schema=USER_VECTORS_SCHEMA)
+def with_user_hashes(table: pa.Table) -> pa.Table:
+ """`table`, a `user_vectors` table, with the `user_hash` of each row's (organization, user).
+
+ The hash reads each kind's name, `n_reports` and vector bytes, in kind order. `last_at` and
+ `as_of` stay out, so a day that adds no report to a user's history keeps the user's hash.
+ """
+ rows = table.select(["organization_id", "user_distinct_id", "kind", "n_reports"]).to_pylist()
+ vectors = table.column("vector").to_numpy(zero_copy_only=False)
+ parts: dict[tuple[str, str], list[bytes]] = defaultdict(list)
+ for index in sorted(range(len(rows)), key=lambda i: rows[i]["kind"]):
+ row = rows[index]
+ parts[(row["organization_id"], row["user_distinct_id"])] += [
+ row["kind"].encode(),
+ int(row["n_reports"]).to_bytes(4, "little"),
+ np.asarray(vectors[index], dtype="<f4").tobytes(),
+ ]
+ hashes = {user: hashlib.sha256(b"".join(user_parts)).hexdigest() for user, user_parts in parts.items()}
+ return table.append_column(
+ "user_hash",
+ pa.array([hashes[(row["organization_id"], row["user_distinct_id"])] for row in rows], pa.string()),
+ )
+
+
+def prior_views(
+ interactions: pa.Table, as_of: datetime.datetime, *, organizations: Mapping[int, str]
+) -> Counter[OrganizationUser]:
+ """The views before `as_of` of each (organization, user) in `interactions`. A user with
+ interaction rows and no such view counts 0."""
+ view_column = USER_HISTORY_COLUMNS[USER_OPEN]
+ views: Counter[OrganizationUser] = Counter()
+ for row in interactions.select(["report_team_id", "user_distinct_id", view_column]).to_pylist():
+ organization_id = organizations.get(row["report_team_id"])
+ if organization_id is not None and row["user_distinct_id"] is not None:
+ user = OrganizationUser(organization_id=organization_id, user_distinct_id=row["user_distinct_id"])
+ views[user] += int(known_before(row[view_column], as_of))
+ return views
+
+
+def history_buckets(views: Counter[OrganizationUser]) -> dict[str, int]:
+ """How many users of `views` fall in each of `HISTORY_BUCKETS`."""
+ return {
+ name: sum(1 for count in views.values() if count >= lowest and (highest is None or count <= highest))
+ for name, lowest, highest in HISTORY_BUCKETS
+ }
+
+
def known_before(moment: datetime.datetime | None, as_of: datetime.datetime) -> TypeGuard[datetime.datetime]:
"""The as-of rule of every context vector: a moment counts only when it is strictly before `as_of`."""
return moment is not None and moment < as_of
diff --git a/products/signals/dags/inbox_ranking/nn/pairs.py b/products/signals/dags/inbox_ranking/nn/pairs.py
index b84db8583807..54e4a5f81ad4 100644
--- a/products/signals/dags/inbox_ranking/nn/pairs.py
+++ b/products/signals/dags/inbox_ranking/nn/pairs.py
@@ -17,7 +17,6 @@ user vectors from the dt=D interactions and embeddings snapshots through `user_v
"""
import datetime
-from collections import Counter
from collections.abc import Mapping
import numpy as np
@@ -27,7 +26,6 @@ import pyarrow as pa
import pyarrow.compute as pc
from posthog import settings
-from posthog.models import Team
from products.signals.backend.ranking.features import DAYS_SINCE_LAST_VIEW_CAP, EMBEDDING_DIMENSIONS
from products.signals.dags.inbox_ranking.common import (
@@ -41,15 +39,23 @@ from products.signals.dags.inbox_ranking.common import (
write_parquet,
)
from products.signals.dags.inbox_ranking.consent import training_consent_team_ids
-from products.signals.dags.inbox_ranking.dataset.dag import EMBEDDINGS_TABLE, REVIEWERS_TABLE, USER_INTERACTIONS_TABLE
+from products.signals.dags.inbox_ranking.dataset.dag import (
+ EMBEDDINGS_TABLE,
+ REVIEWERS_TABLE,
+ USER_INTERACTIONS_TABLE,
+ team_organizations,
+)
from products.signals.dags.inbox_ranking.dataset.project_vectors import (
EMBEDDING_COLUMNS,
+ HISTORY_BUCKETS,
UI_ACTION_FIRST_COLUMNS,
USER_DISMISS,
USER_INTERACTION_COLUMNS,
USER_OPEN,
USER_VECTOR_KINDS,
+ OrganizationUser,
known_before,
+ prior_views,
user_vectors,
)
from products.signals.dags.inbox_ranking.nn.dag import NN_ASSET_KWARGS, NN_EXAMPLES_TABLE
@@ -65,14 +71,6 @@ _REPORT_HEAD = HEADS_BY_NAME["open"]
PAIR_HORIZON_DAYS = _REPORT_HEAD.horizon_days
PAIR_HEADS = PAIR_HEAD_NAMES
DAYS_SINCE_VIEW_CAP = DAYS_SINCE_LAST_VIEW_CAP
-# (name, lowest, highest) prior views per bucket. The within-user grade needs enough rows in the
-# buckets with real history to be readable.
-HISTORY_BUCKETS: tuple[tuple[str, int, int | None], ...] = (
- ("0", 0, 0),
- ("1_9", 1, 9),
- ("10_49", 10, 49),
- ("50_plus", 50, None),
-)
REVIEWERS_SOURCE = "reviewers"
IMPRESSIONS_SOURCE = "impressions"
@@ -85,7 +83,6 @@ LABEL_INTERACTION_COLUMNS = (
*UI_ACTION_FIRST_COLUMNS,
)
REVIEWER_COLUMNS = ("report_id", "user_distinct_id")
-_VIEW_COLUMN = "first_viewed_at"
_PAIR_EXAMPLES_FIELDS: list[tuple[str, pa.DataType]] = [
("snapshot_date", pa.date32()),
@@ -153,11 +150,7 @@ def pair_rows(
.drop_columns(["vector"])
.to_pylist()
}
- prior_views: Counter[tuple[str, str]] = Counter()
- for row in users_history.select(["report_team_id", "user_distinct_id", _VIEW_COLUMN]).to_pylist():
- organization_id = organizations.get(row["report_team_id"])
- if organization_id is not None and known_before(row[_VIEW_COLUMN], as_of):
- prior_views[(organization_id, row["user_distinct_id"])] += 1
+ views = prior_views(users_history, as_of, organizations=organizations)
suggested = (
{(row["report_id"], row["user_distinct_id"]) for row in reviewers.select(list(REVIEWER_COLUMNS)).to_pylist()}
if reviewers is not None
@@ -188,7 +181,11 @@ def pair_rows(
),
"user_open_reports": open_vector["n_reports"] if open_vector is not None else 0,
"user_dismiss_reports": dismiss_vector["n_reports"] if dismiss_vector is not None else 0,
- "prior_views": prior_views[(organization_id, user)] if organization_id is not None else 0,
+ "prior_views": (
+ views[OrganizationUser(organization_id=organization_id, user_distinct_id=user)]
+ if organization_id is not None
+ else 0
+ ),
"days_since_last_view": days_since_last_view,
"has_history": open_vector is not None or dismiss_vector is not None,
"is_suggested_reviewer": (
@@ -363,13 +360,6 @@ def build_pair_examples(
}
-def team_organizations(team_ids: frozenset[int]) -> dict[int, str]:
- return {
- team_id: str(organization_id)
- for team_id, organization_id in Team.objects.filter(id__in=team_ids).values_list("id", "organization_id")
- }
-
-
@dagster.asset(
name=NN_PAIR_EXAMPLES_TABLE,
deps=[
diff --git a/products/signals/dags/inbox_ranking/tests/test_dataset.py b/products/signals/dags/inbox_ranking/tests/test_dataset.py
index 7f10e33f74f6..bac4dbd9bdc8 100644
--- a/products/signals/dags/inbox_ranking/tests/test_dataset.py
+++ b/products/signals/dags/inbox_ranking/tests/test_dataset.py
@@ -1349,6 +1349,7 @@ def test_the_leaf_snapshots_run_in_the_job_without_feeding_the_join():
dag.inbox_report_reviewers,
dag.inbox_user_report_interactions,
dag.inbox_project_vectors,
+ dag.inbox_user_vectors,
dag.inbox_engagement_export,
dag.inbox_activity_summary_vectors,
]
@@ -1359,6 +1360,7 @@ def test_the_leaf_snapshots_run_in_the_job_without_feeding_the_join():
dag.REVIEWERS_TABLE,
dag.USER_INTERACTIONS_TABLE,
dag.PROJECT_VECTORS_TABLE,
+ dag.USER_VECTORS_TABLE,
dag.ENGAGEMENT_EXPORT,
dag.ACTIVITY_SUMMARY_VECTORS_TABLE,
):
diff --git a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
index 1d60284ea798..bf25aba6a9ca 100644
--- a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
+++ b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
@@ -20,6 +20,7 @@ from products.signals.backend.ranking.serving_manifest import (
ServingManifestEntry,
serving_engagement_key,
serving_project_vectors_key,
+ serving_user_vectors_key,
)
from products.signals.dags.inbox_ranking import common
from products.signals.dags.inbox_ranking.dataset import dag
@@ -31,12 +32,17 @@ from products.signals.dags.inbox_ranking.dataset.project_vectors import (
PROJECT_VECTORS_SCHEMA,
PROJECT_VECTORS_SCHEMA_VERSION,
USER_HISTORY_LIMIT,
+ USER_VECTOR_KINDS,
+ USER_VECTORS_DAY_SCHEMA,
USER_VECTORS_SCHEMA,
+ USER_VECTORS_SCHEMA_VERSION,
project_vectors,
user_vectors,
+ with_user_hashes,
)
from products.signals.dags.inbox_ranking.mirror import dag as mirror_dag
from products.signals.dags.inbox_ranking.multihead.prototype import project_context
+from products.signals.dags.inbox_ranking.nn.pairs import pair_user_vectors
from products.signals.dags.inbox_ranking.training import (
dag as training_dag,
telemetry,
@@ -235,6 +241,148 @@ def test_the_asset_writes_only_teams_opted_in_to_ai_training(monkeypatch) -> Non
assert set(captured["table"].column("as_of").to_pylist()) == {AS_OF}
+def _run_user_vectors_asset(monkeypatch, interactions: pa.Table, embeddings: pa.Table, consent: frozenset[int]):
+ partition_key = (AS_OF.date() - datetime.timedelta(days=1)).isoformat()
+ snapshots = {
+ common.partition_object_key("inbox_ranking", dag.EMBEDDINGS_TABLE, partition_key): embeddings,
+ common.partition_object_key("inbox_ranking", dag.USER_INTERACTIONS_TABLE, partition_key): interactions,
+ }
+ captured: dict[str, Any] = {}
+
+ def fake_write_parquet(client, bucket, key, table, snapshot_date=None, schema_version=None):
+ captured.update(key=key, table=table, schema_version=schema_version)
+
+ monkeypatch.setattr(dag, "skip_unconfigured", lambda context: False)
+ monkeypatch.setattr(dag, "dataset_bucket", lambda: "test-bucket")
+ monkeypatch.setattr(dag, "s3_client", lambda: None)
+ monkeypatch.setattr(dag, "read_parquet", lambda client, bucket, key, columns=None: snapshots[key].select(columns))
+ monkeypatch.setattr(dag, "write_parquet", fake_write_parquet)
+ monkeypatch.setattr(dag, "training_consent_team_ids", lambda: consent)
+ monkeypatch.setattr(
+ dag, "team_organizations", lambda team_ids: {t: o for t, o in ORGANIZATIONS.items() if t in team_ids}
+ )
+ monkeypatch.setattr(dag.settings, "INBOX_RANKING_DATASET_S3_PREFIX", "inbox_ranking")
+ dag.inbox_user_vectors(dagster.build_asset_context(partition_key=partition_key))
+ assert captured["key"] == common.partition_object_key("inbox_ranking", dag.USER_VECTORS_TABLE, partition_key)
+ assert captured["schema_version"] == USER_VECTORS_SCHEMA_VERSION
+ return captured["table"]
+
+
+def test_the_user_vectors_asset_matches_the_pair_lane_for_a_report_born_the_next_day(monkeypatch) -> None:
+ reports = [(team_id, index, float(team_id + index), BEFORE) for team_id in (2, 3, 7) for index in range(3)]
+ reports.append((2, 9, 50.0, AFTER))
+ views = [
+ (2, 0, "u1", "first_viewed_at", BEFORE),
+ (3, 1, "u1", "first_viewed_at", BEFORE + datetime.timedelta(hours=1)),
+ (2, 2, "u1", "first_ui_dismissed_at", BEFORE),
+ (2, 1, "u1", "first_viewed_at", AFTER),
+ (2, 9, "u1", "first_viewed_at", BEFORE),
+ (3, 2, "u2", "first_viewed_at", BEFORE),
+ (7, 0, "u1", "first_viewed_at", BEFORE),
+ (7, 1, "u3", "first_viewed_at", BEFORE),
+ ]
+ interactions, embeddings = _interactions(views), _embeddings(reports)
+
+ table = _run_user_vectors_asset(monkeypatch, interactions, embeddings, consent=frozenset({2, 3}))
+
+ assert table.schema == USER_VECTORS_DAY_SCHEMA
+ rows = _user_rows(table)
+ # Team 7 has no training consent, so neither its user nor its organization gets a row.
+ assert set(rows) == {("org-a", "u1", "open"), ("org-a", "u1", "dismiss"), ("org-a", "u2", "open")}
+ assert rows[("org-a", "u1", "open")]["n_reports"] == 2
+ pairs = pd.DataFrame(
+ {
+ "snapshot_date": [AS_OF.date()] * 3,
+ "report_team_id": [2, 3, 2],
+ "user_distinct_id": ["u1", "u1", "u2"],
+ }
+ )
+ consented = {t: o for t, o in ORGANIZATIONS.items() if t in (2, 3)}
+ rebuilt = pair_user_vectors(pairs, interactions, embeddings, organizations=consented)
+ for position, user in enumerate(pairs["user_distinct_id"]):
+ for kind_index, kind in enumerate(USER_VECTOR_KINDS):
+ row = rows.get(("org-a", user, kind))
+ expected = np.zeros(EMBEDDING_DIMENSIONS, dtype=np.float32) if row is None else row["vector"]
+ np.testing.assert_array_equal(rebuilt[position, kind_index], expected)
+
+
+def _hashes(table: pa.Table) -> dict[tuple[str, str], str]:
+ return {
+ (row["organization_id"], row["user_distinct_id"]): row["user_hash"]
+ for row in with_user_hashes(table).to_pylist()
+ }
+
+
+ [
+ ("rows_reordered", lambda rows: list(reversed(rows)), False),
+ ("last_at_moved", lambda rows: [{**rows[0], "last_at": AFTER}, *rows[1:]], False),
+ ("count_changed", lambda rows: [{**rows[0], "n_reports": rows[0]["n_reports"] + 1}, *rows[1:]], True),
+ (
+ "vector_changed",
+ lambda rows: [{**rows[0], "vector": [0.5] * EMBEDDING_DIMENSIONS}, *rows[1:]],
+ True,
+ ),
+ ("kind_dropped", lambda rows: rows[1:], True),
+ ]
+)
+def test_the_user_hash_changes_only_when_a_kind_count_or_vector_changes(_name: str, edit, changes: bool) -> None:
+ reports = [(2, index, float(index), BEFORE) for index in range(3)]
+ views = [
+ (2, 0, "u1", "first_viewed_at", BEFORE),
+ (2, 1, "u1", "first_ui_dismissed_at", BEFORE),
+ (2, 2, "u2", "first_viewed_at", BEFORE),
+ ]
+ table = user_vectors(_interactions(views), _embeddings(reports), AS_OF, organizations=ORGANIZATIONS)
+ rows = table.to_pylist()
+ assert [(row["user_distinct_id"], row["kind"]) for row in rows[:2]] == [("u1", "dismiss"), ("u1", "open")]
+
+ before = _hashes(table)
+ after = _hashes(pa.Table.from_pylist(edit(rows), schema=USER_VECTORS_SCHEMA))
+
+ assert len(set(before.values())) == 2
+ assert (after[("org-a", "u1")] != before[("org-a", "u1")]) is changes
+ assert after[("org-a", "u2")] == before[("org-a", "u2")]
+
+
+def test_serving_reads_the_published_user_vectors_of_the_users_it_asks_for(monkeypatch: pytest.MonkeyPatch) -> None:
+ reports = [(2, 0, 1.0, BEFORE), (7, 0, 2.0, BEFORE)]
+ views = [(2, 0, "u1", "first_viewed_at", BEFORE), (7, 0, "u1", "first_viewed_at", BEFORE)]
+ table = with_user_hashes(
+ user_vectors(_interactions(views), _embeddings(reports), AS_OF, organizations=ORGANIZATIONS)
+ )
+ key = serving_user_vectors_key("inbox_ranking", (AS_OF.date() - datetime.timedelta(days=1)).isoformat())
+ buffer = io.BytesIO()
+ pq.write_table(table, buffer)
+ monkeypatch.setattr(
+ model_store.object_storage,
+ "read_bytes",
+ lambda name, bucket=None, *, missing_ok=False: buffer.getvalue() if name == key else None,
+ )
+ entry = ServingManifestEntry(
+ key="nn@2026-07-29",
+ model_name="nn",
+ model_version="2026-07-29",
+ model_kind="onnx",
+ roles=[SERVED_ROLE],
+ prefix="inbox_ranking/serving/models/nn@2026-07-29",
+ heads=["open"],
+ model_file="model.onnx",
+ )
+ manifest = ServingManifest(manifest_version=AS_OF.isoformat(), models=[entry], user_vectors=key)
+
+ served = model_store.load_user_vectors(manifest, [("org-a", "u1"), ("org-a", "u9")])
+ missing = model_store.load_user_vectors(manifest.model_copy(update={"user_vectors": None}), [("org-a", "u1")])
+
+ assert served.available and not missing.available
+ assert set(served.by_user) == {("org-a", "u1")}
+ open_vector = served.by_user[("org-a", "u1")]["open"]
+ assert open_vector.n_reports == 1
+ assert open_vector.last_at == BEFORE
+ np.testing.assert_array_equal(open_vector.vector, np.ones(EMBEDDING_DIMENSIONS, dtype=np.float32))
+ assert served.hashes[("org-a", "u1")] == _hashes(table.drop_columns(["user_hash"]))[("org-a", "u1")]
+
+
def test_serving_reads_the_published_day_as_training_builds_it(monkeypatch: pytest.MonkeyPatch) -> None:
reports = [(2, index, float(index + 1), BEFORE) for index in range(MIN_HISTORY_REPORTS)]
reports += [(3, 0, 7.0, BEFORE)]
diff --git a/products/signals/dags/inbox_ranking/tests/test_training.py b/products/signals/dags/inbox_ranking/tests/test_training.py
index 23ec41e7ac83..148368bc43f4 100644
--- a/products/signals/dags/inbox_ranking/tests/test_training.py
+++ b/products/signals/dags/inbox_ranking/tests/test_training.py
@@ -56,11 +56,20 @@ from products.signals.backend.ranking.serving_manifest import (
serving_manifest_key,
serving_model_prefix,
serving_project_vectors_key,
+ serving_user_vectors_key,
)
from products.signals.backend.ranking.sinks import REPORT_SCORED_EVENT
from products.signals.dags.inbox_ranking.common import partition_object_key
-from products.signals.dags.inbox_ranking.dataset.dag import LABELS_TABLE, PROJECT_VECTORS_TABLE, STATE_TABLE
-from products.signals.dags.inbox_ranking.dataset.project_vectors import PROJECT_VECTORS_SCHEMA_VERSION
+from products.signals.dags.inbox_ranking.dataset.dag import (
+ LABELS_TABLE,
+ PROJECT_VECTORS_TABLE,
+ STATE_TABLE,
+ USER_VECTORS_TABLE,
+)
+from products.signals.dags.inbox_ranking.dataset.project_vectors import (
+ PROJECT_VECTORS_SCHEMA_VERSION,
+ USER_VECTORS_SCHEMA_VERSION,
+)
from products.signals.dags.inbox_ranking.training.calibration import (
BUCKETS,
calibration_buckets,
@@ -2215,14 +2224,14 @@ class _AppObjectStore:
self.metadata = dict(metadata or {})
self.fail_on = fail_on
self.written: list[str] = []
- self.extras: dict[str, dict[str, str] | None] = {}
+ self.extras: dict[str, dict[str, Any] | None] = {}
def head_object(self, bucket: str, file_key: str) -> dict[str, Any] | None:
if file_key not in self.objects:
return None
return {"ContentLength": len(self.objects[file_key]), "Metadata": self.metadata.get(file_key, {})}
- def write(self, bucket: str, key: str, content: str | bytes, extras: dict[str, str] | None) -> None:
+ def write(self, bucket: str, key: str, content: str | bytes, extras: dict[str, Any] | None) -> None:
if self.fail_on is not None and self.fail_on in key:
raise RuntimeError("object store write failed")
self.objects[key] = content if isinstance(content, bytes) else content.encode()
@@ -2392,6 +2401,40 @@ def test_project_vectors_are_published_before_the_manifest_that_names_them(
assert mirror.objects == {}
+ "source_present,already_published,write_fails",
+ [(True, False, False), (True, True, False), (False, False, False), (True, False, True)],
+)
+def test_user_vectors_are_published_when_present_and_never_block_the_manifest_or_reach_the_mirror(
+ monkeypatch: pytest.MonkeyPatch, source_present: bool, already_published: bool, write_fails: bool
+) -> None:
+ prefix = settings.INBOX_RANKING_DATASET_S3_PREFIX
+ partition_key = "2026-08-19"
+ target = serving_user_vectors_key(prefix, partition_key)
+ store = _AppObjectStore(
+ {target: b"published before"} if already_published else None, fail_on="/user_vectors/" if write_fails else None
+ )
+ mirror = _AppObjectStore()
+ dataset = _serving_dataset_s3(prefix, partition_key)
+ if source_present:
+ dataset.objects[partition_object_key(prefix, USER_VECTORS_TABLE, partition_key)] = b"user vectors"
+ capture = _run_serving_manifest(monkeypatch, store, mirror=mirror, dataset_client=dataset)
+
+ named = source_present and not write_fails
+ manifest = ServingManifest.model_validate_json(store.objects[serving_manifest_key(prefix)])
+ assert manifest.user_vectors == (target if named else None)
+ assert _manifest_event(capture)["user_vectors_key"] == manifest.user_vectors
+ assert store.written[-1] == serving_manifest_key(prefix)
+ if named and not already_published:
+ assert store.objects[target] == b"user vectors"
+ assert store.extras[target] == {
+ "Metadata": {PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(USER_VECTORS_SCHEMA_VERSION)}
+ }
+ # Per-user data stays in the deployment that owns the users.
+ assert not any("/user_vectors/" in key for key in mirror.objects)
+ assert ServingManifest.model_validate_json(mirror.objects[serving_manifest_key(prefix)]).user_vectors is None
+
+
_MIRROR_PARTITION = "2026-08-19"
diff --git a/products/signals/dags/inbox_ranking/training/dag.py b/products/signals/dags/inbox_ranking/training/dag.py
index 6cd8e3bcb438..70c98a69471c 100644
--- a/products/signals/dags/inbox_ranking/training/dag.py
+++ b/products/signals/dags/inbox_ranking/training/dag.py
@@ -48,10 +48,12 @@ from products.signals.backend.ranking.overrides import read_ranking_overrides
from products.signals.backend.ranking.serving_manifest import (
PROJECT_VECTORS_ENGAGEMENT_DATE_METADATA_KEY,
PROJECT_VECTORS_SCHEMA_VERSION_METADATA_KEY,
+ USER_VECTORS_SCHEMA_VERSION_METADATA_KEY,
ServingManifest,
model_key,
serving_manifest_key,
serving_project_vectors_key,
+ serving_user_vectors_key,
)
from products.signals.dags.inbox_ranking.common import (
DATASET_VERSION,
@@ -67,8 +69,16 @@ from products.signals.dags.inbox_ranking.common import (
serving_mirror_storage,
skip_unconfigured,
)
-from products.signals.dags.inbox_ranking.dataset.dag import LABELS_TABLE, PROJECT_VECTORS_TABLE, STATE_TABLE
-from products.signals.dags.inbox_ranking.dataset.project_vectors import PROJECT_VECTORS_SCHEMA_VERSION
+from products.signals.dags.inbox_ranking.dataset.dag import (
+ LABELS_TABLE,
+ PROJECT_VECTORS_TABLE,
+ STATE_TABLE,
+ USER_VECTORS_TABLE,
+)
+from products.signals.dags.inbox_ranking.dataset.project_vectors import (
+ PROJECT_VECTORS_SCHEMA_VERSION,
+ USER_VECTORS_SCHEMA_VERSION,
+)
from products.signals.dags.inbox_ranking.training.examples import (
BASE_STATE_COLUMNS,
PROVENANCE_LABEL_COLUMNS,
@@ -402,16 +412,24 @@ class _ServingStore:
# A cross-account bucket refuses a write that does not hand ownership to the bucket owner.
acl: str | None = None
- def write(self, key: str, content: str | bytes, content_type: str | None = None) -> None:
- extras: dict[str, str] = {}
+ def write(
+ self,
+ key: str,
+ content: str | bytes,
+ content_type: str | None = None,
+ metadata: dict[str, str] | None = None,
+ ) -> None:
+ extras: dict[str, Any] = {}
if content_type is not None:
extras["ContentType"] = content_type
+ if metadata is not None:
+ extras["Metadata"] = metadata
if self.acl is not None:
extras["ACL"] = self.acl
self.storage.write(bucket=self.bucket, key=key, content=content, extras=extras or None)
[email protected](name=SERVING_MANIFEST_ASSET, deps=[PROJECT_VECTORS_TABLE], **COMMON_ASSET_KWARGS)
[email protected](name=SERVING_MANIFEST_ASSET, deps=[PROJECT_VECTORS_TABLE, USER_VECTORS_TABLE], **COMMON_ASSET_KWARGS)
def inbox_ranking_serving_manifest(context: dagster.AssetExecutionContext) -> None:
"""Publish the models the scoring sweep serves, and the manifest naming them.
@@ -424,6 +442,9 @@ def inbox_ranking_serving_manifest(context: dagster.AssetExecutionContext) -> No
When an entry reads project vectors, the dt=D `inbox_project_vectors` day is copied the same
way, and the manifest names it. Its `as_of` is the end of dt=D, so it is never newer than a
report the sweep scores after this publish.
+
+ The dt=D `inbox_user_vectors` day is copied too, and the manifest names it in `user_vectors`. No
+ model reads it yet, so a missing day leaves the key out and never blocks the publish.
"""
if skip_unconfigured(context):
return
@@ -511,6 +532,17 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
manifest = manifest.model_copy(update={"project_vectors": serving_project_vectors_key(prefix, partition_key)})
manifest_key = serving_manifest_key(prefix)
primary = _ServingStore(storage=object_storage.object_storage_client(), bucket=settings.OBJECT_STORAGE_BUCKET)
+ user_vectors_key = serving_user_vectors_key(prefix, partition_key)
+ user_vectors_bytes = _publish_user_vectors(
+ context,
+ client,
+ bucket,
+ partition_object_key(prefix, USER_VECTORS_TABLE, partition_key),
+ user_vectors_key,
+ primary,
+ )
+ if user_vectors_bytes is not None:
+ manifest = manifest.model_copy(update={"user_vectors": user_vectors_key})
publication = publish_serving_models(
context, client, bucket, prefix, manifest, primary, project_vectors_source=project_vectors_source
)
@@ -528,7 +560,7 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
reason=decision.reason,
copied_keys=publication.copied,
present_keys=publication.present,
- bytes_copied=publication.bytes_copied,
+ bytes_copied=publication.bytes_copied + (user_vectors_bytes or 0),
mirror=mirror,
recipe_hashes={
model_key(metadata["model_name"], metadata["model_version"]): metadata.get(RECIPE_HASH_KEY)
@@ -562,8 +594,9 @@ def _publish_manifest(context: RunContext, partition_key: str, run_id: str) -> d
),
"entries_copied": dagster.MetadataValue.int(len(publication.copied)),
"entries_already_present": dagster.MetadataValue.int(len(publication.present)),
- "bytes_copied": dagster.MetadataValue.int(publication.bytes_copied),
+ "bytes_copied": dagster.MetadataValue.int(publication.bytes_copied + (user_vectors_bytes or 0)),
"project_vectors_key": dagster.MetadataValue.text(manifest.project_vectors or ""),
+ "user_vectors_key": dagster.MetadataValue.text(manifest.user_vectors or ""),
**pin_metadata,
**mirror_metadata,
}
@@ -765,6 +798,9 @@ def _publish_mirror(
`publish_serving_models` gets no source and copies none. Without such a day the mirror's
manifest names none, and its sweep skips the models that read them. A served model that reads
them could not score there, so then the mirror keeps its previous manifest.
+
+ User vectors are per-user data of this deployment, so they never go to the mirror either, and
+ the mirror's manifest names none.
"""
if not settings.INBOX_RANKING_SERVING_MIRROR_BUCKET:
return None
@@ -789,7 +825,7 @@ def _publish_mirror(
client,
bucket,
prefix,
- manifest.model_copy(update={"project_vectors": project_vectors_key}),
+ manifest.model_copy(update={"project_vectors": project_vectors_key, "user_vectors": None}),
mirror,
)
except Exception as error:
@@ -887,6 +923,39 @@ def _publish_project_vectors(
return len(body)
+def _publish_user_vectors(
+ context: RunContext,
+ client,
+ bucket: str,
+ source: str,
+ target_key: str,
+ target: _ServingStore,
+) -> int | None:
+ """Copy one day of user vectors to `target_key`, stamped with its schema version. Returns the
+ bytes moved, or None when the day is not in `target` after the call.
+
+ No model reads user vectors yet, so a missing day or a failed copy only logs, and the publish
+ goes on without them. A day already at the target is left alone, like a project vectors day.
+ """
+ try:
+ if target.storage.head_object(bucket=target.bucket, file_key=target_key) is not None:
+ return 0
+ body = _read_bytes_if_exists(client, bucket, source)
+ if body is None:
+ context.log.warning(f"user vectors {source} are missing in the dataset bucket; manifest names none")
+ return None
+ target.write(
+ target_key,
+ body,
+ metadata={USER_VECTORS_SCHEMA_VERSION_METADATA_KEY: str(USER_VECTORS_SCHEMA_VERSION)},
+ )
+ except Exception as error:
+ context.log.exception(f"user vectors {source} not copied: {error!r}")
+ return None
+ context.log.info(f"copied user vectors to {target.bucket}/{target_key}")
+ return len(body)
+
+
# dt=D evaluates scores from birth through maturity, so the mapping reaches back as far as the
# longest head horizon. Partitions before the scores asset existed have no upstream to map to.
_HORIZON_MAPPING = dagster.TimeWindowPartitionMapping(
diff --git a/products/signals/dags/inbox_ranking/training/telemetry.py b/products/signals/dags/inbox_ranking/training/telemetry.py
index 419fbdf0a5f4..203b8d8d8a14 100644
--- a/products/signals/dags/inbox_ranking/training/telemetry.py
+++ b/products/signals/dags/inbox_ranking/training/telemetry.py
@@ -250,6 +250,7 @@ def serving_manifest_event(
"model_keys": [entry.key for entry in manifest.models],
"model_roles": {entry.key: entry.roles for entry in manifest.models},
"model_recipe_hashes": {entry.key: (recipe_hashes or {}).get(entry.key) for entry in manifest.models},
+ "user_vectors_key": manifest.user_vectors,
}
if mirror is not None:
properties |= {