Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): add user-history pair families for inbox ranking (#125196)

posthog[bot]44071dc1b528
diff --git a/posthog/dags/locations/signals_ml.py b/posthog/dags/locations/signals_ml.py
index 22a2336dd711..8b590e09b978 100644
--- a/posthog/dags/locations/signals_ml.py
+++ b/posthog/dags/locations/signals_ml.py
@@ -21,6 +21,7 @@ if is_inbox_ranking_registered():
inbox_ranking_nn.inbox_ranking_nn_with_prio_model_candidate,
inbox_ranking_nn.inbox_ranking_nn_with_prio_model_champion,
inbox_ranking_nn_experiments.inbox_ranking_nn_experiment_candidates,
+ inbox_ranking_nn_experiments.inbox_ranking_nn_pair_experiment_candidates,
inbox_ranking_nn_pairs.inbox_ranking_nn_pair_examples,
],
jobs=[
diff --git a/products/signals/backend/ranking/features.py b/products/signals/backend/ranking/features.py
index b26dc26f83b9..587bb0502f5d 100644
--- a/products/signals/backend/ranking/features.py
+++ b/products/signals/backend/ranking/features.py
@@ -14,6 +14,9 @@ are the served contract: training (`products/signals/dags/inbox_ranking/training
time, and reads the latest priority judgment only for a model whose set declares the `priority`
state column. `report_embeddings_priority_sources` adds the source products of the report's signals.
Only an experiment family trains on it, and the sweep cannot build it yet.
+
+The pair sets (`PairUserHistoryFeatureSet`) build one row per (report, user) for the pair-grain
+experiments: the priority set's columns, then the user's history and reviewer columns.
"""
import abc
@@ -88,6 +91,29 @@ PROJECT_VECTOR_KINDS = ("typical", "open", "action", "dismiss")
# vector and the kind has no history.
MIN_PROJECT_HISTORY_REPORTS = 5
+# The user-history kinds a pair network reads, in its input order. The dataset dag builds one user
+# vector per kind (`user_vectors()`), and each kind has a project-vector kind of the same name.
+USER_HISTORY_KINDS = ("open", "dismiss")
+# The pair examples' user-input columns: the reports in each kind's user vector, the capped days
+# since the user's last view, and whether the user had any usable history.
+USER_REPORTS_COLUMNS = tuple(f"user_{kind}_reports" for kind in USER_HISTORY_KINDS)
+DAYS_SINCE_LAST_VIEW_COLUMN = "days_since_last_view"
+USER_HAS_HISTORY_COLUMN = "has_history"
+SUGGESTED_REVIEWER_COLUMN = "is_suggested_reviewer"
+DAYS_SINCE_LAST_VIEW_CAP = 90.0
+PAIR_STATE_COLUMNS = (
+ *USER_REPORTS_COLUMNS,
+ DAYS_SINCE_LAST_VIEW_COLUMN,
+ USER_HAS_HISTORY_COLUMN,
+ SUGGESTED_REVIEWER_COLUMN,
+)
+USER_HISTORY_FEATURE_NAMES = (
+ *(f"user_{kind}_log_reports" for kind in USER_HISTORY_KINDS),
+ "user_days_since_last_view",
+ "has_user_history",
+)
+PAIR_FEATURE_NAMES = (*USER_HISTORY_FEATURE_NAMES, SUGGESTED_REVIEWER_COLUMN)
+
# How many rows one report contributes to a head's examples. The default is `birth`, one row per
# report at the snapshot of the day it was created; `examples.py` holds the reasoning.
# `scoring_moment` is one row per (report, snapshot), and `report` is the first snapshot of the
@@ -325,6 +351,67 @@ class ReportEmbeddingsPrioritySourcesFeatureSet(ReportEmbeddingsPriorityFeatureS
return one_hot
+class PairUserHistoryFeatureSet(FeatureSet):
+ """The input of one (report, user) pair: a report set's columns, then `PAIR_FEATURE_NAMES`.
+
+ The report part comes from `report_feature_set` and is the same for every pair of a report. The
+ pair part is per pair and comes from the pair examples' `PAIR_STATE_COLUMNS`: `log1p` of the
+ reports in each user vector, the days since the last view over `DAYS_SINCE_LAST_VIEW_CAP`,
+ `has_user_history` and `is_suggested_reviewer`. A network also reads the user's vectors as
+ offsets from the team's project vectors (`user_offsets`), outside this matrix.
+
+ With `user_history` False, every user-history column is zero whatever the rows hold, and the
+ caller holds the user offsets at zero too. Only `is_suggested_reviewer` stays. That set is the
+ pair baseline: the same width and the same reviewer flag, without the user's history.
+
+ A pair set is not a report-state set, so it stays out of `FEATURE_SETS`: the state snapshot has
+ no pair columns, and the scoring sweep cannot build a pair input.
+ """
+
+ schema_version = 1
+
+ def __init__(self, *, name: str, report_feature_set: FeatureSet, user_history: bool) -> None:
+ self.name = name
+ self.report_feature_set = report_feature_set
+ self.user_history = user_history
+ self.feature_names = (*report_feature_set.feature_names, *PAIR_FEATURE_NAMES)
+ self.state_columns = (*report_feature_set.state_columns, *PAIR_STATE_COLUMNS)
+ self.extras_keys = report_feature_set.extras_keys
+ self.example_grain = report_feature_set.example_grain
+ self.max_examples_per_head = report_feature_set.max_examples_per_head
+
+ def build_matrix(
+ self, rows: pd.DataFrame, extras: Extras = NO_EXTRAS, *, as_of: datetime.datetime | None = None
+ ) -> pd.DataFrame:
+ matrix = self.report_feature_set.build_matrix(rows, extras, as_of=as_of)
+ pair = self.pair_matrix(rows)
+ for column, name in enumerate(PAIR_FEATURE_NAMES):
+ matrix[name] = pair[:, column]
+ return matrix
+
+ def buildable(
+ self, rows: pd.DataFrame, extras: Extras = NO_EXTRAS, *, as_of: datetime.datetime | None = None
+ ) -> pd.Series:
+ return self.report_feature_set.buildable(rows, extras, as_of=as_of)
+
+ def pair_matrix(self, rows: pd.DataFrame) -> np.ndarray:
+ """(rows, `PAIR_FEATURE_NAMES`) float32, one row per pair of `rows`."""
+ missing = [column for column in PAIR_STATE_COLUMNS if column not in rows]
+ if missing:
+ raise ValueError(f"the {self.name} feature set needs the {', '.join(missing)} columns")
+ matrix = np.zeros((len(rows), len(PAIR_FEATURE_NAMES)), dtype=np.float32)
+ matrix[:, PAIR_FEATURE_NAMES.index(SUGGESTED_REVIEWER_COLUMN)] = rows[SUGGESTED_REVIEWER_COLUMN].to_numpy(
+ dtype=np.float32
+ )
+ if self.user_history:
+ kinds = len(USER_HISTORY_KINDS)
+ matrix[:, :kinds] = np.log1p(rows[list(USER_REPORTS_COLUMNS)].to_numpy(dtype=np.float32))
+ days = np.minimum(rows[DAYS_SINCE_LAST_VIEW_COLUMN].to_numpy(dtype=np.float32), DAYS_SINCE_LAST_VIEW_CAP)
+ matrix[:, kinds] = days / DAYS_SINCE_LAST_VIEW_CAP
+ matrix[:, kinds + 1] = rows[USER_HAS_HISTORY_COLUMN].to_numpy(dtype=np.float32)
+ return matrix
+
+
REPORT_EMBEDDINGS_FEATURE_SET = ReportEmbeddingsFeatureSet(name="report_embeddings", extras_key=REPORT_EMBEDDINGS_EXTRA)
REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET = ReportEmbeddingsPriorityFeatureSet(
name="report_embeddings_priority", extras_key=REPORT_EMBEDDINGS_EXTRA
@@ -332,6 +419,16 @@ REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET = ReportEmbeddingsPriorityFeatureSet(
REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET = ReportEmbeddingsPrioritySourcesFeatureSet(
name="report_embeddings_priority_sources", extras_key=REPORT_EMBEDDINGS_EXTRA, sources_key=REPORT_SOURCES_EXTRA
)
+PAIR_BASE_FEATURE_SET = PairUserHistoryFeatureSet(
+ name="pair_report_embeddings_priority",
+ report_feature_set=REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
+ user_history=False,
+)
+PAIR_USER_HISTORY_FEATURE_SET = PairUserHistoryFeatureSet(
+ name="pair_report_embeddings_priority_user_history",
+ report_feature_set=REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
+ user_history=True,
+)
# Every set this build can produce, by name. A model that names a set absent from here cannot be
# scored, the same way a model whose feature names have moved on cannot.
@@ -381,6 +478,23 @@ def project_inputs(project_vectors: pd.DataFrame, team_ids: Sequence[int]) -> Pr
return ProjectInputs(vectors=vectors, has_history=has_history)
+def user_offsets(
+ user_vectors: np.ndarray, user_reports: np.ndarray, project_vectors: np.ndarray, project_has_history: np.ndarray
+) -> np.ndarray:
+ """(n, `USER_HISTORY_KINDS`, D) float32: each user vector minus the team's project vector of the
+ same kind, zero where the user has no report of the kind or the team's kind has no history.
+
+ `user_vectors` is (n, user kinds, D) and `user_reports` (n, user kinds), in `USER_HISTORY_KINDS`
+ order. `project_vectors` and `project_has_history` are the `ProjectInputs` arrays of each row's
+ team, in `PROJECT_VECTOR_KINDS` order. An offset says how this person's taste differs from the
+ team's, so a user who reads what the team reads gets a small one.
+ """
+ kinds = [PROJECT_VECTOR_KINDS.index(kind) for kind in USER_HISTORY_KINDS]
+ present = (user_reports > 0) & (project_has_history[:, kinds] > 0)
+ offsets = np.where(present[..., None], user_vectors - project_vectors[:, kinds], 0.0)
+ return offsets.astype(np.float32, copy=False)
+
+
def feature_set_by_name(name: str | None) -> FeatureSet | None:
"""The named set, or None when the name is absent or this build cannot produce it."""
return None if name is None else FEATURE_SETS.get(name)
diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.md
index 37636ff02bd8..67683a596f2f 100644
--- a/products/signals/dags/inbox_ranking/AGENTS.md
+++ b/products/signals/dags/inbox_ranking/AGENTS.md
@@ -44,8 +44,9 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- `nn_v3` and `nn_with_prio_v2` are trained by the `signals_ml` location (`nn/dag.py`), never by the main one. They share one `inbox_ranking_nn_examples` object, and each has its own candidate and champion asset from `family_assets`. The `nn_v3` assets keep the names `inbox_ranking_nn_model_{candidate,champion}`, so their Dagster history continues. A later family's candidate depends on the previous family's champion, for memory and not for data: two candidates at once can exceed the job's memory limit. A family whose set reads a state column the examples object lacks skips with a warning rather than training on zeros. torch is imported only inside the candidate asset's body (through `nn/train.py` and `multihead/network.py`). `signals_ml` must import without torch, and `test_multihead_network.py` pins that. Its decision goes through `decide_family_champion` in `training/dag.py`, graded through the incumbent's `model.holdout.onnx`. Keep `signals_ml` commented out in `.dagster_home/workspace.yaml`, and keep its schedule (04:30 UTC) running by default on prod US only.
- An experiment is a `MODEL_FAMILIES` entry with `experiment=True`, a `baseline` serving family and an `expires` day. Only `inbox_ranking_nn_experiments_job` (`nn/experiments.py`) trains it, with its own schedule and resource tags, so a modeling idea never adds to the `nn` job's runtime or memory budget. It reads the day's `inbox_ranking_nn_examples` object and never rebuilds it. An experiment never enters the serving manifest, and `_pin_metadata` refuses its key, so the promote job, the served pointer and the pins cannot name it. To serve its recipe, register it as a serving family under a new `model_name`. Its `metadata.json` carries a `vs_baseline` block: the experiment and the baseline fit with each of `EXPERIMENT_SEEDS` on the same train rows and graded on the same holdout rows. An extra side input for an experiment goes through the registry in `training/side_inputs.py`, which refuses a frame without a timezone-aware `as_of_column` and drops rows known after the partition end. A feature set reads it through `rows_as_of`.
-- `inbox_ranking_nn_pair_examples` (`nn/pairs.py`) has one row per (report example, impressed user), with its own `PAIR_EXAMPLES_SCHEMA_VERSION` and its own job without a schedule. It takes its report rows and holdout from the `open` head's rows of the day's `inbox_ranking_nn_examples` object and never rebuilds them. Its impressions and labels come from the dt=B+`PAIR_HORIZON_DAYS` interactions partition, and its user inputs are as of the start of day B, so the target report is never in the user's history. The object holds no vector: a reader rebuilds the user vectors with `pair_user_vectors`. `user_vectors()` lives next to `project_vectors()` and shares `known_before` and `_usable_embeddings` with it; never give it its own as-of rule or embedding gate.
+- `inbox_ranking_nn_pair_examples` (`nn/pairs.py`) has one row per (report example, impressed user), with its own `PAIR_EXAMPLES_SCHEMA_VERSION`. Its own job has no schedule, and the experiments job builds it for the same partition before the pair experiments, only on a day with a live pair experiment. Keep the report experiments asset free of a dependency on it, so a failed pair step skips the pair experiments only. It takes its report rows and holdout from the `open` head's rows of the day's `inbox_ranking_nn_examples` object and never rebuilds them. Its impressions and labels come from the dt=B+`PAIR_HORIZON_DAYS` interactions partition, and its user inputs are as of the start of day B, so the target report is never in the user's history. The object holds no vector: a reader rebuilds the user vectors with `pair_user_vectors`. `user_vectors()` lives next to `project_vectors()` and shares `known_before` and `_usable_embeddings` with it; never give it its own as-of rule or embedding gate.
- A pair-grain family (`grain="pair"`) is always an experiment, trained by `nn/pair_lane.py` in the experiments job. Its baseline is another pair experiment and its `reference` a report-grain serving family. `family_problems` checks both at import. Keep the report-grain `vs_baseline` block, its events and the report families' recipe free of pair fields: an added recipe key changes every serving family's hash, and `check_recipe` then fails each retrain. Fit the reference on report train rows only, so no holdout pair's report trains it.
+- A pair set (`PairUserHistoryFeatureSet`) is the user-history contract of a pair family. Its set with `user_history` off is the pair baseline: the same width, the same reviewer flag, and zero in every user column and user offset whatever the pair rows hold. Build each arm's inputs from the pair rows for that arm's set, never from another arm's inputs. The report part may be built once per report, but the user columns and offsets are per pair, so one user's history never reaches another user's pair of the same report. A pair set stays out of `FEATURE_SETS`, because the state loader reads every registered set's columns and the state snapshot has no pair columns.
## Invariants — do not break
diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md
index 9d8eac9977b1..e093e232ff6e 100644
--- a/products/signals/dags/inbox_ranking/README.md
+++ b/products/signals/dags/inbox_ranking/README.md
@@ -438,13 +438,13 @@ dagster job launch -w .dagster_home/workspace.yaml --location posthog.dags.locat
An experiment is a `MODEL_FAMILIES` entry with `experiment=True`, a `baseline` serving family and an `expires` day (`training/families.py`). It tests a modeling idea without a serving family's costs:
-- **Its own job.** `inbox_ranking_nn_experiments_job` (`nn/experiments.py`, on `signals_ml`) trains every live experiment, one after the other. It has its own runtime and memory tags, so it never uses the `nn` job's budget. Its schedule (08:00 UTC, after the `nn` job's 3h budget) skips when no experiment is live.
+- **Its own job.** `inbox_ranking_nn_experiments_job` (`nn/experiments.py`, on `signals_ml`) trains every live experiment, one after the other. It has its own runtime and memory tags, so it never uses the `nn` job's budget. It holds three assets, run one step at a time in one process: `inbox_ranking_nn_experiment_candidates` (the report-grain experiments), `inbox_ranking_nn_pair_examples` and `inbox_ranking_nn_pair_experiment_candidates` (the pair-grain experiments, after the pair examples). The report experiments do not depend on the pair examples, so a failed pair step skips only the pair experiments. Its schedule (08:00 UTC, after the `nn` job's 3h budget) selects the report asset when a report experiment is live and the two pair assets when a pair experiment is live, and skips when nothing is live.
- **No new examples.** It reads the day's `inbox_ranking_nn_examples` object. A day without that object is skipped, and so is a family after its `expires` day.
- **Never served.** The serving manifest reads only `serving_families()`. The pin check refuses an experiment key, so the promote job, the served pointer and the pins cannot name one. To serve the recipe, register it as a serving family under a new `model_name`.
- **Paired read.** The experiment and its baseline each fit once per seed in `EXPERIMENT_SEEDS`, on the same train rows. `metadata.json` holds a `vs_baseline` block per head: each seed's within-team and pooled holdout AUC for both arms, and the mean, min and max of the gaps. One `inbox_ranking_experiment_compared` event per head carries the gaps.
- **Side inputs.** A side input beyond the report vectors needs a loader in `training/side_inputs.py`. The registry refuses a frame without a timezone-aware `as_of_column` and drops the rows known after the partition end. A feature set in the dags reads the frame through `rows_as_of`. A set in `features.py` cannot import the dags, so it applies the same `<= as_of` cut on the loader's column itself.
-The live experiment is `nn_with_prio_sources_v1`, graded against `nn_with_prio_v2` until 2026-11-11. Its set, `report_embeddings_priority_sources`, adds the source products of the report's signals: one column per product in a fixed vocabulary, `source_other`, `source_count` and `has_sources`. The `report_sources` loader reads only the metadata columns of `inbox_signal_embeddings`, from 30 days before the training lookback to dt=D. A moment takes the latest emission per `(team_id, signal_id)` inserted at or before it and drops a retracted signal. It never reads the impression labels' `source_products` column, which holds the latest impression's value. The scoring sweep cannot build this set, so the recipe needs serving work before it can become a serving family.
+The live report-grain experiment is `nn_with_prio_sources_v1`, graded against `nn_with_prio_v2` until 2026-11-11. The pair-grain experiments are under [User-history families](#user-history-families). Its set, `report_embeddings_priority_sources`, adds the source products of the report's signals: one column per product in a fixed vocabulary, `source_other`, `source_count` and `has_sources`. The `report_sources` loader reads only the metadata columns of `inbox_signal_embeddings`, from 30 days before the training lookback to dt=D. A moment takes the latest emission per `(team_id, signal_id)` inserted at or before it and drops a retracted signal. It never reads the impression labels' `source_products` column, which holds the latest impression's value. The scoring sweep cannot build this set, so the recipe needs serving work before it can become a serving family.
The schedule is a fallback. To read an experiment on a chosen day, launch the job for that partition from the Dagster UI or the CLI:
@@ -458,7 +458,7 @@ The newest day you can launch is the newest day the `nn` job finished. Adjacent
### Pair examples
`inbox_ranking_nn_pair_examples` (`nn/pairs.py`) holds the report-user pairs for a personal ranking lane: a model that reads a vector of the reports one user viewed before, next to the report.
-It writes `inbox_ranking_nn_pair_examples/v1/dt=D/` with its own `PAIR_EXAMPLES_SCHEMA_VERSION`, and runs in its own job, `inbox_ranking_nn_pair_examples_job`, which has no schedule. Launch it for a partition after the `nn` job finished that day.
+It writes `inbox_ranking_nn_pair_examples/v1/dt=D/` with its own `PAIR_EXAMPLES_SCHEMA_VERSION`, and runs in its own job, `inbox_ranking_nn_pair_examples_job`, which has no schedule. The experiments job also builds it for the partition it trains, before the pair experiments. Launch the pair examples job alone to inspect a partition before any model reads it.
- **Rows**: one per (report example, user), for each user impressed with the report within `PAIR_HORIZON_DAYS` (7) of birth. The report rows are the `open` head's rows of the dt=D `inbox_ranking_nn_examples` object, so the report inputs, the holdout split (`holdout` is the row's `open_holdout`) and the day windows are the report lane's. A report born on day B reads its impressions from the dt=B+7 `inbox_user_report_interactions` partition, the partition the report's `open` label reads.
- **Labels**, from the same partition and before its end: `open_label` is the user's `first_opened_at`, and `action_label` is any of the user's UI action `first_*` columns from `UI_ACTION_COLUMNS`, the set the `action` project vector reads. There is no `fixed` label, because a fix belongs to the report and not to a user.
@@ -483,7 +483,21 @@ A family with `grain="pair"` scores one row per (report, impressed user) from `i
- **Reference.** For each seed, the reference fits at report grain on the report train rows born on a pair train day, and each holdout pair takes its report's score. The reference fits before the pair matrices are built, so its report matrix and the pair matrices are never in memory together.
- **Grade.** `vs_baseline` (`nn/pair_grade.py`) has `grain: "pair"` and, per head, within-user AUCs for each arm and seed. A user is one person in one team, and a user with one outcome class is left out. `macro` weighs each user the same, and `pooled` weighs each user by its pairs. The scopes are `all` and `non_dogfood` (without the dogfood project's users). The slices are `all`, the prior-view buckets, `history_10_plus`, `suggested_reviewer` and `not_suggested_reviewer`. `vs_baseline` and `vs_reference` give the mean, min and max delta over the paired seeds, and a team bootstrap interval for the macro delta. One `inbox_ranking_experiment_compared` event per head, scope and slice carries `grain`, `scope`, `slice` and the deltas against both arms. Report-grain experiments keep their block and their events as they were.
-No pair family is registered yet. The user-input feature set and the two pair families that read it come in a follow-up.
+#### User-history families
+
+`nn_pair_user_v1` and `nn_pair_base_v1` ask whether a user's own inbox history improves the ranking. Both train until 2026-11-25, with heads `open` and `action` and `nn_with_prio_v2` as the reference. `nn_pair_user_v1` names `nn_pair_base_v1` as its baseline, and the base has no pair baseline.
+
+- **Same network.** Both read the `report_embeddings_priority` columns, the team's project vectors, `is_suggested_reviewer` and the same user-history inputs, in one input shape, on the same pairs and seeds.
+- **User inputs** (`PairUserHistoryFeatureSet` in `products/signals/backend/ranking/features.py`): `log1p` of `user_open_reports` and `user_dismiss_reports`, `days_since_last_view` over its 90-day cap, `has_user_history`, and a `user` graph input. That input holds, per kind (`open`, `dismiss`), the user's vector minus the team's project vector of the kind (`user_offsets`). An offset is zero when the user has no report of the kind or the team's kind has too thin a history. A cold-start user reads zero offsets, zero counts and the capped recency, and gets a finite score from the same network.
+- **The base reads no history.** `nn_pair_base_v1`'s set holds every user column and offset at zero, whatever the pair rows hold, and keeps the reviewer flag. So `nn_pair_user_v1` against `nn_pair_base_v1` is the uplift of the user's history alone, and the change of grain and the reviewer flag are in both arms.
+- **Per pair.** The lane builds each arm's inputs from the pair rows. The report part is built once per report, and the user columns and offsets come from each pair's own user, so two users of one report never share them. The user vectors come from `pair_user_vectors` on the dt=D interactions snapshot of teams in `training_consent_team_ids()`, inside each team's organization, as of the start of the birth day. A family that reads user history skips a partition without that snapshot.
+- **Metadata.** A pair candidate records `user_vector_kinds`, and its recipe records them with `user_vector_settings`. The asset metadata adds `pair_rows`, `pair_users` and `pair_rows_with_prior_views` per family.
+
+To run both families for one partition, launch the experiments job on the `posthog.dags.locations.signals_ml` location with the tag `dagster/partition` set to the day (`YYYY-MM-DD`), as in the command above. The job needs no run config. A launch without an asset selection runs all three assets, so it also trains the report-grain experiments. To train only the pair families, select `inbox_ranking_nn_pair_examples` and `inbox_ranking_nn_pair_experiment_candidates`. Launch a day the `nn` job finished, and whose dt=D interactions and embeddings snapshots and dt=B+7 interactions partitions exist.
+
+Before the first read, launch `inbox_ranking_nn_pair_examples_job` alone on a recent day and check its metadata: `rows`, `users`, positives per label, rows and users per prior-view bucket, and `suggested_reviewer_share`. The `history_10_plus` holdout needs users with both classes in enough teams for the within-user grade and its team bootstrap to read.
+
+Lead the read with the macro within-user AUC delta of `nn_pair_user_v1` against `nn_pair_base_v1`, in `scope=non_dogfood`, `slice=all`, for `open` and `action`, next to its `vs_reference` delta. Then read the history buckets, `history_10_plus` and the reviewer slices. The predeclared bar for a go is a positive macro delta in `history_10_plus` outside dogfood on `open` or `action`, on at least 3 partitions about 7 days apart, with a team bootstrap interval above zero on at least one, and no bucket below 10 views worse than the base beyond seed noise.
## Configuration
diff --git a/products/signals/dags/inbox_ranking/dataset/project_vectors.py b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
index 14520cda263f..610b51bd2377 100644
--- a/products/signals/dags/inbox_ranking/dataset/project_vectors.py
+++ b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
@@ -36,6 +36,7 @@ from products.signals.backend.ranking.features import (
EMBEDDING_DIMENSIONS,
MIN_PROJECT_HISTORY_REPORTS,
PROJECT_VECTOR_KINDS,
+ USER_HISTORY_KINDS,
)
from products.signals.dags.inbox_ranking.dataset.queries import ACTION_TYPES
from products.signals.dags.inbox_ranking.training.heads import UI_ACTION_COLUMNS
@@ -65,7 +66,8 @@ _PROJECT_VECTORS_FIELDS: list[tuple[str, pa.DataType]] = [
PROJECT_VECTORS_SCHEMA = pa.schema(_PROJECT_VECTORS_FIELDS)
USER_HISTORY_LIMIT = 20
-USER_OPEN, USER_DISMISS = "open", "dismiss"
+# The backend owns the user kinds too, because the pair network reads them in this order.
+USER_OPEN, USER_DISMISS = USER_HISTORY_KINDS
# The durable VIEW action row dates an open, because the `Inbox report opened` event misses the
# surfaces that send no client event.
USER_HISTORY_COLUMNS: Mapping[str, str] = {USER_OPEN: "first_viewed_at", USER_DISMISS: DISMISS_COLUMN}
diff --git a/products/signals/dags/inbox_ranking/multihead/inputs.py b/products/signals/dags/inbox_ranking/multihead/inputs.py
index 9e28c392d198..a1b981657f6a 100644
--- a/products/signals/dags/inbox_ranking/multihead/inputs.py
+++ b/products/signals/dags/inbox_ranking/multihead/inputs.py
@@ -5,8 +5,9 @@ import numpy as np
from posthog.dataclasses import frozen
-# The exported graph's names. A network without context takes only the first input.
-ONNX_INPUTS = ("report", "project", "has_history")
+# The exported graph's names. A network without context takes only the first input, and a network
+# without user history only the first three.
+ONNX_INPUTS = ("report", "project", "has_history", "user")
ONNX_OUTPUT = "probabilities"
@@ -34,11 +35,28 @@ class ProjectContext:
index: np.ndarray
+@frozen
+class UserContext:
+ """The user-history offsets per (team, user, birth day), shared by every pair of that key."""
+
+ # (keys, user kinds, D) float32: the user's vector minus the team's project vector of the same
+ # kind (`user_offsets`), all zero where either side is missing.
+ vectors: np.ndarray
+ # (rows,) int, the key each pair row reads.
+ index: np.ndarray
+
+
@frozen
class NetworkInputs:
# (rows, D + report features) float32: the report vector, then any columns the feature set adds.
reports: np.ndarray
context: ProjectContext | None = None
+ # Only a pair network reads user history, and only next to a project context.
+ users: UserContext | None = None
+
+ def __post_init__(self) -> None:
+ if self.users is not None and self.context is None:
+ raise ValueError("a network that reads user history needs project vectors")
def __len__(self) -> int:
return len(self.reports)
diff --git a/products/signals/dags/inbox_ranking/multihead/network.py b/products/signals/dags/inbox_ranking/multihead/network.py
index 60199d065f72..079c956c7394 100644
--- a/products/signals/dags/inbox_ranking/multihead/network.py
+++ b/products/signals/dags/inbox_ranking/multihead/network.py
@@ -1,13 +1,20 @@
-"""A multi-head network over a report's vector and, optionally, its team's project vectors.
+"""A multi-head network over a report's vector and, optionally, its team's project vectors and a
+user's history.
report vector ─┐
├─ shared Linear(1536 → k) ─ concat ─ dropout ─ trunk (ReLU) ─ dropout ─ one logit per head
- project vectors┘ (and report ⊙ project per kind, plus a history flag per kind)
+ project vectors┤ (and report ⊙ project per kind, plus a history flag per kind)
+ user offsets ──┘ (and report ⊙ user offset per kind)
One projection serves every vector, so the report and the team's history land in one learned space
and their element-wise product can say "this report looks like what the team engaged with". Weight
decay is decoupled (AdamW), and dropout sits on the trunk's input and output.
+A pair network also takes `user`: per user-history kind, the user's vector minus the team's project
+vector of that kind. It goes through the same projection, so its product with the report can say
+"this report looks like what this person reads more than the team does". With the default of zero
+user kinds, the graph has no `user` input.
+
The `report` input can carry `report_features` columns after the vector, such as the report's
priority levels. Those columns skip the projection and go straight into the trunk input. With the
default of zero, the graph is the vector-only one.
@@ -40,14 +47,17 @@ ONNX_OPSET = 17
class _Model(nn.Module):
- def __init__(self, *, heads: int, kinds: int, config: NetworkConfig, report_features: int = 0) -> None:
+ def __init__(
+ self, *, heads: int, kinds: int, config: NetworkConfig, report_features: int = 0, user_kinds: int = 0
+ ) -> None:
super().__init__()
self.kinds = kinds
self.report_features = report_features
+ self.user_kinds = user_kinds
k, h = config.projection_dim, config.trunk_dim
self.projection = nn.Linear(EMBEDDING_DIMENSIONS, k)
self.input_dropout = nn.Dropout(config.dropout)
- self.trunk = nn.Linear(k + report_features + kinds * (2 * k + 1), h)
+ self.trunk = nn.Linear(k + report_features + kinds * (2 * k + 1) + user_kinds * 2 * k, h)
self.output_dropout = nn.Dropout(config.dropout)
self.heads = nn.Linear(h, heads)
for layer in (self.projection, self.trunk, self.heads):
@@ -55,10 +65,14 @@ class _Model(nn.Module):
nn.init.zeros_(layer.bias)
def forward(
- self, report: torch.Tensor, project: torch.Tensor | None = None, has_history: torch.Tensor | None = None
+ self,
+ report: torch.Tensor,
+ project: torch.Tensor | None = None,
+ has_history: torch.Tensor | None = None,
+ user: torch.Tensor | None = None,
) -> torch.Tensor:
- """(N, heads) logits from `report` (N, D + report_features), `project` (N, kinds, D) and
- `has_history` (N, kinds)."""
+ """(N, heads) logits from `report` (N, D + report_features), `project` (N, kinds, D),
+ `has_history` (N, kinds) and `user` (N, user_kinds, D)."""
# Sliced only when there are extra columns, so a vector-only graph stays as it was exported.
vector = report[:, :EMBEDDING_DIMENSIONS] if self.report_features else report
projected = self.projection(vector * INPUT_SCALE)
@@ -74,6 +88,11 @@ class _Model(nn.Module):
(projected.unsqueeze(1) * projected_context).flatten(1),
has_history,
]
+ if self.user_kinds:
+ if user is None:
+ raise ValueError("a network built with user history needs user offsets")
+ projected_user = self.projection(user * INPUT_SCALE)
+ parts += [projected_user.flatten(1), (projected.unsqueeze(1) * projected_user).flatten(1)]
trunk_input = torch.cat(parts, dim=1) if len(parts) > 1 else projected
trunk_out = F.relu(self.trunk(self.input_dropout(trunk_input)))
return self.heads(self.output_dropout(trunk_out))
@@ -107,18 +126,26 @@ class MultiHeadNetwork:
kinds: int,
config: NetworkConfig,
report_features: int = 0,
+ user_kinds: int = 0,
) -> None:
+ if user_kinds and not with_context:
+ raise ValueError("a network that reads user history needs project vectors")
self.head_names = tuple(heads)
self.with_context = with_context
self.kinds = kinds if with_context else 0
self.report_features = report_features
+ self.user_kinds = user_kinds
self.config = config
# Every random draw (initialization, shuffle, dropout) comes from this seeded state, and the
# global torch state stays as it was.
self._rng_state = torch.Generator().manual_seed(config.seed).get_state()
with self._seeded():
self.model = _Model(
- heads=len(self.head_names), kinds=self.kinds, config=config, report_features=report_features
+ heads=len(self.head_names),
+ kinds=self.kinds,
+ config=config,
+ report_features=report_features,
+ user_kinds=user_kinds,
)
weights = [parameter for name, parameter in self.model.named_parameters() if name.endswith("weight")]
biases = [parameter for name, parameter in self.model.named_parameters() if name.endswith("bias")]
@@ -169,11 +196,16 @@ class MultiHeadNetwork:
if inputs.context is None:
raise ValueError("a network built with context needs project vectors")
pairs = inputs.context.index[rows]
- return (
+ tensors = (
report,
torch.from_numpy(inputs.context.vectors[pairs]).float(),
torch.from_numpy(inputs.context.has_history[pairs]).float(),
)
+ if not self.user_kinds:
+ return tensors
+ if inputs.users is None:
+ raise ValueError("a network built with user history needs user offsets")
+ return (*tensors, torch.from_numpy(inputs.users.vectors[inputs.users.index[rows]]).float())
def _batches(self, rows: np.ndarray, *, shuffle: bool) -> Iterator[np.ndarray]:
order = rows[torch.randperm(len(rows)).numpy()] if shuffle else rows
@@ -192,15 +224,18 @@ class MultiHeadNetwork:
def export_onnx(network: MultiHeadNetwork, path: Path) -> tuple[str, ...]:
"""Write the network to `path` as an ONNX graph and return its head order.
- The inputs are `report` (N, D + report_features) and, for a network with context, `project` (N, kinds, D) and
- `has_history` (N, kinds), all float32. The one output is `probabilities` (N, heads), in the
- returned head order. N is a dynamic axis.
+ The inputs are `report` (N, D + report_features), for a network with context `project` (N, kinds, D)
+ and `has_history` (N, kinds), and for a network with user history `user` (N, user_kinds, D), all
+ float32. The one output is `probabilities` (N, heads), in the returned head order. N is a dynamic
+ axis.
"""
module = _Probabilities(network.model).eval()
batch = 2
example: tuple[torch.Tensor, ...] = (torch.zeros(batch, EMBEDDING_DIMENSIONS + network.report_features),)
if network.with_context:
example += (torch.zeros(batch, network.kinds, EMBEDDING_DIMENSIONS), torch.zeros(batch, network.kinds))
+ if network.user_kinds:
+ example += (torch.zeros(batch, network.user_kinds, EMBEDDING_DIMENSIONS),)
input_names = list(ONNX_INPUTS[: len(example)])
# torch 2.9 defaults to the dynamo exporter, which needs onnxscript. The TorchScript exporter
# covers these layers without it.
diff --git a/products/signals/dags/inbox_ranking/nn/experiments.py b/products/signals/dags/inbox_ranking/nn/experiments.py
index fcb0e6285ecb..ce8b5aa65e31 100644
--- a/products/signals/dags/inbox_ranking/nn/experiments.py
+++ b/products/signals/dags/inbox_ranking/nn/experiments.py
@@ -13,8 +13,11 @@ after its `expires` day, and a partition without examples is skipped and keeps w
A pair-grain experiment trains in the pair lane (`nn/pair_lane.py`) on the day's
`inbox_ranking_nn_pair_examples` object as well, and is graded within user against its pair baseline
-and its report-level reference (`nn/pair_grade.py`). A partition without pair examples skips the pair
-families only.
+and its report-level reference (`nn/pair_grade.py`). The job builds that object for the same
+partition first, from the day's report examples, and the pair experiments asset depends on it. The
+report and the pair experiments are two assets, so a failed pair examples step skips the pair
+experiments only. The job runs its steps in one process, one after the other, so two vector-heavy
+steps never share the pod's memory. A partition without pair examples skips the pair families only.
"""
import datetime
@@ -34,16 +37,41 @@ from products.signals.dags.inbox_ranking.common import (
)
from products.signals.dags.inbox_ranking.nn.dag import NN_ASSET_KWARGS, NN_EXAMPLES_TABLE, train_candidate
from products.signals.dags.inbox_ranking.nn.pair_lane import train_pair_candidate
+from products.signals.dags.inbox_ranking.nn.pairs import NN_PAIR_EXAMPLES_TABLE
from products.signals.dags.inbox_ranking.training.dag import _object_exists
-from products.signals.dags.inbox_ranking.training.families import PAIR_GRAIN, ModelFamily, experiment_families
+from products.signals.dags.inbox_ranking.training.families import (
+ PAIR_GRAIN,
+ REPORT_GRAIN,
+ ModelFamily,
+ experiment_families,
+)
+# The report-grain experiments keep the asset name they had before pair grain, so their Dagster
+# history continues.
NN_EXPERIMENTS_ASSET = "inbox_ranking_nn_experiment_candidates"
+NN_PAIR_EXPERIMENTS_ASSET = "inbox_ranking_nn_pair_experiment_candidates"
def live_experiments(families: tuple[ModelFamily, ...], day: datetime.date) -> tuple[ModelFamily, ...]:
return tuple(family for family in families if not family.expired(day))
+def grain_experiments(families: tuple[ModelFamily, ...], grain: str) -> tuple[ModelFamily, ...]:
+ return tuple(family for family in families if family.grain == grain)
+
+
+def experiment_selection(families: tuple[ModelFamily, ...], day: datetime.date) -> list[str]:
+ """The assets a run for `day` materializes: the report experiments when one is live, and the pair
+ examples with the pair experiments when a pair experiment is live. Empty when nothing is live."""
+ live = live_experiments(families, day)
+ selection: list[str] = []
+ if grain_experiments(live, REPORT_GRAIN):
+ selection.append(NN_EXPERIMENTS_ASSET)
+ if grain_experiments(live, PAIR_GRAIN):
+ selection += [NN_PAIR_EXAMPLES_TABLE, NN_PAIR_EXPERIMENTS_ASSET]
+ return selection
+
+
def run_experiments(
context: dagster.AssetExecutionContext, client, bucket: str, partition_key: str, families: tuple[ModelFamily, ...]
) -> dict[str, dagster.MetadataValue]:
@@ -74,24 +102,50 @@ def inbox_ranking_nn_experiment_candidates(context: dagster.AssetExecutionContex
if skip_unconfigured(context):
return
context.add_output_metadata(
- run_experiments(context, s3_client(), dataset_bucket(), context.partition_key, experiment_families())
+ run_experiments(
+ context,
+ s3_client(),
+ dataset_bucket(),
+ context.partition_key,
+ grain_experiments(experiment_families(), REPORT_GRAIN),
+ )
+ )
+
+
[email protected](name=NN_PAIR_EXPERIMENTS_ASSET, deps=[NN_EXAMPLES_TABLE, NN_PAIR_EXAMPLES_TABLE], **NN_ASSET_KWARGS)
+def inbox_ranking_nn_pair_experiment_candidates(context: dagster.AssetExecutionContext) -> None:
+ if skip_unconfigured(context):
+ return
+ context.add_output_metadata(
+ run_experiments(
+ context,
+ s3_client(),
+ dataset_bucket(),
+ context.partition_key,
+ grain_experiments(experiment_families(), PAIR_GRAIN),
+ )
)
inbox_ranking_nn_experiments_job = dagster.define_asset_job(
name="inbox_ranking_nn_experiments_job",
- selection=[NN_EXPERIMENTS_ASSET],
+ selection=[NN_EXPERIMENTS_ASSET, NN_PAIR_EXAMPLES_TABLE, NN_PAIR_EXPERIMENTS_ASSET],
+ # One step at a time: the multiprocess default would run the report experiments beside the pair
+ # examples, and the memory limit below covers one of them.
+ executor_def=dagster.in_process_executor,
tags={
**owner_tags,
# Per experiment: the five network fits of a candidate, then one fit per seed for each of
# the experiment and its baseline. The experiments run one after the other. A pair
# experiment adds one report-grain fit per seed for its reference, and its fits run over
- # the pair rows, a few times the report rows.
- "dagster/max_runtime": str(4 * 60 * 60),
+ # the pair rows, a few times the report rows. The pair examples step comes before the pair
+ # experiments and is bounded by its own job's 2h budget.
+ "dagster/max_runtime": str(6 * 60 * 60),
# As the nn job's candidate, plus the baseline's report matrix, held for the paired fits.
# A pair experiment holds its own and its baseline's pair report matrices (pair rows x
# 1536 floats each) and fits the reference before building them, so the reference's report
# matrix is gone by then. Pair rows a few times the report rows stay well inside the limit.
+ # The user offsets are one vector per (team, user, day), not per pair.
"dagster-k8s/config": {
"container_config": {
"resources": {
@@ -120,8 +174,11 @@ def inbox_ranking_nn_experiments_schedule(
if dataset_unconfigured():
return dagster.SkipReason(f"{S3_BUCKET_ENV} is not set; skipping until the dedicated bucket is provisioned")
previous_day = context.scheduled_execution_time.date() - datetime.timedelta(days=1)
- if not live_experiments(experiment_families(), previous_day):
+ selection = experiment_selection(experiment_families(), previous_day)
+ if not selection:
return dagster.SkipReason(f"no experiment family is live on {previous_day.isoformat()}")
return dagster.RunRequest(
- partition_key=previous_day.isoformat(), run_key=f"nn-experiments-{previous_day.isoformat()}"
+ partition_key=previous_day.isoformat(),
+ run_key=f"nn-experiments-{previous_day.isoformat()}",
+ asset_selection=[dagster.AssetKey(name) for name in selection],
)
diff --git a/products/signals/dags/inbox_ranking/nn/model.py b/products/signals/dags/inbox_ranking/nn/model.py
index 973544bab85c..dae792e24c1a 100644
--- a/products/signals/dags/inbox_ranking/nn/model.py
+++ b/products/signals/dags/inbox_ranking/nn/model.py
@@ -16,10 +16,15 @@ import numpy as np
from posthog import settings
from posthog.dataclasses import frozen
+from products.signals.backend.ranking.features import (
+ DAYS_SINCE_LAST_VIEW_CAP,
+ USER_HISTORY_KINDS,
+ PairUserHistoryFeatureSet,
+)
from products.signals.backend.ranking.model_store import ONNX_OUTPUT_HEADS_KEY
from products.signals.backend.ranking.serving_manifest import ONNX_MODEL_FILE, ONNX_MODEL_KIND
from products.signals.dags.inbox_ranking.common import DATASET_VERSION
-from products.signals.dags.inbox_ranking.dataset.project_vectors import VECTOR_KINDS
+from products.signals.dags.inbox_ranking.dataset.project_vectors import USER_HISTORY_LIMIT, VECTOR_KINDS
from products.signals.dags.inbox_ranking.multihead.inputs import ONNX_INPUTS, ONNX_OUTPUT, NetworkConfig, NetworkInputs
from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable
from products.signals.dags.inbox_ranking.nn.examples import NN_HEAD_NAMES, NN_HEADS
@@ -53,6 +58,8 @@ class TrainedNetwork:
refit_rows: int
# The project-vector kinds the graphs take, empty for a report-only network.
vector_kinds: tuple[str, ...]
+ # The user-history kinds the graphs take, empty for a network without a `user` input.
+ user_vector_kinds: tuple[str, ...] = ()
def network_vector_kinds(inputs: NetworkInputs) -> tuple[str, ...]:
@@ -60,6 +67,17 @@ def network_vector_kinds(inputs: NetworkInputs) -> tuple[str, ...]:
return VECTOR_KINDS if inputs.context is not None else ()
+def network_user_kinds(inputs: NetworkInputs) -> tuple[str, ...]:
+ """The user-history kinds a network fit on `inputs` takes, empty for a network without them."""
+ return USER_HISTORY_KINDS if inputs.users is not None else ()
+
+
+def family_user_kinds(family: ModelFamily) -> tuple[str, ...]:
+ """The user-history kinds the pair lane gives a `family` network. A pair set's network takes
+ them even when the set holds them at zero, so the two arms of a comparison share one shape."""
+ return USER_HISTORY_KINDS if isinstance(family.feature_set, PairUserHistoryFeatureSet) else ()
+
+
def family_heads(family: ModelFamily) -> tuple[Head, ...]:
"""The heads a `family` network is fit on, in its label and output column order."""
if family.grain == PAIR_GRAIN:
@@ -72,9 +90,12 @@ def network_recipe(
config: NetworkConfig,
vector_kinds: Sequence[str],
pair_settings: Mapping[str, Any] | None = None,
+ *,
+ user_vector_kinds: Sequence[str] = (),
) -> dict[str, Any]:
- """`pair_settings` is the pair examples' label horizon and schema, for a pair-grain family only,
- so a report-grain family's recipe and hash stay as they were."""
+ """`pair_settings` is the pair examples' label horizon and schema, and `user_vector_kinds` the
+ user-history kinds the graphs take, both for a pair-grain family only, so a report-grain family's
+ recipe and hash stay as they were."""
recipe = model_recipe(
model_kind=ONNX_MODEL_KIND,
feature_set=family.feature_set,
@@ -83,7 +104,18 @@ def network_recipe(
project_vector_kinds=vector_kinds,
)
if family.grain == PAIR_GRAIN:
- recipe = {**recipe, "grain": PAIR_GRAIN, "pair": dict(pair_settings or {})}
+ recipe = {
+ **recipe,
+ "grain": PAIR_GRAIN,
+ "pair": dict(pair_settings or {}),
+ "user_vector_kinds": list(user_vector_kinds),
+ # Both change the user inputs, so they matter only to a model that reads user history.
+ "user_vector_settings": (
+ {"history_limit": USER_HISTORY_LIMIT, "days_since_last_view_cap": DAYS_SINCE_LAST_VIEW_CAP}
+ if user_vector_kinds
+ else {}
+ ),
+ }
return recipe
@@ -142,7 +174,13 @@ def candidate_metadata(
}
if family.grain == PAIR_GRAIN:
metadata["grain"] = PAIR_GRAIN
- return with_recipe(metadata, network_recipe(family, config, trained.vector_kinds, pair_settings))
+ metadata["user_vector_kinds"] = list(trained.user_vector_kinds)
+ return with_recipe(
+ metadata,
+ network_recipe(
+ family, config, trained.vector_kinds, pair_settings, user_vector_kinds=trained.user_vector_kinds
+ ),
+ )
def onnx_probabilities(model: bytes, inputs: NetworkInputs, output_heads: Sequence[str]) -> dict[str, np.ndarray]:
@@ -155,7 +193,7 @@ def onnx_probabilities(model: bytes, inputs: NetworkInputs, output_heads: Sequen
session = ort.InferenceSession(model, providers=["CPUExecutionProvider"])
declared = {node.name for node in session.get_inputs()}
- report_input, project_input, history_input = ONNX_INPUTS
+ report_input, project_input, history_input, user_input = ONNX_INPUTS
feeds: dict[str, np.ndarray] = {report_input: np.ascontiguousarray(inputs.reports, dtype=np.float32)}
if project_input in declared:
if inputs.context is None:
@@ -163,6 +201,10 @@ def onnx_probabilities(model: bytes, inputs: NetworkInputs, output_heads: Sequen
pairs = inputs.context.index
feeds[project_input] = np.ascontiguousarray(inputs.context.vectors[pairs], dtype=np.float32)
feeds[history_input] = np.ascontiguousarray(inputs.context.has_history[pairs], dtype=np.float32)
+ if user_input in declared:
+ if inputs.users is None:
+ raise ValueError("the graph takes user offsets and the inputs carry none")
+ feeds[user_input] = np.ascontiguousarray(inputs.users.vectors[inputs.users.index], dtype=np.float32)
(probabilities,) = session.run([ONNX_OUTPUT], {name: value for name, value in feeds.items() if name in declared})
return {head: probabilities[:, column] for column, head in enumerate(output_heads)}
diff --git a/products/signals/dags/inbox_ranking/nn/pair_lane.py b/products/signals/dags/inbox_ranking/nn/pair_lane.py
index e6b304c4c4bf..9125fa61c53d 100644
--- a/products/signals/dags/inbox_ranking/nn/pair_lane.py
+++ b/products/signals/dags/inbox_ranking/nn/pair_lane.py
@@ -7,28 +7,51 @@ the day's `inbox_ranking_nn_examples` object, and its team's project vectors as
birth day. The experiment and its pair baseline fit with each of `EXPERIMENT_SEEDS` on the same train
pairs and are graded on the same holdout pairs.
+A family on a pair set (`PairUserHistoryFeatureSet`) also reads the pair's user columns and the
+user's history as offsets from the team's project vectors. Each arm builds its own inputs from the
+pair rows: the report part is built once per report and repeated, but the user columns and offsets
+are per pair, so two users of one report never share them. A set with `user_history` off gets zero
+user columns and zero offsets, whatever the pair rows hold. The user vectors are rebuilt from the
+dt=D interactions and embeddings snapshots through `pair_user_vectors`, on the history of teams with
+training consent only, as the pair examples asset builds them.
+
The reference is a report-grain serving family. For each seed it is fit on the report rows of the
pair train days, at report grain with every `nn` head, and each holdout pair takes its report's
score. That answers "better than a report-level score" on the same days without loading the served
model, and the read stays paired on seeds. The reference is fit first and its report inputs are
dropped before the pair inputs are built, so the two matrices are never held at once.
-A partition without pair examples, report examples or the dt=D snapshots keeps what it holds.
+A partition without pair examples, report examples, the dt=D snapshots or, for a family that reads
+user history, the dt=D interactions snapshot keeps what it holds.
"""
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
from typing import Any
import numpy as np
import pandas as pd
import dagster
+import pyarrow as pa
+import pyarrow.compute as pc
from posthog import settings
-
-from products.signals.backend.ranking.features import Extras, FeatureSet
+from posthog.dataclasses import frozen
+
+from products.signals.backend.ranking.features import (
+ EMBEDDING_DIMENSIONS,
+ PAIR_STATE_COLUMNS,
+ USER_HISTORY_KINDS,
+ USER_REPORTS_COLUMNS,
+ Extras,
+ FeatureSet,
+ PairUserHistoryFeatureSet,
+ user_offsets,
+)
from products.signals.dags.inbox_ranking.common import partition_object_key, read_parquet_if_exists
-from products.signals.dags.inbox_ranking.dataset.project_vectors import VECTOR_KINDS
-from products.signals.dags.inbox_ranking.multihead.inputs import NetworkInputs
+from products.signals.dags.inbox_ranking.consent import training_consent_team_ids
+from products.signals.dags.inbox_ranking.dataset.dag import USER_INTERACTIONS_TABLE
+from products.signals.dags.inbox_ranking.dataset.project_vectors import USER_INTERACTION_COLUMNS, VECTOR_KINDS
+from products.signals.dags.inbox_ranking.multihead.inputs import NetworkInputs, ProjectContext, UserContext
from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable, project_context, report_matrix
from products.signals.dags.inbox_ranking.nn.dag import (
EXPERIMENT_SEEDS,
@@ -39,12 +62,14 @@ from products.signals.dags.inbox_ranking.nn.dag import (
write_candidate,
)
from products.signals.dags.inbox_ranking.nn.examples import NN_HEAD_NAMES, NN_HEADS, examples_from_arrow, network_inputs
-from products.signals.dags.inbox_ranking.nn.model import family_heads, network_recipe
+from products.signals.dags.inbox_ranking.nn.model import family_heads, family_user_kinds, network_recipe
from products.signals.dags.inbox_ranking.nn.pair_grade import pair_vs_baseline_block
from products.signals.dags.inbox_ranking.nn.pairs import (
NN_PAIR_EXAMPLES_TABLE,
PAIR_EXAMPLES_SCHEMA_VERSION,
PAIR_HORIZON_DAYS,
+ pair_user_vectors,
+ team_organizations,
)
from products.signals.dags.inbox_ranking.training.dag import check_recipe
from products.signals.dags.inbox_ranking.training.families import MODEL_FAMILIES, ModelFamily
@@ -56,16 +81,28 @@ PAIR_SETTINGS: dict[str, Any] = {
"examples_schema_version": PAIR_EXAMPLES_SCHEMA_VERSION,
}
_REPORT_KEY = ["report_id", "snapshot_date"]
+# One user's history as of one birth day, read against one team's project vectors.
+_USER_KEY = ["report_team_id", "user_distinct_id", "snapshot_date"]
_PAIR_COLUMNS = [
"report_id",
"snapshot_date",
"report_team_id",
"user_distinct_id",
"prior_views",
- "is_suggested_reviewer",
+ *PAIR_STATE_COLUMNS,
]
+@frozen
+class UserHistory:
+ """What `pair_user_vectors` reads beside the embeddings snapshot."""
+
+ # The dt=D `inbox_user_report_interactions` snapshot, only the rows of teams with training consent.
+ interactions: pa.Table
+ # Each consented team's organization, so a user's history stays inside one organization.
+ organizations: Mapping[int, str]
+
+
def pair_example_table(pairs: pd.DataFrame, report_rows: pd.DataFrame, heads: Sequence[str]) -> ExampleTable:
"""One row per pair, with labels and holdout flags in `heads` order.
@@ -110,15 +147,99 @@ def reference_train_rows(report_table: ExampleTable, pair_table: ExampleTable) -
def pair_report_matrix(rows: pd.DataFrame, extras: Extras, feature_set: FeatureSet) -> np.ndarray:
- """The report input of every pair. Built once per report moment, then repeated for its pairs."""
+ """The report input of every pair. The report part is built once per report moment, then
+ repeated for its pairs. A pair set's user columns are built per pair after it."""
reports = rows.drop_duplicates(_REPORT_KEY).reset_index(drop=True)
- return report_matrix(reports, extras, feature_set)[report_positions(reports, rows)]
+ report_set = feature_set.report_feature_set if isinstance(feature_set, PairUserHistoryFeatureSet) else feature_set
+ report_part = report_matrix(reports, extras, report_set)
+ positions = report_positions(reports, rows)
+ if not isinstance(feature_set, PairUserHistoryFeatureSet):
+ return report_part[positions]
+ width = report_part.shape[1]
+ matrix = np.empty((len(rows), len(feature_set.feature_names)), dtype=np.float32)
+ matrix[:, :width] = report_part[positions]
+ matrix[:, width:] = feature_set.pair_matrix(rows)
+ return matrix
+
+
+def user_context(
+ rows: pd.DataFrame, project: ProjectContext, history: UserHistory, embeddings: pa.Table
+) -> UserContext:
+ """The user offsets each pair of `rows` reads: its own user's vectors as of the start of its birth
+ day, minus its team's project vectors from `project` (`user_offsets`).
+
+ One vector per (team, user, birth day), so two pairs of one user that day share it and two users
+ of one report never do. A kind counts only when the pair's stored `user_<kind>_reports` says the
+ user had a report of it, the same count the pair object records.
+ """
+ keys = rows[_USER_KEY].drop_duplicates()
+ index = pd.MultiIndex.from_frame(keys).get_indexer(pd.MultiIndex.from_frame(rows[_USER_KEY]))
+ first_rows = keys.index.to_numpy()
+ vectors = pair_user_vectors(
+ keys.reset_index(drop=True), history.interactions, embeddings, organizations=history.organizations
+ )
+ projects = project.index[first_rows]
+ offsets = user_offsets(
+ vectors,
+ rows[list(USER_REPORTS_COLUMNS)].to_numpy(dtype=np.float32)[first_rows],
+ project.vectors[projects],
+ project.has_history[projects],
+ )
+ return UserContext(vectors=offsets, index=index.astype(np.int64))
+
+
+def zero_user_context(rows: int) -> UserContext:
+ """A user offset of zero for every row, one shared key, so the arm keeps the input shape and
+ reads nothing of anyone's history."""
+ vectors = np.zeros((1, len(USER_HISTORY_KINDS), EMBEDDING_DIMENSIONS), dtype=np.float32)
+ return UserContext(vectors=vectors, index=np.zeros(rows, dtype=np.int64))
-def pair_network_inputs(table: ExampleTable, extras: Extras, day: DayInputs, feature_set: FeatureSet) -> NetworkInputs:
- return NetworkInputs(
- reports=pair_report_matrix(table.rows, extras, feature_set),
- context=project_context(table.rows, day.labels, day.embeddings),
+def pair_network_inputs(
+ table: ExampleTable,
+ extras: Extras,
+ feature_set: FeatureSet,
+ project: ProjectContext,
+ history: UserHistory | None,
+ embeddings: pa.Table,
+) -> NetworkInputs:
+ """One arm's inputs, built from the pair rows for `feature_set` alone. The project context is the
+ team's and holds nothing personal, so the arms share it."""
+ users = None
+ if isinstance(feature_set, PairUserHistoryFeatureSet):
+ if not feature_set.user_history:
+ users = zero_user_context(len(table.rows))
+ elif history is None:
+ raise ValueError(f"the {feature_set.name} feature set needs the user history")
+ else:
+ users = user_context(table.rows, project, history, embeddings)
+ return NetworkInputs(reports=pair_report_matrix(table.rows, extras, feature_set), context=project, users=users)
+
+
+def load_user_history(client, bucket: str, prefix: str, partition_key: str) -> UserHistory | None:
+ """The dt=D interactions snapshot of the teams with training consent, and their organizations;
+ None when the snapshot is missing."""
+ interactions = read_parquet_if_exists(
+ client,
+ bucket,
+ partition_object_key(prefix, USER_INTERACTIONS_TABLE, partition_key),
+ columns=list(USER_INTERACTION_COLUMNS),
+ )
+ if interactions is None:
+ return None
+ consent_team_ids = training_consent_team_ids()
+ consented = pa.array(sorted(consent_team_ids), pa.int64())
+ return UserHistory(
+ interactions=interactions.filter(pc.is_in(interactions.column("report_team_id"), value_set=consented)),
+ organizations=team_organizations(consent_team_ids),
+ )
+
+
+def _reads_user_history(family: ModelFamily | None) -> bool:
+ return (
+ family is not None
+ and isinstance(family.feature_set, PairUserHistoryFeatureSet)
+ and family.feature_set.user_history
)
@@ -214,6 +335,15 @@ def train_pair_candidate(
f"no {', '.join(missing_extras)} side input for dt={partition_key}; {family.name} keeps this partition "
"as it stands",
)
+ history = None
+ if _reads_user_history(family) or _reads_user_history(baseline):
+ history = load_user_history(client, bucket, prefix, partition_key)
+ if history is None:
+ return _skipped(
+ context,
+ f"no {USER_INTERACTIONS_TABLE} snapshot for dt={partition_key}; {family.name} reads user history "
+ "and keeps this partition as it stands",
+ )
# Pair inputs always carry the project context, so the graphs take every project-vector kind.
check_recipe(
client,
@@ -221,7 +351,9 @@ def train_pair_candidate(
prefix,
family.name,
partition_key,
- network_recipe(family, NETWORK_CONFIG, VECTOR_KINDS, PAIR_SETTINGS),
+ network_recipe(
+ family, NETWORK_CONFIG, VECTOR_KINDS, PAIR_SETTINGS, user_vector_kinds=family_user_kinds(family)
+ ),
)
from products.signals.dags.inbox_ranking.nn.train import ( # noqa: PLC0415 — keeps torch off the location's import path
@@ -234,9 +366,14 @@ def train_pair_candidate(
reference_seed_scores = None
if reference is not None and has_train_pairs:
reference_seed_scores = reference_scores(examples.table, table, reference, extras, day, family.heads)
- inputs = pair_network_inputs(table, extras, day, family.feature_set)
- baseline_reports = pair_report_matrix(table.rows, extras, baseline.feature_set) if baseline is not None else None
- del day
+ project = project_context(table.rows, day.labels, day.embeddings)
+ inputs = pair_network_inputs(table, extras, family.feature_set, project, history, day.embeddings)
+ baseline_inputs = (
+ pair_network_inputs(table, extras, baseline.feature_set, project, history, day.embeddings)
+ if baseline is not None
+ else None
+ )
+ del day, history
trained = train_network(table, inputs, heads, NETWORK_CONFIG)
vs_baseline = None
@@ -249,14 +386,8 @@ def train_pair_candidate(
seeds=EXPERIMENT_SEEDS,
experiment_scores=seed_scores(table, inputs, heads, NETWORK_CONFIG, EXPERIMENT_SEEDS),
baseline_scores=(
- seed_scores(
- table,
- NetworkInputs(reports=baseline_reports, context=inputs.context),
- heads,
- NETWORK_CONFIG,
- EXPERIMENT_SEEDS,
- )
- if baseline_reports is not None
+ seed_scores(table, baseline_inputs, heads, NETWORK_CONFIG, EXPERIMENT_SEEDS)
+ if baseline_inputs is not None
else None
),
reference_scores=reference_seed_scores,
@@ -272,8 +403,13 @@ def train_pair_candidate(
vs_baseline=vs_baseline,
pair_settings=PAIR_SETTINGS,
)
+ with_history = table.rows["prior_views"].to_numpy() > 0
return {
**metadata,
"pair_rows": dagster.MetadataValue.int(len(table.rows)),
+ "pair_users": dagster.MetadataValue.int(
+ int(table.rows[["report_team_id", "user_distinct_id"]].drop_duplicates().shape[0])
+ ),
+ "pair_rows_with_prior_views": dagster.MetadataValue.int(int(with_history.sum())),
"pairs_without_report_row": dagster.MetadataValue.int(without_report),
}
diff --git a/products/signals/dags/inbox_ranking/nn/pairs.py b/products/signals/dags/inbox_ranking/nn/pairs.py
index f7fe3356a232..b84db8583807 100644
--- a/products/signals/dags/inbox_ranking/nn/pairs.py
+++ b/products/signals/dags/inbox_ranking/nn/pairs.py
@@ -29,7 +29,7 @@ import pyarrow.compute as pc
from posthog import settings
from posthog.models import Team
-from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS
+from products.signals.backend.ranking.features import DAYS_SINCE_LAST_VIEW_CAP, EMBEDDING_DIMENSIONS
from products.signals.dags.inbox_ranking.common import (
dataset_bucket,
owner_tags,
@@ -64,7 +64,7 @@ PAIR_EXAMPLES_SCHEMA_VERSION = 1
_REPORT_HEAD = HEADS_BY_NAME["open"]
PAIR_HORIZON_DAYS = _REPORT_HEAD.horizon_days
PAIR_HEADS = PAIR_HEAD_NAMES
-DAYS_SINCE_VIEW_CAP = 90.0
+DAYS_SINCE_VIEW_CAP = DAYS_SINCE_LAST_VIEW_CAP
# (name, lowest, highest) prior views per bucket. The within-user grade needs enough rows in the
# buckets with real history to be readable.
HISTORY_BUCKETS: tuple[tuple[str, int, int | None], ...] = (
diff --git a/products/signals/dags/inbox_ranking/nn/train.py b/products/signals/dags/inbox_ranking/nn/train.py
index 37d674e1cbd6..6cd99d5dfe97 100644
--- a/products/signals/dags/inbox_ranking/nn/train.py
+++ b/products/signals/dags/inbox_ranking/nn/train.py
@@ -24,7 +24,12 @@ from products.signals.dags.inbox_ranking.dataset.project_vectors import VECTOR_K
from products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfig, NetworkInputs
from products.signals.dags.inbox_ranking.multihead.network import MultiHeadNetwork, export_onnx
from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable
-from products.signals.dags.inbox_ranking.nn.model import NetworkHead, TrainedNetwork, network_vector_kinds
+from products.signals.dags.inbox_ranking.nn.model import (
+ NetworkHead,
+ TrainedNetwork,
+ network_user_kinds,
+ network_vector_kinds,
+)
from products.signals.dags.inbox_ranking.training.heads import Head
from products.signals.dags.inbox_ranking.training.train import NULL_PERMUTATIONS, head_metrics
@@ -38,6 +43,7 @@ def _fit(
kinds=len(VECTOR_KINDS),
config=config,
report_features=inputs.reports.shape[1] - EMBEDDING_DIMENSIONS,
+ user_kinds=inputs.users.vectors.shape[1] if inputs.users is not None else 0,
)
network.fit(inputs, labels, rows)
return network
@@ -134,6 +140,7 @@ def train_network(
network_train_rows=len(train_rows),
refit_rows=len(refit_rows),
vector_kinds=network_vector_kinds(inputs),
+ user_vector_kinds=network_user_kinds(inputs),
)
diff --git a/products/signals/dags/inbox_ranking/tests/test_experiments.py b/products/signals/dags/inbox_ranking/tests/test_experiments.py
index e4a75bbfc8f2..e05c88ebb77b 100644
--- a/products/signals/dags/inbox_ranking/tests/test_experiments.py
+++ b/products/signals/dags/inbox_ranking/tests/test_experiments.py
@@ -19,21 +19,24 @@ from products.signals.backend.ranking.features import (
REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
REPORT_SOURCES_EXTRA,
)
-from products.signals.backend.ranking.serving_manifest import ONNX_MODEL_KIND
+from products.signals.backend.ranking.model_store import ONNX_OUTPUT_HEADS_KEY
+from products.signals.backend.ranking.serving_manifest import ONNX_MODEL_FILE, ONNX_MODEL_KIND
from products.signals.dags.inbox_ranking.common import partition_object_key
from products.signals.dags.inbox_ranking.dataset.dag import (
EMBEDDINGS_TABLE,
LABELS_TABLE,
SIGNAL_EMBEDDINGS_SCHEMA,
SIGNAL_EMBEDDINGS_TABLE,
+ USER_INTERACTIONS_SCHEMA,
+ USER_INTERACTIONS_TABLE,
)
from products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfig
from products.signals.dags.inbox_ranking.multihead.local import report_vectors
-from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable, report_matrix
+from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable, project_context, report_matrix
from products.signals.dags.inbox_ranking.nn import experiments, pair_lane
from products.signals.dags.inbox_ranking.nn.dag import NN_EXAMPLES_TABLE
from products.signals.dags.inbox_ranking.nn.examples import NN_HEAD_NAMES, NN_HEADS, build_examples, examples_arrow
-from products.signals.dags.inbox_ranking.nn.model import network_recipe, vs_baseline_block
+from products.signals.dags.inbox_ranking.nn.model import network_recipe, onnx_probabilities, vs_baseline_block
from products.signals.dags.inbox_ranking.nn.pair_grade import DOGFOOD_TEAM_ID, pair_vs_baseline_block
from products.signals.dags.inbox_ranking.nn.pair_lane import (
PAIR_SETTINGS,
@@ -42,13 +45,23 @@ from products.signals.dags.inbox_ranking.nn.pair_lane import (
train_pair_candidate,
)
from products.signals.dags.inbox_ranking.nn.pairs import NN_PAIR_EXAMPLES_TABLE, PAIR_EXAMPLES_SCHEMA
-from products.signals.dags.inbox_ranking.tests.test_multihead import D0, DAYS, HOLDOUT_DAYS, _SyntheticInbox
+from products.signals.dags.inbox_ranking.tests.test_multihead import (
+ D0,
+ DAYS,
+ HOLDOUT_DAYS,
+ TEAMS,
+ _birth,
+ _SyntheticInbox,
+)
from products.signals.dags.inbox_ranking.tests.test_nn import _S3, _parquet
from products.signals.dags.inbox_ranking.tests.test_training import _ModelStoreS3
from products.signals.dags.inbox_ranking.training import side_inputs
from products.signals.dags.inbox_ranking.training.dag import METADATA_FILE, model_object_key
from products.signals.dags.inbox_ranking.training.families import (
+ MODEL_FAMILIES,
NN_MODEL_NAME,
+ NN_PAIR_BASE_MODEL_NAME,
+ NN_PAIR_USER_MODEL_NAME,
NN_WITH_PRIO_FAMILY,
NN_WITH_PRIO_MODEL_NAME,
PAIR_GRAIN,
@@ -149,6 +162,29 @@ def test_an_experiment_must_name_a_registered_serving_baseline_and_an_expiry():
]
+ "families,selection",
+ [
+ ((_experiment("report", datetime.date(2026, 10, 20)),), [experiments.NN_EXPERIMENTS_ASSET]),
+ (
+ (
+ _experiment("expired", datetime.date(2026, 10, 19)),
+ _pair_experiment("pair", expires=datetime.date(2026, 10, 20)),
+ ),
+ [NN_PAIR_EXAMPLES_TABLE, experiments.NN_PAIR_EXPERIMENTS_ASSET],
+ ),
+ ((_pair_experiment("expired_pair", expires=datetime.date(2026, 10, 19)),), []),
+ ],
+ ids=["report_only", "pair_only", "nothing_live"],
+)
+def test_a_run_builds_pair_examples_only_for_a_live_pair_experiment_and_trains_it_after_them(families, selection):
+ assert experiments.experiment_selection(families, datetime.date(2026, 10, 20)) == selection
+ pair_examples = dagster.AssetKey(NN_PAIR_EXAMPLES_TABLE)
+ # The report experiments stay apart from the pair examples, so a failed pair step cannot skip them.
+ assert pair_examples in experiments.inbox_ranking_nn_pair_experiment_candidates.dependency_keys
+ assert pair_examples not in experiments.inbox_ranking_nn_experiment_candidates.dependency_keys
+
+
@pytest.mark.parametrize(
"build,message",
[
@@ -517,33 +553,51 @@ def _pair_examples(report_rows: pd.DataFrame, report_labels: np.ndarray, report_
return pa.Table.from_pylist(rows, schema=PAIR_EXAMPLES_SCHEMA)
[email protected]("pairs_present", [True, False])
-def test_a_pair_experiment_trains_on_pair_rows_and_is_graded_within_user(monkeypatch, pairs_present):
+def _reader_history(inbox: _SyntheticInbox, day: str) -> pa.Table:
+ """Each team's reader viewed every report of the team it opens, the day after its birth."""
+ rows = [
+ {
+ "snapshot_date": datetime.date.fromisoformat(day),
+ "report_id": report_id,
+ "report_team_id": int(report["team"]),
+ "user_distinct_id": f"reader-{report['team']}",
+ "first_viewed_at": (_birth(int(report["born"])) + pd.Timedelta(days=1)).to_pydatetime(),
+ }
+ for report_id, report in inbox.reports.iterrows()
+ if report["open"]
+ ]
+ return pa.Table.from_pylist(rows, schema=USER_INTERACTIONS_SCHEMA)
+
+
+ "model_name,pairs_present",
+ [(NN_PAIR_USER_MODEL_NAME, True), (NN_PAIR_BASE_MODEL_NAME, True), (NN_PAIR_USER_MODEL_NAME, False)],
+ ids=["user_family", "base_family", "user_family_without_pairs"],
+)
+def test_a_registered_pair_family_trains_scores_and_is_graded_within_user(monkeypatch, model_name, pairs_present):
pytest.importorskip("torch")
monkeypatch.setattr("products.signals.dags.inbox_ranking.nn.dag.capture_training_events", lambda *a, **k: None)
monkeypatch.setattr(
pair_lane, "NETWORK_CONFIG", NetworkConfig(projection_dim=8, trunk_dim=8, dropout=0.1, epochs=40, batch_size=32)
)
monkeypatch.setattr(pair_lane, "EXPERIMENT_SEEDS", (0, 1))
- base = _pair_experiment("nn_pair_stub_base")
- family = _pair_experiment("nn_pair_stub", baseline=base.name)
- registry = (NN_WITH_PRIO_FAMILY, base, family)
- assert family_problems(registry) == []
- monkeypatch.setattr(pair_lane, "MODEL_FAMILIES", registry)
+ monkeypatch.setattr(pair_lane, "training_consent_team_ids", lambda: frozenset(TEAMS))
+ monkeypatch.setattr(pair_lane, "team_organizations", lambda team_ids: {t: f"org-{t}" for t in team_ids})
+ family = next(family for family in MODEL_FAMILIES if family.name == model_name)
inbox = _SyntheticInbox()
embeddings, labels = inbox.embeddings(), inbox.labels()
examples = build_examples(inbox.snapshots(), report_vectors(embeddings), holdout_days=HOLDOUT_DAYS)
table = examples.table
day = (D0 + datetime.timedelta(days=DAYS - 1)).isoformat()
+ pairs = _pair_examples(table.rows, table.labels, table.holdout)
objects = {
partition_object_key(PREFIX, NN_EXAMPLES_TABLE, day): _parquet(examples_arrow(examples)),
partition_object_key(PREFIX, EMBEDDINGS_TABLE, day): _parquet(embeddings),
partition_object_key(PREFIX, LABELS_TABLE, day): _parquet(labels),
+ partition_object_key(PREFIX, USER_INTERACTIONS_TABLE, day): _parquet(_reader_history(inbox, day)),
}
if pairs_present:
- objects[partition_object_key(PREFIX, NN_PAIR_EXAMPLES_TABLE, day)] = _parquet(
- _pair_examples(table.rows, table.labels, table.holdout)
- )
+ objects[partition_object_key(PREFIX, NN_PAIR_EXAMPLES_TABLE, day)] = _parquet(pairs)
client = _S3(objects)
with dagster.build_asset_context(partition_key=day) as context:
@@ -556,14 +610,42 @@ def test_a_pair_experiment_trains_on_pair_rows_and_is_graded_within_user(monkeyp
return
metadata = json.loads(client.objects[metadata_key])
assert (metadata["grain"], metadata["recipe"]["grain"]) == (PAIR_GRAIN, PAIR_GRAIN)
+ assert (metadata["feature_set"], metadata["user_vector_kinds"]) == (family.feature_set.name, ["open", "dismiss"])
assert [head["head"] for head in metadata["heads"]] == ["open", "action"]
block = metadata["vs_baseline"]
- assert (block["grain"], block["baseline"], block["reference"]) == (PAIR_GRAIN, base.name, NN_WITH_PRIO_MODEL_NAME)
+ assert (block["grain"], block["baseline"], block["reference"]) == (
+ PAIR_GRAIN,
+ family.baseline,
+ NN_WITH_PRIO_MODEL_NAME,
+ )
for grade in block["heads"].values():
every = grade["scopes"]["all"]["all"]
- assert every["vs_baseline"]["macro"]["paired_seeds"] == every["vs_reference"]["macro"]["paired_seeds"] == 2
+ assert every["vs_reference"]["macro"]["paired_seeds"] == 2
+ if family.baseline is None:
+ assert every["vs_baseline"] is None
+ else:
+ assert every["vs_baseline"]["macro"]["paired_seeds"] == 2
# The skimmers never open, so only the readers have both classes; the dogfood readers drop out.
assert grade["scopes"]["non_dogfood"]["all"]["users"] < every["users"]
opened = block["heads"]["open"]["scopes"]["all"]["all"]
- assert min(opened["experiment"]["macro"]) > 0.75
+ assert min(opened["experiment"]["macro"]) > 0.65
assert min(opened["reference"]["macro"]) > 0.75
+
+ pair_table = pair_lane.pair_example_table(pairs.to_pandas(), table.rows, family.heads)
+ history = pair_lane.load_user_history(client, "bucket", PREFIX, day)
+ inputs = pair_lane.pair_network_inputs(
+ pair_table,
+ report_vectors(embeddings),
+ family.feature_set,
+ project_context(pair_table.rows, labels, embeddings),
+ history,
+ embeddings,
+ )
+ scores = onnx_probabilities(
+ client.objects[model_object_key(PREFIX, family.name, day, ONNX_MODEL_FILE)],
+ inputs,
+ metadata[ONNX_OUTPUT_HEADS_KEY],
+ )
+ cold_start = (pair_table.rows["prior_views"] == 0).to_numpy()
+ assert cold_start.any()
+ assert all(np.isfinite(head_scores).all() for head_scores in scores.values())
diff --git a/products/signals/dags/inbox_ranking/tests/test_multihead_network.py b/products/signals/dags/inbox_ranking/tests/test_multihead_network.py
index d7fd2d67d6a5..88143ca5ad9c 100644
--- a/products/signals/dags/inbox_ranking/tests/test_multihead_network.py
+++ b/products/signals/dags/inbox_ranking/tests/test_multihead_network.py
@@ -8,9 +8,14 @@ import pytest
import numpy as np
from parameterized import parameterized
-from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS
+from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS, USER_HISTORY_KINDS
from products.signals.dags.inbox_ranking.dataset.project_vectors import VECTOR_KINDS
-from products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfig, NetworkInputs, ProjectContext
+from products.signals.dags.inbox_ranking.multihead.inputs import (
+ NetworkConfig,
+ NetworkInputs,
+ ProjectContext,
+ UserContext,
+)
torch = pytest.importorskip("torch")
onnxruntime = pytest.importorskip("onnxruntime")
@@ -42,7 +47,7 @@ print("torch" in sys.modules)
"""
-def _inputs(rows: int, *, with_context: bool, seed: int = 0) -> NetworkInputs:
+def _inputs(rows: int, *, with_context: bool, seed: int = 0, user_kinds: int = 0) -> NetworkInputs:
rng = np.random.default_rng(seed)
reports = rng.normal(0.0, 1 / np.sqrt(EMBEDDING_DIMENSIONS), size=(rows, EMBEDDING_DIMENSIONS)).astype(np.float32)
if not with_context:
@@ -54,7 +59,15 @@ def _inputs(rows: int, *, with_context: bool, seed: int = 0) -> NetworkInputs:
has_history=rng.integers(0, 2, size=(rows, KINDS)).astype(np.float32),
index=np.arange(rows),
)
- return NetworkInputs(reports=reports, context=context)
+ users = None
+ if user_kinds:
+ users = UserContext(
+ vectors=rng.normal(
+ 0.0, 1 / np.sqrt(EMBEDDING_DIMENSIONS), size=(rows, user_kinds, EMBEDDING_DIMENSIONS)
+ ).astype(np.float32),
+ index=np.arange(rows),
+ )
+ return NetworkInputs(reports=reports, context=context, users=users)
def test_a_nan_label_adds_no_gradient_and_no_loss() -> None:
@@ -91,12 +104,16 @@ def test_a_seeded_fit_is_repeatable(_name: str, with_context: bool) -> None:
np.testing.assert_array_equal(runs[0][1], runs[1][1])
[email protected]([("report_only", False), ("with_project_vectors", True)])
-def test_onnx_export_matches_the_torch_network(_name: str, with_context: bool) -> None:
+ [("report_only", False, 0), ("with_project_vectors", True, 0), ("with_user_history", True, len(USER_HISTORY_KINDS))]
+)
+def test_onnx_export_matches_the_torch_network(_name: str, with_context: bool, user_kinds: int) -> None:
rows = 37
- inputs = _inputs(rows, with_context=with_context, seed=2)
+ inputs = _inputs(rows, with_context=with_context, seed=2, user_kinds=user_kinds)
labels = np.random.default_rng(3).integers(0, 2, size=(rows, len(HEADS))).astype(np.float32)
- network = MultiHeadNetwork(heads=HEADS, with_context=with_context, kinds=KINDS, config=CONFIG)
+ network = MultiHeadNetwork(
+ heads=HEADS, with_context=with_context, kinds=KINDS, config=CONFIG, user_kinds=user_kinds
+ )
network.fit(inputs, labels, np.arange(rows))
with tempfile.TemporaryDirectory() as directory:
@@ -107,6 +124,8 @@ def test_onnx_export_matches_the_torch_network(_name: str, with_context: bool) -
feeds = {"report": inputs.reports}
if inputs.context is not None:
feeds |= {"project": inputs.context.vectors, "has_history": inputs.context.has_history}
+ if inputs.users is not None:
+ feeds |= {"user": inputs.users.vectors}
assert [i.name for i in session.get_inputs()] == list(feeds)
assert [o.name for o in session.get_outputs()] == ["probabilities"]
(probabilities,) = session.run(None, feeds)
diff --git a/products/signals/dags/inbox_ranking/tests/test_pairs.py b/products/signals/dags/inbox_ranking/tests/test_pairs.py
index d39b819f7c7b..2365858b50cf 100644
--- a/products/signals/dags/inbox_ranking/tests/test_pairs.py
+++ b/products/signals/dags/inbox_ranking/tests/test_pairs.py
@@ -13,7 +13,7 @@ from parameterized import parameterized
from posthog import settings
-from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS
+from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS, PAIR_FEATURE_NAMES
from products.signals.dags.inbox_ranking.common import partition_object_key
from products.signals.dags.inbox_ranking.dataset.dag import (
EMBEDDINGS_SCHEMA,
@@ -22,8 +22,12 @@ from products.signals.dags.inbox_ranking.dataset.dag import (
USER_INTERACTIONS_SCHEMA,
USER_INTERACTIONS_TABLE,
)
-from products.signals.dags.inbox_ranking.dataset.project_vectors import USER_VECTOR_KINDS
+from products.signals.dags.inbox_ranking.dataset.project_vectors import USER_VECTOR_KINDS, VECTOR_KINDS
+from products.signals.dags.inbox_ranking.multihead.inputs import ProjectContext
+from products.signals.dags.inbox_ranking.multihead.local import report_vectors
+from products.signals.dags.inbox_ranking.nn import pair_lane
from products.signals.dags.inbox_ranking.nn.dag import NN_EXAMPLES_TABLE
+from products.signals.dags.inbox_ranking.nn.pair_lane import load_user_history, pair_example_table, pair_network_inputs
from products.signals.dags.inbox_ranking.nn.pairs import (
IMPRESSIONS_SOURCE,
NN_PAIR_EXAMPLES_TABLE,
@@ -34,6 +38,11 @@ from products.signals.dags.inbox_ranking.nn.pairs import (
pair_user_vectors,
)
from products.signals.dags.inbox_ranking.tests.test_nn import _S3, _parquet
+from products.signals.dags.inbox_ranking.training.families import (
+ MODEL_FAMILIES,
+ NN_PAIR_BASE_MODEL_NAME,
+ NN_PAIR_USER_MODEL_NAME,
+)
PREFIX = settings.INBOX_RANKING_DATASET_S3_PREFIX
BIRTH = datetime.date(2026, 8, 1)
@@ -41,6 +50,7 @@ BIRTH_START = datetime.datetime(2026, 8, 1, tzinfo=datetime.UTC)
LABEL_DAY = BIRTH + datetime.timedelta(days=7)
ORGANIZATIONS = {2: "org-a", 9: "org-b"}
TARGET, PAST = "report-target", "report-past"
+FAMILIES = {family.name: family for family in MODEL_FAMILIES}
def _at(days: float) -> datetime.datetime:
@@ -197,3 +207,108 @@ def test_a_partition_rerun_overwrites_only_its_own_object_and_drops_teams_withou
assert set(table.column("user_distinct_id").to_pylist()) == {"u1", "u2", "u5"}
assert metadata["excluded_no_training_consent_reports"].value == 1
assert (metadata["rows"].value, metadata["history_0_rows"].value, metadata["history_1_9_rows"].value) == (3, 2, 1)
+
+
+def _pair_embeddings() -> pa.Table:
+ extra = [
+ # (report, team, value): a report u2 dismissed, one u1 viewed in another organization, and one
+ # u1 viewed in a team of its organization that has no training consent.
+ ("report-dismissed", 2, 5.0),
+ ("report-elsewhere", 9, 7.0),
+ ("report-unconsented", 5, 11.0),
+ ]
+ rows = [
+ {
+ "snapshot_date": LABEL_DAY,
+ "report_id": report_id,
+ "report_team_id": team_id,
+ "embedding_small": [value] * EMBEDDING_DIMENSIONS,
+ "embedding_inserted_at": _at(-6),
+ "embedding_rendering": "title_summary_v1",
+ "is_tombstone": False,
+ }
+ for report_id, team_id, value in extra
+ ]
+ return pa.concat_tables([_embeddings(), pa.Table.from_pylist(rows, schema=EMBEDDINGS_SCHEMA)])
+
+
+def test_each_pair_arm_builds_its_own_inputs_and_only_the_user_arm_reads_history(monkeypatch) -> None:
+ monkeypatch.setattr(pair_lane, "training_consent_team_ids", lambda: frozenset({2, 9}))
+ monkeypatch.setattr(
+ pair_lane,
+ "team_organizations",
+ lambda team_ids: {t: o for t, o in {**ORGANIZATIONS, 5: "org-a"}.items() if t in team_ids},
+ )
+ partition = LABEL_DAY.isoformat()
+ history_rows = pa.concat_tables(
+ [
+ _interactions(
+ [
+ *HISTORY_ROWS,
+ {"report_id": "report-dismissed", "user_distinct_id": "u2", "first_ui_dismissed_at": _at(-1)},
+ ]
+ ),
+ _interactions([{"report_id": "report-elsewhere", "user_distinct_id": "u1", "first_viewed_at": _at(-1)}], 9),
+ _interactions(
+ [{"report_id": "report-unconsented", "user_distinct_id": "u1", "first_viewed_at": _at(-1)}], 5
+ ),
+ ]
+ )
+ embeddings = _pair_embeddings()
+ client = _S3({partition_object_key(PREFIX, USER_INTERACTIONS_TABLE, partition): _parquet(history_rows)})
+ history = load_user_history(client, "bucket", PREFIX, partition)
+ assert history is not None
+ pairs = pd.DataFrame(
+ pair_rows(
+ REPORTS,
+ _interactions(IMPRESSION_ROWS),
+ history.interactions,
+ embeddings,
+ birth_day=BIRTH,
+ reviewers=None,
+ organizations=history.organizations,
+ )
+ )
+ report_rows = pd.DataFrame(
+ {"report_id": [TARGET], "snapshot_date": [BIRTH], "report_team_id": [2], "priority": ["P1"]}
+ )
+ table = pair_example_table(pairs, report_rows, ("open", "action"))
+ # The team has an `open` history of all ones and too thin a `dismiss` history.
+ project_vectors = np.zeros((1, len(VECTOR_KINDS), EMBEDDING_DIMENSIONS), dtype=np.float32)
+ project_vectors[0, VECTOR_KINDS.index("open")] = 1.0
+ project_has_history = np.zeros((1, len(VECTOR_KINDS)), dtype=np.float32)
+ project_has_history[0, VECTOR_KINDS.index("open")] = 1.0
+ project = ProjectContext(
+ vectors=project_vectors, has_history=project_has_history, index=np.zeros(len(table.rows), dtype=np.int64)
+ )
+ arms = {
+ name: pair_network_inputs(
+ table, report_vectors(embeddings), FAMILIES[name].feature_set, project, history, embeddings
+ )
+ for name in (NN_PAIR_BASE_MODEL_NAME, NN_PAIR_USER_MODEL_NAME)
+ }
+ base, user = arms[NN_PAIR_BASE_MODEL_NAME], arms[NN_PAIR_USER_MODEL_NAME]
+
+ assert base.reports.shape == user.reports.shape
+ width = base.reports.shape[1] - len(PAIR_FEATURE_NAMES)
+ np.testing.assert_array_equal(base.reports[:, :width], user.reports[:, :width])
+ reviewer = width + PAIR_FEATURE_NAMES.index("is_suggested_reviewer")
+ np.testing.assert_array_equal(base.reports[:, reviewer], user.reports[:, reviewer])
+ row = {name: int(table.rows.index[table.rows["user_distinct_id"] == name][0]) for name in ("u1", "u2", "u5")}
+ assert base.reports[row["u2"], reviewer] == 1.0
+ assert not base.reports[:, width:reviewer].any()
+ assert base.users is not None and not base.users.vectors[base.users.index].any()
+ np.testing.assert_allclose(user.reports[row["u1"], width:reviewer], [np.log1p(1), 0.0, 2 / 90, 1.0])
+
+ assert user.users is not None
+ offsets = user.users.vectors[user.users.index]
+ open_kind, dismiss_kind = USER_VECTOR_KINDS.index("open"), USER_VECTOR_KINDS.index("dismiss")
+ # u1's open vector is the one past report it viewed before the birth day in this organization's
+ # consented teams: the target report, the other organization's and the unconsented team's stay out.
+ np.testing.assert_allclose(offsets[row["u1"], open_kind], 3.0 - 1.0)
+ # u2 viewed nothing, and the team's dismiss history is too thin to offset its dismissals from.
+ assert not offsets[row["u1"], dismiss_kind].any()
+ assert not offsets[row["u2"]].any()
+ assert not offsets[row["u5"]].any()
+ # Without history the user reads as the longest time since a view, and nothing else.
+ np.testing.assert_allclose(user.reports[row["u5"], width:reviewer], [0.0, 0.0, 1.0, 0.0])
diff --git a/products/signals/dags/inbox_ranking/training/families.py b/products/signals/dags/inbox_ranking/training/families.py
index c2db886a916d..79c8bc540876 100644
--- a/products/signals/dags/inbox_ranking/training/families.py
+++ b/products/signals/dags/inbox_ranking/training/families.py
@@ -21,6 +21,8 @@ from typing import Final, Literal
from posthog.dataclasses import frozen
from products.signals.backend.ranking.features import (
+ PAIR_BASE_FEATURE_SET,
+ PAIR_USER_HISTORY_FEATURE_SET,
REPORT_EMBEDDINGS_FEATURE_SET,
REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
@@ -35,6 +37,9 @@ from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND
NN_MODEL_NAME = "nn_v3"
NN_WITH_PRIO_MODEL_NAME = "nn_with_prio_v2"
NN_WITH_PRIO_SOURCES_MODEL_NAME = "nn_with_prio_sources_v1"
+NN_PAIR_BASE_MODEL_NAME = "nn_pair_base_v1"
+NN_PAIR_USER_MODEL_NAME = "nn_pair_user_v1"
+PAIR_EXPERIMENT_EXPIRES = datetime.date(2026, 11, 25)
# The Dagster code locations that train a family: `signals` on the main image, and `signals_ml` on
# the image that carries torch.
MAIN_LOCATION = "signals"
@@ -109,6 +114,10 @@ class ModelFamily:
# `nn_with_prio_v2` is the same network with the report's priority levels as extra report inputs.
# `nn_with_prio_sources_v1` is an experiment: `nn_with_prio_v2` plus the source products of the
# report's signals as of the moment, graded against `nn_with_prio_v2`.
+# `nn_pair_base_v1` and `nn_pair_user_v1` are the user-history comparison at pair grain: one network,
+# one input shape, the same pairs and seeds, and the same reviewer flag. The base holds every
+# user-history input at zero, so `nn_pair_user_v1` against it is the uplift of the user's history
+# alone. Both are also read against the report-level `nn_with_prio_v2`.
MODEL_FAMILIES: tuple[ModelFamily, ...] = (
ModelFamily(
name=NN_MODEL_NAME, feature_set=REPORT_EMBEDDINGS_FEATURE_SET, learner=ONNX_MODEL_KIND, location=ML_LOCATION
@@ -128,6 +137,29 @@ MODEL_FAMILIES: tuple[ModelFamily, ...] = (
baseline=NN_WITH_PRIO_MODEL_NAME,
expires=datetime.date(2026, 11, 11),
),
+ ModelFamily(
+ name=NN_PAIR_BASE_MODEL_NAME,
+ feature_set=PAIR_BASE_FEATURE_SET,
+ learner=ONNX_MODEL_KIND,
+ location=ML_LOCATION,
+ experiment=True,
+ expires=PAIR_EXPERIMENT_EXPIRES,
+ grain=PAIR_GRAIN,
+ heads=PAIR_HEAD_NAMES,
+ reference=NN_WITH_PRIO_MODEL_NAME,
+ ),
+ ModelFamily(
+ name=NN_PAIR_USER_MODEL_NAME,
+ feature_set=PAIR_USER_HISTORY_FEATURE_SET,
+ learner=ONNX_MODEL_KIND,
+ location=ML_LOCATION,
+ experiment=True,
+ baseline=NN_PAIR_BASE_MODEL_NAME,
+ expires=PAIR_EXPERIMENT_EXPIRES,
+ grain=PAIR_GRAIN,
+ heads=PAIR_HEAD_NAMES,
+ reference=NN_WITH_PRIO_MODEL_NAME,
+ ),
)
NN_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_MODEL_NAME)
NN_WITH_PRIO_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_WITH_PRIO_MODEL_NAME)