Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): add fixed project-vector experiments to inbox ranking (#126021)

posthog[bot]93e459d2c904
diff --git a/products/signals/backend/ranking/features.py b/products/signals/backend/ranking/features.py
index 587bb0502f5d..f0aafb7a8da1 100644
--- a/products/signals/backend/ranking/features.py
+++ b/products/signals/backend/ranking/features.py
@@ -87,6 +87,9 @@ SOURCE_FEATURE_NAMES = (*(f"source_{product}" for product in SOURCE_PRODUCTS), "
# dag's `inbox_project_vectors` writes one row per (team, kind), and the training and serving
# inputs both read these names, so the two sides cannot order the kinds differently.
PROJECT_VECTOR_KINDS = ("typical", "open", "action", "dismiss")
+# Every kind `project_vectors()` can build. An experiment family can read a kind the stored table
+# and the serving path do not carry, such as `fixed`, which comes from the status stream.
+TRAINING_PROJECT_VECTOR_KINDS = (*PROJECT_VECTOR_KINDS, "fixed")
# Below this many reports, a kind's mean is mostly one report's text, so the row keeps an all-zero
# vector and the kind has no history.
MIN_PROJECT_HISTORY_REPORTS = 5
@@ -445,23 +448,26 @@ FEATURE_SETS: Mapping[str, FeatureSet] = MappingProxyType(
class ProjectInputs:
"""The project vectors of some teams, one row per team in the order the caller asked for."""
- # (teams, kinds, EMBEDDING_DIMENSIONS) float32, in `PROJECT_VECTOR_KINDS` order. All zero where
+ # (teams, kinds, EMBEDDING_DIMENSIONS) float32, in the caller's kind order. All zero where
# the team has no row for the kind or its history is too thin.
vectors: np.ndarray
# (teams, kinds) float32, 1.0 where the kind had at least `MIN_PROJECT_HISTORY_REPORTS` reports.
has_history: np.ndarray
-def project_inputs(project_vectors: pd.DataFrame, team_ids: Sequence[int]) -> ProjectInputs:
- """The project vectors of `team_ids` from one day of `inbox_project_vectors`.
+def project_inputs(
+ project_vectors: pd.DataFrame, team_ids: Sequence[int], kinds: Sequence[str] = PROJECT_VECTOR_KINDS
+) -> ProjectInputs:
+ """The project vectors of `team_ids` from one day of `inbox_project_vectors`, in `kinds` order.
`project_vectors` carries the `team_id`, `kind`, `vector` and `n_reports` columns of that table.
Training and serving both build the network's `project` and `has_history` inputs here, so a
team without a row, a kind without a row and a thin kind read the same on both sides.
"""
position = {int(team_id): index for index, team_id in enumerate(team_ids)}
- vectors = np.zeros((len(team_ids), len(PROJECT_VECTOR_KINDS), EMBEDDING_DIMENSIONS), dtype=np.float32)
- has_history = np.zeros((len(team_ids), len(PROJECT_VECTOR_KINDS)), dtype=np.float32)
+ kind_position = {kind: index for index, kind in enumerate(kinds)}
+ vectors = np.zeros((len(team_ids), len(kinds), EMBEDDING_DIMENSIONS), dtype=np.float32)
+ has_history = np.zeros((len(team_ids), len(kinds)), dtype=np.float32)
for team_id, kind, vector, n_reports in zip(
project_vectors["team_id"],
project_vectors["kind"],
@@ -470,9 +476,9 @@ def project_inputs(project_vectors: pd.DataFrame, team_ids: Sequence[int]) -> Pr
strict=True,
):
team = position.get(int(team_id))
- if team is None or kind not in PROJECT_VECTOR_KINDS:
+ kind_index = kind_position.get(kind)
+ if team is None or kind_index is None:
continue
- kind_index = PROJECT_VECTOR_KINDS.index(kind)
vectors[team, kind_index] = vector
has_history[team, kind_index] = float(int(n_reports) >= MIN_PROJECT_HISTORY_REPORTS)
return ProjectInputs(vectors=vectors, has_history=has_history)
diff --git a/products/signals/dags/inbox_ranking/AGENTS.md b/products/signals/dags/inbox_ranking/AGENTS.md
index cacbf478c005..111a8ac77439 100644
--- a/products/signals/dags/inbox_ranking/AGENTS.md
+++ b/products/signals/dags/inbox_ranking/AGENTS.md
@@ -7,7 +7,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- A new ranking dag (training, eval, ...) is a **sibling subpackage** (`inbox_ranking/<name>/dag.py`) importing shared plumbing from `common.py` — don't grow `dataset/` sideways and don't duplicate the S3/partition helpers.
- `inbox_report_title_embeddings` is a **leaf**: never add it to `inbox_report_model_data`'s `deps` or give that join a title column, or a failure in the title snapshot fails the training table. Its own `deps=[MODEL_DATA_TABLE]` exists for memory, not data — two full sets of 1536-float vectors must not be held at once in the run pod. Raising the pod memory limit is not the first resort.
- `inbox_user_report_interactions` is a **leaf** with its own `USER_INTERACTIONS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`, and never stamp it with `FEATURE_SCHEMA_VERSION`, or a user-grain change sets off the labels refresh sensor. It writes a row only for a report in this region's Postgres whose team is in `training_consent_team_ids()`.
-- `inbox_project_vectors` is a **leaf** with its own `PROJECT_VECTORS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`. Its `deps=[EMBEDDINGS_TABLE, LABELS_TABLE]` keep it after the embeddings snapshot, so the two vector-heavy steps do not share the run pod's memory. `project_vectors` takes a report into a vector only when its vector landed and its UI engagement happened strictly before `as_of`; never widen that to the latest snapshot value. A report born on day D reads the dt=D−1 vectors (see the README).
+- `inbox_project_vectors` is a **leaf** with its own `PROJECT_VECTORS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`. Its `deps=[EMBEDDINGS_TABLE, LABELS_TABLE]` keep it after the embeddings snapshot, so the two vector-heavy steps do not share the run pod's memory. `project_vectors` takes a report into a vector only when its vector landed and its engagement moment (a UI event, or `first_fixed_at` for `fixed`) happened strictly before `as_of`; never widen that to the latest snapshot value. A report born on day D reads the dt=D−1 vectors (see the README). Keep `first_fixed_at` out of `LABEL_COLUMNS`, because the engagement export carries exactly those columns; training readers load `CONTEXT_LABEL_COLUMNS`.
- `inbox_user_vectors` is a **leaf** with its own `USER_VECTORS_SCHEMA_VERSION`: keep it out of `inbox_report_model_data`'s `deps`. Its `deps=[PROJECT_VECTORS_TABLE, USER_INTERACTIONS_TABLE]` keep it after the project vectors, so two reads of the embeddings snapshot do not share the run pod's memory. It builds its rows through `user_vectors()` with the same consent re-filter and `team_organizations` mapping as the pair lane, so its dt=D rows equal what `pair_user_vectors` rebuilds for a report born on D+1; a test pins that parity. `user_hash` covers each kind's name, `n_reports` and vector only, so a day that adds no report keeps a user's hash.
- `inbox_engagement_export` is a **leaf** with its own `ENGAGEMENT_EXPORT_SCHEMA_VERSION`, read by another region. It writes to the mirror store only under `serving_engagement_key`, because the mirror's write grant covers only `<prefix>/serving/`. Keep its columns exactly `LABEL_COLUMNS` and its rows limited to reports with no `SignalReport` in this region: no team ids, counts, text or vectors leave the region. A mirror failure logs and must not fail the asset. Keep the local-share guard before the write: when the local report lookup finds no local report (`no_local_reports`) or a share below `ENGAGEMENT_EXPORT_MIN_LOCAL_SHARE` (`local_share_collapsed`), the asset writes nothing and still succeeds. Its `inbox_ranking_engagement_export_published` event carries counts only, never a report id or a team id.
- `inbox_mirror_project_vectors` (`mirror/dag.py`) is the only inbox-ranking asset that registers on EU (`is_inbox_ranking_mirror_registered()`). It reads this region's report vectors through `report_embeddings_table` and the engagement export through `serving_engagement_key`, and writes only to this region's object store at `serving_project_vectors_key`. Keep it on the same `project_vectors()` and the same consent rule as `inbox_project_vectors`, so the two regions build their vectors the same way. A missing, too old or unknown-schema export fails the asset rather than writing vectors without engagement.
@@ -41,7 +41,7 @@ Read `README.md` first for what the dataset is and how partitions behave. This f
- The `recent` role keeps the newest `INBOX_RANKING_SERVING_RECENT_VERSIONS` versions of the served `model_name`, read from the last 2N partitions, so each version gets several days of online scores. The `challenger` role keeps the newest readable version of each other serving family (`serving_families()`), read from its last 2 partitions. Each one passes the pin check before composition, because one missing file fails the publish. Keep the order under the cap as served, recent, challenger, pinned, cross-family. Each recent entry is one more model per sweep pass, so measure the sweep's duration, peak RSS and scored and deferred counts on the `inbox_ranking_sweep_finished` log before and after a change to N.
- Publishing order is load-bearing. Every model copy finishes before `manifest.json` is written, so a failed copy leaves the previous manifest and the previous models serving. A model version is immutable, so a version whose `metadata.json` is already at the target is not copied again. A run with nothing to publish, such as a run without `served.json`, writes nothing rather than an empty manifest. The served model passes the same check as a pin (`_pin_metadata`) before composition.
- The manifest asset copies the dt=D `inbox_user_vectors` day to `serving/user_vectors/<D>/` before `manifest.json` and names it in `user_vectors`. No model reads it yet, so a missing day or a failed copy only logs and leaves the key out; it never blocks a publish. Never copy user vectors to the mirror: the mirror's manifest always names none.
-- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror, and never read the body of a mirror object: they are per-team data of the deployment that owns the teams. The mirror's manifest names the mirror region's own day only after `_mirror_project_vectors` checks its object metadata (`PROJECT_VECTORS_*_METADATA_KEY` in `serving_manifest.py`) within `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS`. Without such a day it names none, and a served model that reads them keeps the mirror on its previous manifest. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there.
+- When a manifest entry has `reads_project_vectors`, the dt=D `inbox_project_vectors` day is copied to `serving/project_vectors/<D>/` before `manifest.json`, and the manifest names it in `project_vectors`. A missing day fails the publish, like a missing model file. Never copy project vectors to the mirror, and never read the body of a mirror object: they are per-team data of the deployment that owns the teams. The mirror's manifest names the mirror region's own day only after `_mirror_project_vectors` checks its object metadata (`PROJECT_VECTORS_*_METADATA_KEY` in `serving_manifest.py`) within `MIRROR_PROJECT_VECTORS_MAX_LAG_DAYS`. Without such a day it names none, and a served model that reads them keeps the mirror on its previous manifest. Training and serving build the `project` and `has_history` inputs through the one `project_inputs` in `features.py`; keep `PROJECT_VECTOR_KINDS` and `MIN_PROJECT_HISTORY_REPORTS` there. Only a report-grain experiment can set its family's `project_vector_kinds` to other kinds from `TRAINING_PROJECT_VECTOR_KINDS`; the stored table, serving and the pair lane stay on `PROJECT_VECTOR_KINDS`.
- `HEADS` holds only the heads the inbox reads (`HeadRanker` in `backend/ranking/inbox_sort.py`). Add a head there only together with a surface that reads it. The promotion gate skips a champion head that is not in `HEADS`, so a trim does not block promotion.
- `nn_v3` and `nn_with_prio_v2` are trained by the `signals_ml` location (`nn/dag.py`), never by the main one. They share one `inbox_ranking_nn_examples` object, and each has its own candidate and champion asset from `family_assets`. The `nn_v3` assets keep the names `inbox_ranking_nn_model_{candidate,champion}`, so their Dagster history continues. A later family's candidate depends on the previous family's champion, for memory and not for data: two candidates at once can exceed the job's memory limit. A family whose set reads a state column the examples object lacks skips with a warning rather than training on zeros. torch is imported only inside the candidate asset's body (through `nn/train.py` and `multihead/network.py`). `signals_ml` must import without torch, and `test_multihead_network.py` pins that. Its decision goes through `decide_family_champion` in `training/dag.py`, graded through the incumbent's `model.holdout.onnx`. Keep `signals_ml` commented out in `.dagster_home/workspace.yaml`, and keep its schedule (04:30 UTC) running by default on prod US only.
diff --git a/products/signals/dags/inbox_ranking/dataset/project_vectors.py b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
index 437033bc40af..f6728f25f2a4 100644
--- a/products/signals/dags/inbox_ranking/dataset/project_vectors.py
+++ b/products/signals/dags/inbox_ranking/dataset/project_vectors.py
@@ -8,10 +8,13 @@ gets one vector per kind, as of a moment:
open the mean of the last `HISTORY_LIMIT` reports anyone on the team opened
action the same, over the reports anyone acted on through a UI intent action
dismiss the same, over the reports anyone dismissed from the UI
+ fixed the same, over the reports the team got fixed (`first_fixed_at`)
-Only human UI engagement counts: the open, action and dismiss moments come from the inbox app
-events, so MCP reads and agent actions never reach a vector. A project is a team, not an
-organization.
+The open, action and dismiss moments come from the inbox app events, so only human UI engagement
+counts there, and MCP reads and agent actions never reach those vectors. The fixed moment comes from
+the status stream, so a fix through an agent or a merged PR counts. The stored table and serving
+carry the first four kinds only. `fixed` is built only for a family that asks for it in training,
+from the labels snapshot's `first_fixed_at`. A project is a team, not an organization.
`user_vectors` builds the same kind of vector for one person in one organization, from the
`inbox_user_report_interactions` rows of the reports that person viewed or dismissed:
@@ -42,6 +45,7 @@ from products.signals.backend.ranking.features import (
EMBEDDING_DIMENSIONS,
MIN_PROJECT_HISTORY_REPORTS,
PROJECT_VECTOR_KINDS,
+ TRAINING_PROJECT_VECTOR_KINDS,
USER_HISTORY_KINDS,
)
from products.signals.dags.inbox_ranking.dataset.queries import ACTION_TYPES
@@ -54,12 +58,16 @@ HISTORY_LIMIT = 50
# The backend owns the kinds and the threshold, because the scoring sweep builds the same inputs.
MIN_HISTORY_REPORTS = MIN_PROJECT_HISTORY_REPORTS
VECTOR_KINDS = PROJECT_VECTOR_KINDS
-TYPICAL, OPEN, ACTION, DISMISS = VECTOR_KINDS
+CONTEXT_KINDS = TRAINING_PROJECT_VECTOR_KINDS
+TYPICAL, OPEN, ACTION, DISMISS, FIXED = CONTEXT_KINDS
OPEN_COLUMN = "first_opened_at"
DISMISS_COLUMN = "first_ui_dismissed_at"
+FIXED_COLUMN = "first_fixed_at"
UI_ACTION_FIRST_COLUMNS = tuple(first for _, count, first in ACTION_TYPES if count in UI_ACTION_COLUMNS)
+# The engagement export carries exactly these columns to the other region, so `fixed` stays out.
LABEL_COLUMNS = ("report_id", OPEN_COLUMN, DISMISS_COLUMN, *UI_ACTION_FIRST_COLUMNS)
+CONTEXT_LABEL_COLUMNS = (*LABEL_COLUMNS, FIXED_COLUMN)
EMBEDDING_COLUMNS = ("report_id", "report_team_id", "embedding_small", "embedding_inserted_at", "is_tombstone")
_PROJECT_VECTORS_FIELDS: list[tuple[str, pa.DataType]] = [
@@ -112,8 +120,10 @@ HISTORY_BUCKETS: tuple[tuple[str, int, int | None], ...] = (
)
-def project_vectors(labels: pa.Table, embeddings: pa.Table, as_of: datetime.datetime) -> pa.Table:
- """One row per (team, kind) for every team with a report vector before `as_of`.
+def project_vectors(
+ labels: pa.Table, embeddings: pa.Table, as_of: datetime.datetime, kinds: Sequence[str] = VECTOR_KINDS
+) -> pa.Table:
+ """One row per (team, kind of `kinds`) for every team with a report vector before `as_of`.
`labels` is an `inbox_report_labels` snapshot and `embeddings` an `inbox_report_embeddings`
snapshot, both at or after `as_of`. A later snapshot rebuilds an earlier day: a report enters a
@@ -122,8 +132,12 @@ def project_vectors(labels: pa.Table, embeddings: pa.Table, as_of: datetime.date
out rather than read with text it did not have yet.
A kind with fewer than `MIN_HISTORY_REPORTS` reports keeps its row, with its real `n_reports`
- and an all-zero vector, so a reader can tell a thin history from a missing team.
+ and an all-zero vector, so a reader can tell a thin history from a missing team. `kinds` defaults
+ to the stored table's kinds. Asking for `fixed` needs the `first_fixed_at` column in `labels`.
"""
+ unknown = [kind for kind in kinds if kind not in CONTEXT_KINDS]
+ if unknown:
+ raise ValueError(f"unknown project vector kinds {unknown}; pick from {list(CONTEXT_KINDS)}")
usable = _usable_embeddings(embeddings, as_of)
matrix = _embedding_matrix(usable)
report_ids = [str(report_id) for report_id in usable.column("report_id").to_pylist()]
@@ -131,13 +145,15 @@ def project_vectors(labels: pa.Table, embeddings: pa.Table, as_of: datetime.date
for index, team_id in enumerate(usable.column("report_team_id").to_pylist()):
if team_id is not None:
rows_by_team[team_id].append(index)
- engaged_at = _engaged_at(labels)
+ engaged_at = _engaged_at(labels, with_fixed=FIXED in kinds)
rows: list[dict[str, object]] = []
for team_id in sorted(rows_by_team):
team_rows = rows_by_team[team_id]
- rows.append(_vector_row(team_id, TYPICAL, matrix, team_rows, as_of))
- for kind in (OPEN, ACTION, DISMISS):
+ for kind in kinds:
+ if kind == TYPICAL:
+ rows.append(_vector_row(team_id, TYPICAL, matrix, team_rows, as_of))
+ continue
moments = engaged_at[kind]
engaged = [
(moment, index) for index in team_rows if known_before(moment := moments.get(report_ids[index]), as_of)
@@ -257,12 +273,12 @@ def known_before(moment: datetime.datetime | None, as_of: datetime.datetime) ->
return moment is not None and moment < as_of
-def teams_with_history(table: pa.Table) -> dict[str, int]:
+def teams_with_history(table: pa.Table, kinds: Sequence[str] = VECTOR_KINDS) -> dict[str, int]:
"""How many teams of a `project_vectors` table have enough history for a real vector, per kind."""
counts = table.select(["kind", "n_reports"]).to_pylist()
return {
kind: sum(1 for row in counts if row["kind"] == kind and row["n_reports"] >= MIN_HISTORY_REPORTS)
- for kind in VECTOR_KINDS
+ for kind in kinds
}
@@ -304,15 +320,18 @@ def _latest(engaged: list[tuple[datetime.datetime, int]], limit: int) -> list[tu
return sorted(engaged, reverse=True)[:limit]
-def _engaged_at(labels: pa.Table) -> dict[str, dict[str, datetime.datetime]]:
- """The first UI engagement moment of each kind per report."""
- engaged_at: dict[str, dict[str, datetime.datetime]] = {OPEN: {}, ACTION: {}, DISMISS: {}}
- for row in labels.select(list(LABEL_COLUMNS)).to_pylist():
+def _engaged_at(labels: pa.Table, *, with_fixed: bool) -> dict[str, dict[str, datetime.datetime]]:
+ """The first engagement moment of each kind per report: the UI kinds, and `fixed` when asked."""
+ engaged_at: dict[str, dict[str, datetime.datetime]] = {OPEN: {}, ACTION: {}, DISMISS: {}, FIXED: {}}
+ columns = CONTEXT_LABEL_COLUMNS if with_fixed else LABEL_COLUMNS
+ for row in labels.select(list(columns)).to_pylist():
report_id = row["report_id"]
if row[OPEN_COLUMN] is not None:
engaged_at[OPEN][report_id] = row[OPEN_COLUMN]
if row[DISMISS_COLUMN] is not None:
engaged_at[DISMISS][report_id] = row[DISMISS_COLUMN]
+ if with_fixed and row[FIXED_COLUMN] is not None:
+ engaged_at[FIXED][report_id] = row[FIXED_COLUMN]
actions = [row[column] for column in UI_ACTION_FIRST_COLUMNS if row[column] is not None]
if actions:
engaged_at[ACTION][report_id] = min(actions)
diff --git a/products/signals/dags/inbox_ranking/multihead/inputs.py b/products/signals/dags/inbox_ranking/multihead/inputs.py
index a1b981657f6a..85c81feb4c7e 100644
--- a/products/signals/dags/inbox_ranking/multihead/inputs.py
+++ b/products/signals/dags/inbox_ranking/multihead/inputs.py
@@ -5,6 +5,8 @@ import numpy as np
from posthog.dataclasses import frozen
+from products.signals.backend.ranking.features import PROJECT_VECTOR_KINDS
+
# The exported graph's names. A network without context takes only the first input, and a network
# without user history only the first three.
ONNX_INPUTS = ("report", "project", "has_history", "user")
@@ -33,6 +35,11 @@ class ProjectContext:
has_history: np.ndarray
# (rows,) int, the pair each example row reads.
index: np.ndarray
+ kinds: tuple[str, ...] = PROJECT_VECTOR_KINDS
+
+ def __post_init__(self) -> None:
+ if self.has_history.shape[1:] != (len(self.kinds),):
+ raise ValueError(f"project context has {self.has_history.shape[1:]} history columns for kinds {self.kinds}")
@frozen
diff --git a/products/signals/dags/inbox_ranking/multihead/prototype.py b/products/signals/dags/inbox_ranking/multihead/prototype.py
index 09845092ae37..44cf2908fc15 100644
--- a/products/signals/dags/inbox_ranking/multihead/prototype.py
+++ b/products/signals/dags/inbox_ranking/multihead/prototype.py
@@ -150,31 +150,32 @@ def report_matrix(rows: pd.DataFrame, extras: Extras, feature_set: FeatureSet =
return matrix
-def project_context(rows: pd.DataFrame, labels: pa.Table, embeddings: pa.Table) -> ProjectContext:
- """The project vectors each row reads: its team's, as of the start of its snapshot day.
+def project_context(
+ rows: pd.DataFrame, labels: pa.Table, embeddings: pa.Table, kinds: Sequence[str] = VECTOR_KINDS
+) -> ProjectContext:
+ """The project vectors of `kinds` each row reads: its team's, as of the start of its snapshot day.
One `project_vectors()` call per snapshot day, and only the needed teams' rows are kept from it,
so the context holds a vector per (team, day) rather than one per example row. `project_inputs`
turns a day into vectors, the same function the scoring sweep reads the published day with.
"""
- kinds = len(VECTOR_KINDS)
pair_keys = rows[["report_team_id", "snapshot_date"]].drop_duplicates().reset_index(drop=True)
- vectors = np.zeros((len(pair_keys), kinds, EMBEDDING_DIMENSIONS), dtype=np.float32)
- has_history = np.zeros((len(pair_keys), kinds), dtype=np.float32)
+ vectors = np.zeros((len(pair_keys), len(kinds), EMBEDDING_DIMENSIONS), dtype=np.float32)
+ has_history = np.zeros((len(pair_keys), len(kinds)), dtype=np.float32)
pair_of: dict[tuple[int, datetime.date], int] = {}
for date, group in pair_keys.groupby("snapshot_date", sort=True):
as_of, _ = snapshot_bounds(date.isoformat())
pairs = group.index.to_numpy()
team_ids = [int(team_id) for team_id in group["report_team_id"]]
pair_of.update({(team_id, date): int(pair) for team_id, pair in zip(team_ids, pairs, strict=True)})
- day = project_inputs(project_vectors(labels, embeddings, as_of).to_pandas(), team_ids)
+ day = project_inputs(project_vectors(labels, embeddings, as_of, kinds).to_pandas(), team_ids, kinds)
vectors[pairs] = day.vectors
has_history[pairs] = day.has_history
index = np.array(
[pair_of[(int(team_id), date)] for team_id, date in zip(rows["report_team_id"], rows["snapshot_date"])],
dtype=np.int64,
)
- return ProjectContext(vectors=vectors, has_history=has_history, index=index)
+ return ProjectContext(vectors=vectors, has_history=has_history, index=index, kinds=tuple(kinds))
@frozen
diff --git a/products/signals/dags/inbox_ranking/nn/dag.py b/products/signals/dags/inbox_ranking/nn/dag.py
index 50ce7a70dec2..50bc7a05dce0 100644
--- a/products/signals/dags/inbox_ranking/nn/dag.py
+++ b/products/signals/dags/inbox_ranking/nn/dag.py
@@ -48,10 +48,10 @@ from products.signals.dags.inbox_ranking.common import (
)
from products.signals.dags.inbox_ranking.consent import training_consent_team_ids
from products.signals.dags.inbox_ranking.dataset.dag import EMBEDDINGS_TABLE, LABELS_TABLE, STATE_TABLE
-from products.signals.dags.inbox_ranking.dataset.project_vectors import EMBEDDING_COLUMNS, LABEL_COLUMNS
-from products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfig, NetworkInputs
+from products.signals.dags.inbox_ranking.dataset.project_vectors import CONTEXT_LABEL_COLUMNS, EMBEDDING_COLUMNS
+from products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfig, NetworkInputs, ProjectContext
from products.signals.dags.inbox_ranking.multihead.local import report_vectors
-from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable, report_matrix
+from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable, project_context, report_matrix
from products.signals.dags.inbox_ranking.nn.examples import (
NN_HEADS,
build_examples,
@@ -131,7 +131,10 @@ def _day_inputs(
client, bucket, partition_object_key(prefix, EMBEDDINGS_TABLE, partition_key), columns=list(EMBEDDING_COLUMNS)
)
labels = read_parquet_if_exists(
- client, bucket, partition_object_key(prefix, LABELS_TABLE, partition_key), columns=list(LABEL_COLUMNS)
+ client,
+ bucket,
+ partition_object_key(prefix, LABELS_TABLE, partition_key),
+ columns=list(CONTEXT_LABEL_COLUMNS),
)
if embeddings is None or labels is None:
missing = [name for name, table in ((EMBEDDINGS_TABLE, embeddings), (LABELS_TABLE, labels)) if table is None]
@@ -205,6 +208,8 @@ class _ExperimentInputs:
extras: Extras
# The baseline family's report input for every examples row, graded against the experiment's.
baseline_reports: np.ndarray
+ # None when the baseline reads the experiment's kinds.
+ baseline_context: ProjectContext | None = None
def _experiment_inputs(
@@ -238,8 +243,13 @@ def _experiment_inputs(
"as it stands"
)
return None
+ baseline_context = None
+ if baseline.project_vector_kinds != family.project_vector_kinds:
+ baseline_context = project_context(table.rows, day.labels, day.embeddings, baseline.project_vector_kinds)
return _ExperimentInputs(
- extras=extras, baseline_reports=report_matrix(table.rows, day.extras, baseline.feature_set)
+ extras=extras,
+ baseline_reports=report_matrix(table.rows, day.extras, baseline.feature_set),
+ baseline_context=baseline_context,
)
@@ -267,13 +277,16 @@ def train_candidate(
if day is None:
return {"skipped": dagster.MetadataValue.bool(True)}
extras = day.extras
- baseline_reports = None
+ baseline_reports = baseline_context = None
if family.experiment:
experiment = _experiment_inputs(context, client, bucket, prefix, partition_key, family, examples.table, day)
if experiment is None:
return {"skipped": dagster.MetadataValue.bool(True)}
extras, baseline_reports = experiment.extras, experiment.baseline_reports
- inputs = network_inputs(examples.table, extras, day.labels, day.embeddings, family.feature_set)
+ baseline_context = experiment.baseline_context
+ inputs = network_inputs(
+ examples.table, extras, day.labels, day.embeddings, family.feature_set, family.project_vector_kinds
+ )
del day
check_recipe(
client,
@@ -292,7 +305,7 @@ def train_candidate(
trained = train_network(examples.table, inputs, NN_HEADS, NETWORK_CONFIG)
vs_baseline = None
if baseline_reports is not None and family.baseline is not None:
- baseline_inputs = NetworkInputs(reports=baseline_reports, context=inputs.context)
+ baseline_inputs = NetworkInputs(reports=baseline_reports, context=baseline_context or inputs.context)
vs_baseline = vs_baseline_block(
examples.table,
baseline=family.baseline,
@@ -300,9 +313,22 @@ def train_candidate(
experiment_scores=seed_scores(examples.table, inputs, NN_HEADS, NETWORK_CONFIG, EXPERIMENT_SEEDS),
baseline_scores=seed_scores(examples.table, baseline_inputs, NN_HEADS, NETWORK_CONFIG, EXPERIMENT_SEEDS),
)
- return write_candidate(
+ metadata = write_candidate(
context, client, bucket, partition_key, family, trained, windows=examples.windows, vs_baseline=vs_baseline
)
+ return {**metadata, **history_shares(inputs)}
+
+
+def history_shares(inputs: NetworkInputs) -> dict[str, dagster.MetadataValue]:
+ """The share of example rows whose team has history for each project-vector kind, so a kind that
+ is all zero for most rows shows in the run before anyone reads its grade."""
+ if inputs.context is None or len(inputs) == 0:
+ return {}
+ shares = inputs.context.has_history[inputs.context.index].mean(axis=0)
+ return {
+ f"has_{kind}_history_share": dagster.MetadataValue.float(float(share))
+ for kind, share in zip(inputs.context.kinds, shares, strict=True)
+ }
def write_candidate(
diff --git a/products/signals/dags/inbox_ranking/nn/examples.py b/products/signals/dags/inbox_ranking/nn/examples.py
index 346c1defb77d..c3e9e55aae7d 100644
--- a/products/signals/dags/inbox_ranking/nn/examples.py
+++ b/products/signals/dags/inbox_ranking/nn/examples.py
@@ -14,7 +14,7 @@ of those two snapshots.
import json
import datetime
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
import numpy as np
import pandas as pd
@@ -22,7 +22,7 @@ import pyarrow as pa
from posthog.dataclasses import frozen
-from products.signals.backend.ranking.features import PRIORITY_COLUMN, Extras, FeatureSet
+from products.signals.backend.ranking.features import PRIORITY_COLUMN, PROJECT_VECTOR_KINDS, Extras, FeatureSet
from products.signals.dags.inbox_ranking.multihead.inputs import NetworkInputs
from products.signals.dags.inbox_ranking.multihead.prototype import (
FEATURE_SET,
@@ -119,12 +119,17 @@ def examples_from_arrow(arrow: pa.Table) -> NNExamples:
def network_inputs(
- table: ExampleTable, extras: Extras, labels: pa.Table, embeddings: pa.Table, feature_set: FeatureSet = FEATURE_SET
+ table: ExampleTable,
+ extras: Extras,
+ labels: pa.Table,
+ embeddings: pa.Table,
+ feature_set: FeatureSet = FEATURE_SET,
+ project_vector_kinds: Sequence[str] = PROJECT_VECTOR_KINDS,
) -> NetworkInputs:
"""The report input of each row as of its moment, built by `feature_set`, and its team's project
- vectors as of the start of its snapshot day, the ones serving would read for a report born that
- day."""
+ vectors of `project_vector_kinds` as of the start of its snapshot day, the ones serving would read
+ for a report born that day."""
return NetworkInputs(
reports=report_matrix(table.rows, extras, feature_set),
- context=project_context(table.rows, labels, embeddings),
+ context=project_context(table.rows, labels, embeddings, project_vector_kinds),
)
diff --git a/products/signals/dags/inbox_ranking/nn/model.py b/products/signals/dags/inbox_ranking/nn/model.py
index dae792e24c1a..c11de3f643e7 100644
--- a/products/signals/dags/inbox_ranking/nn/model.py
+++ b/products/signals/dags/inbox_ranking/nn/model.py
@@ -24,7 +24,8 @@ from products.signals.backend.ranking.features import (
from products.signals.backend.ranking.model_store import ONNX_OUTPUT_HEADS_KEY
from products.signals.backend.ranking.serving_manifest import ONNX_MODEL_FILE, ONNX_MODEL_KIND
from products.signals.dags.inbox_ranking.common import DATASET_VERSION
-from products.signals.dags.inbox_ranking.dataset.project_vectors import USER_HISTORY_LIMIT, VECTOR_KINDS
+from products.signals.dags.inbox_ranking.dataset.project_vectors import USER_HISTORY_LIMIT
+from products.signals.dags.inbox_ranking.dataset.queries import LABELS_TEAM_ID
from products.signals.dags.inbox_ranking.multihead.inputs import ONNX_INPUTS, ONNX_OUTPUT, NetworkConfig, NetworkInputs
from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable
from products.signals.dags.inbox_ranking.nn.examples import NN_HEAD_NAMES, NN_HEADS
@@ -64,7 +65,7 @@ class TrainedNetwork:
def network_vector_kinds(inputs: NetworkInputs) -> tuple[str, ...]:
"""The project-vector kinds a network fit on `inputs` takes, empty for a report-only network."""
- return VECTOR_KINDS if inputs.context is not None else ()
+ return inputs.context.kinds if inputs.context is not None else ()
def network_user_kinds(inputs: NetworkInputs) -> tuple[str, ...]:
@@ -258,7 +259,8 @@ def vs_baseline_block(
Each seed fits both arms on the same train rows, so the two AUCs of one seed form a pair. One
seed's gap can be noise from the network's initialization alone, so the block keeps the range
of the gaps across seeds. `within_team` is the read the inbox order depends on, and `pooled` is
- there for reference. A seed where either arm has no AUC drops out of the pairs.
+ there for reference. `within_team_non_dogfood` leaves out PostHog's own project, whose inbox use
+ is not a customer's. A seed where either arm has no AUC drops out of the pairs.
"""
train_rows = table.network_train_rows()
heads: dict[str, dict[str, Any]] = {}
@@ -266,6 +268,7 @@ def vs_baseline_block(
rows = table.head_rows(index, holdout=True)
y = table.labels[rows, index].astype(int)
team_ids = table.team_ids(rows)
+ customer = team_ids != LABELS_TEAM_ID
heads[name] = {
"train_rows": int((~np.isnan(table.labels[train_rows, index])).sum()),
"holdout_rows": len(rows),
@@ -274,6 +277,16 @@ def vs_baseline_block(
[within_team_auc(y, scores[rows, index], team_ids).auc for scores in experiment_scores],
[within_team_auc(y, scores[rows, index], team_ids).auc for scores in baseline_scores],
),
+ "within_team_non_dogfood": _paired(
+ [
+ within_team_auc(y[customer], scores[rows, index][customer], team_ids[customer]).auc
+ for scores in experiment_scores
+ ],
+ [
+ within_team_auc(y[customer], scores[rows, index][customer], team_ids[customer]).auc
+ for scores in baseline_scores
+ ],
+ ),
"pooled": _paired(
[_auc(y, scores[rows, index]) for scores in experiment_scores],
[_auc(y, scores[rows, index]) for scores in baseline_scores],
diff --git a/products/signals/dags/inbox_ranking/nn/train.py b/products/signals/dags/inbox_ranking/nn/train.py
index 6cd99d5dfe97..ed5344a55a77 100644
--- a/products/signals/dags/inbox_ranking/nn/train.py
+++ b/products/signals/dags/inbox_ranking/nn/train.py
@@ -20,7 +20,6 @@ from sklearn.metrics import roc_auc_score
from posthog.dataclasses import frozen
from products.signals.backend.ranking.features import EMBEDDING_DIMENSIONS
-from products.signals.dags.inbox_ranking.dataset.project_vectors import VECTOR_KINDS
from products.signals.dags.inbox_ranking.multihead.inputs import NetworkConfig, NetworkInputs
from products.signals.dags.inbox_ranking.multihead.network import MultiHeadNetwork, export_onnx
from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable
@@ -40,7 +39,7 @@ def _fit(
network = MultiHeadNetwork(
heads=[head.name for head in heads],
with_context=inputs.context is not None,
- kinds=len(VECTOR_KINDS),
+ kinds=len(network_vector_kinds(inputs)),
config=config,
report_features=inputs.reports.shape[1] - EMBEDDING_DIMENSIONS,
user_kinds=inputs.users.vectors.shape[1] if inputs.users is not None else 0,
diff --git a/products/signals/dags/inbox_ranking/tests/test_experiments.py b/products/signals/dags/inbox_ranking/tests/test_experiments.py
index e05c88ebb77b..6ca33d11a4d5 100644
--- a/products/signals/dags/inbox_ranking/tests/test_experiments.py
+++ b/products/signals/dags/inbox_ranking/tests/test_experiments.py
@@ -14,6 +14,7 @@ from posthog import settings
from products.signals.backend.ranking.features import (
EMBEDDING_DIMENSIONS,
+ PROJECT_VECTOR_KINDS,
REPORT_EMBEDDINGS_EXTRA,
REPORT_EMBEDDINGS_FEATURE_SET,
REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
@@ -162,6 +163,26 @@ def test_an_experiment_must_name_a_registered_serving_baseline_and_an_expiry():
]
+ "build",
+ [
+ lambda: ModelFamily(
+ name="served_fixed",
+ feature_set=REPORT_EMBEDDINGS_FEATURE_SET,
+ project_vector_kinds=(*PROJECT_VECTOR_KINDS, "fixed"),
+ ),
+ lambda: dataclasses.replace(_pair_experiment("pair_fixed"), project_vector_kinds=("typical", "fixed")),
+ lambda: dataclasses.replace(
+ _experiment("unknown_kind", datetime.date(2026, 12, 31)), project_vector_kinds=("typical", "seen")
+ ),
+ ],
+ ids=["serving_family", "pair_experiment", "unknown_kind"],
+)
+def test_only_a_report_experiment_can_read_other_project_vector_kinds(build):
+ with pytest.raises(ValueError, match="project vector kinds"):
+ build()
+
+
@pytest.mark.parametrize(
"families,selection",
[
@@ -357,6 +378,18 @@ def test_vs_baseline_pairs_each_seed_inside_each_team_on_the_same_holdout_rows()
assert block["heads"][third]["within_team"]["paired_seeds"] == 0
assert block["heads"][third]["within_team"]["mean_delta_auc"] is None
+ # The dogfood team ranks backwards, so only the read without it sees the customer team's order.
+ dogfood = (table.rows["report_team_id"].to_numpy() == DOGFOOD_TEAM_ID)[:, None]
+ mixed = vs_baseline_block(
+ table,
+ baseline=NN_MODEL_NAME,
+ seeds=(0,),
+ experiment_scores=[np.where(dogfood, inverted, perfect)],
+ baseline_scores=[inverted],
+ )
+ assert mixed["heads"][first]["within_team"]["experiment_aucs"] == [0.5]
+ assert mixed["heads"][first]["within_team_non_dogfood"]["experiment_aucs"] == [1.0]
+
def _signal_log(signals: list[tuple[str, str, str, str]]) -> bytes:
table = pa.Table.from_pylist(
diff --git a/products/signals/dags/inbox_ranking/tests/test_nn.py b/products/signals/dags/inbox_ranking/tests/test_nn.py
index ec3a8a0c6e9c..56bc72f84a4b 100644
--- a/products/signals/dags/inbox_ranking/tests/test_nn.py
+++ b/products/signals/dags/inbox_ranking/tests/test_nn.py
@@ -58,9 +58,11 @@ from products.signals.dags.inbox_ranking.training.dag import (
served_object_key,
)
from products.signals.dags.inbox_ranking.training.families import (
+ MODEL_FAMILIES,
NN_FAMILY,
NN_MODEL_NAME,
NN_WITH_PRIO_FAMILY,
+ NN_WITH_PRIO_FIXED_MODEL_NAME,
NN_WITH_PRIO_MODEL_NAME,
ModelFamily,
)
@@ -262,8 +264,14 @@ _PRIO_EXPERIMENT = ModelFamily(
expires=datetime.date(2026, 12, 31),
)
+NN_WITH_PRIO_FIXED_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_WITH_PRIO_FIXED_MODEL_NAME)
[email protected]("family", [NN_FAMILY, NN_WITH_PRIO_FAMILY, _PRIO_EXPERIMENT], ids=lambda family: family.name)
+
+ "family",
+ [NN_FAMILY, NN_WITH_PRIO_FAMILY, _PRIO_EXPERIMENT, NN_WITH_PRIO_FIXED_FAMILY],
+ ids=lambda family: family.name,
+)
def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_order(
monkeypatch, no_side_effects, family
):
@@ -286,16 +294,21 @@ def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_
)
with dagster.build_asset_context(partition_key=day) as context:
- nn_dag.train_candidate(context, client, "bucket", day, family)
+ asset_metadata = nn_dag.train_candidate(context, client, "bucket", day, family)
metadata = json.loads(client.objects[model_object_key(PREFIX, family.name, day, METADATA_FILE)])
+ assert metadata["project_vector_kinds"] == metadata["recipe"]["project_vector_kinds"]
+ assert metadata["project_vector_kinds"] == list(family.project_vector_kinds)
+ assert {key for key in asset_metadata if key.endswith("_history_share")} == {
+ f"has_{kind}_history_share" for kind in family.project_vector_kinds
+ }
assert metadata["model_kind"] == ONNX_MODEL_KIND
assert metadata["feature_set"] == family.feature_set.name
assert metadata[ONNX_OUTPUT_HEADS_KEY] == list(NN_HEAD_NAMES)
assert model_mismatch(metadata) is None
# Only an experiment is graded against a baseline, with one paired AUC per seed and head.
if family.experiment:
- assert metadata["vs_baseline"]["baseline"] == NN_MODEL_NAME
+ assert metadata["vs_baseline"]["baseline"] == family.baseline
assert {
name: len(grade["within_team"]["experiment_aucs"])
== len(grade["within_team"]["baseline_aucs"])
@@ -322,7 +335,7 @@ def test_the_nn_candidate_writes_a_graph_whose_columns_follow_its_recorded_head_
holdout = subset(table, np.flatnonzero(table.holdout.any(axis=1)))
probabilities = onnx_probabilities(
client.objects[model_object_key(PREFIX, family.name, day, HOLDOUT_MODEL_FILE)],
- network_inputs(holdout, extras, labels, embeddings, family.feature_set),
+ network_inputs(holdout, extras, labels, embeddings, family.feature_set, family.project_vector_kinds),
metadata[ONNX_OUTPUT_HEADS_KEY],
)
for head in metadata["heads"]:
diff --git a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
index bf25aba6a9ca..7ec46008b049 100644
--- a/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
+++ b/products/signals/dags/inbox_ranking/tests/test_project_vectors.py
@@ -26,6 +26,7 @@ from products.signals.dags.inbox_ranking import common
from products.signals.dags.inbox_ranking.dataset import dag
from products.signals.dags.inbox_ranking.dataset.dag import EMBEDDINGS_SCHEMA, LABELS_SCHEMA, USER_INTERACTIONS_SCHEMA
from products.signals.dags.inbox_ranking.dataset.project_vectors import (
+ CONTEXT_KINDS,
HISTORY_LIMIT,
LABEL_COLUMNS,
MIN_HISTORY_REPORTS,
@@ -95,6 +96,7 @@ def _rows_by_key(table: pa.Table) -> dict[tuple[int, str], dict[str, Any]]:
("action", "first_implement_clicked_at"),
("action", "first_reviewer_removed_at"),
("dismiss", "first_ui_dismissed_at"),
+ ("fixed", "first_fixed_at"),
]
)
def test_only_engagement_and_vectors_before_as_of_enter_a_vector(kind: str, column: str) -> None:
@@ -104,7 +106,7 @@ def test_only_engagement_and_vectors_before_as_of_enter_a_vector(kind: str, colu
# Engaged after as_of, and engaged in time with a vector re-embedded after as_of.
engagements += [(2, 100, column, AFTER), (2, 101, column, BEFORE)]
- rows = _rows_by_key(project_vectors(_labels(engagements), _embeddings(reports), AS_OF))
+ rows = _rows_by_key(project_vectors(_labels(engagements), _embeddings(reports), AS_OF, CONTEXT_KINDS))
assert rows[(2, kind)]["n_reports"] == MIN_HISTORY_REPORTS
assert rows[(2, kind)]["vector"] == pytest.approx([1.0] * EMBEDDING_DIMENSIONS)
@@ -136,6 +138,7 @@ def test_a_team_below_the_history_threshold_keeps_a_zero_vector_row() -> None:
thin = MIN_HISTORY_REPORTS - 1
reports = [(2, index, 1.0, BEFORE) for index in range(thin)]
engagements = [(2, index, "first_create_pr_clicked_at", BEFORE) for index in range(thin)]
+ engagements += [(2, index, "first_fixed_at", BEFORE) for index in range(thin)]
rows = _rows_by_key(project_vectors(_labels(engagements), _embeddings(reports), AS_OF))
diff --git a/products/signals/dags/inbox_ranking/training/families.py b/products/signals/dags/inbox_ranking/training/families.py
index 79c8bc540876..fdc28089dfcc 100644
--- a/products/signals/dags/inbox_ranking/training/families.py
+++ b/products/signals/dags/inbox_ranking/training/families.py
@@ -23,9 +23,11 @@ from posthog.dataclasses import frozen
from products.signals.backend.ranking.features import (
PAIR_BASE_FEATURE_SET,
PAIR_USER_HISTORY_FEATURE_SET,
+ PROJECT_VECTOR_KINDS,
REPORT_EMBEDDINGS_FEATURE_SET,
REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
+ TRAINING_PROJECT_VECTOR_KINDS,
FeatureSet,
)
from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND, ONNX_MODEL_KIND
@@ -37,6 +39,9 @@ from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND
NN_MODEL_NAME = "nn_v3"
NN_WITH_PRIO_MODEL_NAME = "nn_with_prio_v2"
NN_WITH_PRIO_SOURCES_MODEL_NAME = "nn_with_prio_sources_v1"
+NN_WITH_PRIO_FIXED_MODEL_NAME = "nn_with_prio_fixed_v1"
+NN_WITH_PRIO_FIXED_NODISMISS_MODEL_NAME = "nn_with_prio_fixed_nodismiss_v1"
+FIXED_EXPERIMENT_EXPIRES = datetime.date(2026, 11, 27)
NN_PAIR_BASE_MODEL_NAME = "nn_pair_base_v1"
NN_PAIR_USER_MODEL_NAME = "nn_pair_user_v1"
PAIR_EXPERIMENT_EXPIRES = datetime.date(2026, 11, 25)
@@ -59,6 +64,10 @@ class ModelFamily:
An experiment names the serving family it is graded against in `baseline`, and the last
partition day it trains in `expires`. A serving family has neither.
+ `project_vector_kinds` are the team context kinds the network reads, in its input order. Only a
+ report-grain experiment can change them: the stored table, the serving path and the pair lane
+ read `PROJECT_VECTOR_KINDS` only.
+
A pair-grain family is always an experiment and declares its `heads` from `PAIR_HEAD_NAMES`. Its
`baseline` is another pair experiment, and it names a `baseline`, a `reference`, or both: the
first pair family of a comparison has no pair baseline and is graded against the reference only."""
@@ -74,8 +83,10 @@ class ModelFamily:
# A pair-grain family's heads. A report-grain family trains every `nn` head and declares none.
heads: tuple[str, ...] = ()
reference: str | None = None
+ project_vector_kinds: tuple[str, ...] = PROJECT_VECTOR_KINDS
def __post_init__(self) -> None:
+ self._check_project_vector_kinds()
if self.grain == PAIR_GRAIN:
self._check_pair()
return
@@ -86,6 +97,19 @@ class ModelFamily:
if not self.experiment and (self.baseline is not None or self.expires is not None):
raise ValueError(f"serving family {self.name} cannot have a baseline or an expires day")
+ def _check_project_vector_kinds(self) -> None:
+ kinds = self.project_vector_kinds
+ if not kinds or len(set(kinds)) != len(kinds) or not set(kinds) <= set(TRAINING_PROJECT_VECTOR_KINDS):
+ raise ValueError(
+ f"family {self.name} declares project vector kinds {list(kinds)}; pick distinct kinds from "
+ f"{list(TRAINING_PROJECT_VECTOR_KINDS)}"
+ )
+ if kinds != PROJECT_VECTOR_KINDS and (not self.experiment or self.grain == PAIR_GRAIN):
+ raise ValueError(
+ f"family {self.name} declares project vector kinds {list(kinds)}; only a report-grain experiment "
+ f"can read other kinds than {list(PROJECT_VECTOR_KINDS)}"
+ )
+
def _check_pair(self) -> None:
if not self.experiment:
raise ValueError(
@@ -114,6 +138,8 @@ class ModelFamily:
# `nn_with_prio_v2` is the same network with the report's priority levels as extra report inputs.
# `nn_with_prio_sources_v1` is an experiment: `nn_with_prio_v2` plus the source products of the
# report's signals as of the moment, graded against `nn_with_prio_v2`.
+# `nn_with_prio_fixed_v1` adds a `fixed` project vector to `nn_with_prio_v2`, because fixes reach
+# teams whose UI kinds are thin. `nn_with_prio_fixed_nodismiss_v1` swaps `dismiss` for `fixed`.
# `nn_pair_base_v1` and `nn_pair_user_v1` are the user-history comparison at pair grain: one network,
# one input shape, the same pairs and seeds, and the same reviewer flag. The base holds every
# user-history input at zero, so `nn_pair_user_v1` against it is the uplift of the user's history
@@ -137,6 +163,26 @@ MODEL_FAMILIES: tuple[ModelFamily, ...] = (
baseline=NN_WITH_PRIO_MODEL_NAME,
expires=datetime.date(2026, 11, 11),
),
+ ModelFamily(
+ name=NN_WITH_PRIO_FIXED_MODEL_NAME,
+ feature_set=REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
+ learner=ONNX_MODEL_KIND,
+ location=ML_LOCATION,
+ experiment=True,
+ baseline=NN_WITH_PRIO_MODEL_NAME,
+ expires=FIXED_EXPERIMENT_EXPIRES,
+ project_vector_kinds=("typical", "open", "action", "dismiss", "fixed"),
+ ),
+ ModelFamily(
+ name=NN_WITH_PRIO_FIXED_NODISMISS_MODEL_NAME,
+ feature_set=REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
+ learner=ONNX_MODEL_KIND,
+ location=ML_LOCATION,
+ experiment=True,
+ baseline=NN_WITH_PRIO_MODEL_NAME,
+ expires=FIXED_EXPERIMENT_EXPIRES,
+ project_vector_kinds=("typical", "open", "action", "fixed"),
+ ),
ModelFamily(
name=NN_PAIR_BASE_MODEL_NAME,
feature_set=PAIR_BASE_FEATURE_SET,