:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): add a report sources experiment for inbox ranking (#121623)
posthog[bot]17f60c6486b3
diff --git a/products/signals/backend/ranking/features.py b/products/signals/backend/ranking/features.pyindex 21ed05b718d7..b26dc26f83b9 100644--- a/products/signals/backend/ranking/features.py+++ b/products/signals/backend/ranking/features.py@@ -6,13 +6,14 @@ matrix. A model records the set it was fit on in its `metadata.json`, so a run csets at once: the examples Parquet, the training matrix and the scoring matrix are all built perset, and a model whose declared set this build cannot produce is left unscored.-Two sets live here. `report_embeddings` is the report's combined-text embedding vector, and+Three sets live here. `report_embeddings` is the report's combined-text embedding vector, and`report_embeddings_priority` is that vector plus the report's priority, one column per level. Theyare the served contract: training (`products/signals/dags/inbox_ranking/training/`) and serving(the scoring sweep in this package) build features through the same `build_matrix`, so the model's`feature_names` match the serving matrix by construction. The sweep reads the vector at scoringtime, and reads the latest priority judgment only for a model whose set declares the `priority`-state column.+state column. `report_embeddings_priority_sources` adds the source products of the report's signals.+Only an experiment family trains on it, and the sweep cannot build it yet."""import abc@@ -47,6 +48,38 @@ EMBEDDING_FEATURE_NAMES = tuple(f"emb_{index}" for index in range(EMBEDDING_DIMEPRIORITY_COLUMN = "priority"PRIORITY_LEVELS = ("P0", "P1", "P2", "P3", "P4")+# The source products of a report's signals, under this key in `extras`: the metadata columns of the+# dataset dag's `inbox_signal_embeddings` emission log, one row per signal emission, with the moment+# the emission landed in `SIGNAL_INSERTED_AT_COLUMN`.+REPORT_SOURCES_EXTRA = "report_sources"+SIGNAL_INSERTED_AT_COLUMN = "embedding_inserted_at"+REPORT_SOURCES_COLUMNS = (+ "team_id",+ "signal_id",+ "report_id",+ "source_product",+ "is_deleted",+ SIGNAL_INSERTED_AT_COLUMN,+)+# A fixed vocabulary, so a new source product does not change the feature names. A product outside+# it sets `source_other`.+SOURCE_PRODUCTS = (+ "signals_scout",+ "error_tracking",+ "session_replay",+ "replay_vision",+ "conversations",+ "zendesk",+ "github",+ "linear",+ "jira",+ "llm_analytics",+ "logs",+ "health_checks",+ "signals_check",+)+SOURCE_FEATURE_NAMES = (*(f"source_{product}" for product in SOURCE_PRODUCTS), "source_other")+# The team's project-vector kinds, in the order a network that reads them takes them. The dataset# dag's `inbox_project_vectors` writes one row per (team, kind), and the training and serving# inputs both read these names, so the two sides cannot order the kinds differently.@@ -232,10 +265,73 @@ class ReportEmbeddingsPriorityFeatureSet(ReportEmbeddingsFeatureSet):return matrix+class ReportEmbeddingsPrioritySourcesFeatureSet(ReportEmbeddingsPriorityFeatureSet):+ """The priority set, then the source products of the report's signals as of the moment.++ One column per product in `SOURCE_PRODUCTS`, `source_other` for any other or a missing product,+ `source_count` for the number of distinct products, and `has_sources`. The signals come from the+ `REPORT_SOURCES_EXTRA` side input, an emission log with one row per signal emission. A moment+ reads only the emissions inserted at or before it, takes the latest emission per+ (team_id, signal_id), and drops a signal whose latest emission is a retraction. A signal can+ move to another report when it is regrouped, so the report comes from that latest emission.++ The impression labels also carry a `source_products` column, but it holds the value of the+ latest impression, so it can postdate the moment. This set never reads it.++ A report with no live signal at the moment gets zero in every source column, `has_sources`+ included. `buildable` is the parent's, so this set keeps the same rows as its baseline.+ """++ schema_version = 1+ feature_names = (+ *ReportEmbeddingsPriorityFeatureSet.feature_names,+ *SOURCE_FEATURE_NAMES,+ "source_count",+ "has_sources",+ )++ def __init__(self, *, name: str, extras_key: str, sources_key: str) -> None:+ super().__init__(name=name, extras_key=extras_key)+ self.extras_keys = (extras_key, sources_key)++ def build_matrix(+ self, rows: pd.DataFrame, extras: Extras = NO_EXTRAS, *, as_of: datetime.datetime | None = None+ ) -> pd.DataFrame:+ if as_of is None:+ raise ValueError(f"the {self.name} feature set reads its sources only as of a moment")+ matrix = super().build_matrix(rows, extras, as_of=as_of)+ sources = self._sources_as_of(extras[self.extras_keys[1]], rows.index, as_of)+ for column in (*SOURCE_FEATURE_NAMES, "source_count", "has_sources"):+ matrix[column] = sources[column].to_numpy(dtype=np.float32)+ return matrix++ @staticmethod+ def _sources_as_of(signals: pd.DataFrame, report_ids: pd.Index, as_of: datetime.datetime) -> pd.DataFrame:+ known = signals.loc[signals[SIGNAL_INSERTED_AT_COLUMN] <= as_of]+ latest = known.sort_values(SIGNAL_INSERTED_AT_COLUMN, kind="stable").drop_duplicates(+ ["team_id", "signal_id"], keep="last"+ )+ live = latest.loc[~latest["is_deleted"].astype(bool) & latest["report_id"].isin(report_ids)]+ product = live["source_product"].where(live["source_product"].isin(SOURCE_PRODUCTS), "other")+ one_hot = (+ pd.crosstab(live["report_id"], "source_" + product.astype(str))+ .clip(upper=1)+ .reindex(index=report_ids, columns=list(SOURCE_FEATURE_NAMES), fill_value=0)+ )+ # A missing product counts as one source of its own, so `has_sources` implies a count of 1 or more.+ distinct = live["source_product"].fillna("").groupby(live["report_id"]).nunique()+ one_hot["source_count"] = distinct.reindex(report_ids, fill_value=0).to_numpy()+ one_hot["has_sources"] = (one_hot["source_count"] > 0).astype(int).to_numpy()+ return one_hot++REPORT_EMBEDDINGS_FEATURE_SET = ReportEmbeddingsFeatureSet(name="report_embeddings", extras_key=REPORT_EMBEDDINGS_EXTRA)REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET = ReportEmbeddingsPriorityFeatureSet(name="report_embeddings_priority", extras_key=REPORT_EMBEDDINGS_EXTRA)+REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET = ReportEmbeddingsPrioritySourcesFeatureSet(+ name="report_embeddings_priority_sources", extras_key=REPORT_EMBEDDINGS_EXTRA, sources_key=REPORT_SOURCES_EXTRA+)# Every set this build can produce, by name. A model that names a set absent from here cannot be# scored, the same way a model whose feature names have moved on cannot.@@ -243,6 +339,7 @@ FEATURE_SETS: Mapping[str, FeatureSet] = MappingProxyType({REPORT_EMBEDDINGS_FEATURE_SET.name: REPORT_EMBEDDINGS_FEATURE_SET,REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET.name: REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,+ REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET.name: REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,})diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.mdindex 3495dc77ad8a..f75b42065b50 100644--- a/products/signals/dags/inbox_ranking/README.md+++ b/products/signals/dags/inbox_ranking/README.md@@ -396,7 +396,9 @@ An experiment is a `MODEL_FAMILIES` entry with `experiment=True`, a `baseline` s- **No new examples.** It reads the day's `inbox_ranking_nn_examples` object. A day without that object is skipped, and so is a family after its `expires` day.- **Never served.** The serving manifest reads only `serving_families()`. The pin check refuses an experiment key, so the promote job, the served pointer and the pins cannot name one. To serve the recipe, register it as a serving family under a new `model_name`.- **Paired read.** The experiment and its baseline each fit once per seed in `EXPERIMENT_SEEDS`, on the same train rows. `metadata.json` holds a `vs_baseline` block per head: each seed's within-team and pooled holdout AUC for both arms, and the mean, min and max of the gaps. One `inbox_ranking_experiment_compared` event per head carries the gaps.-- **Side inputs.** A side input beyond the report vectors needs a loader in `training/side_inputs.py`. The registry refuses a frame without a timezone-aware `as_of_column` and drops the rows known after the partition end. A feature set reads the frame through `rows_as_of`.+- **Side inputs.** A side input beyond the report vectors needs a loader in `training/side_inputs.py`. The registry refuses a frame without a timezone-aware `as_of_column` and drops the rows known after the partition end. A feature set in the dags reads the frame through `rows_as_of`. A set in `features.py` cannot import the dags, so it applies the same `<= as_of` cut on the loader's column itself.++The live experiment is `nn_with_prio_sources_v1`, graded against `nn_with_prio_v2` until 2026-11-11. Its set, `report_embeddings_priority_sources`, adds the source products of the report's signals: one column per product in a fixed vocabulary, `source_other`, `source_count` and `has_sources`. The `report_sources` loader reads only the metadata columns of `inbox_signal_embeddings`, from 30 days before the training lookback to dt=D. A moment takes the latest emission per `(team_id, signal_id)` inserted at or before it and drops a retracted signal. It never reads the impression labels' `source_products` column, which holds the latest impression's value. The scoring sweep cannot build this set, so the recipe needs serving work before it can become a serving family.The schedule is a fallback. To read an experiment on a chosen day, launch the job for that partition from the Dagster UI or the CLI:@@ -447,7 +449,7 @@ The dags train only on reports from organizations whose `Organization.is_ai_traiThe filter follows the current setting, so an organization that opts back in is trained on again.The state asset and the `nn` examples asset record `excluded_no_training_consent_reports` and `excluded_no_training_consent_teams` as metadata. They are counts, never ids.The online grade reads the dt=D state snapshot for its newborn pool, so that pool shrinks to consenting teams too. Scoring in the sweep does not change.-The filter does not delete existing partitions or models. A re-run of `inbox_signal_embeddings` stays additive, so rows that an older run wrote remain in that partition. Nothing trains on signal embeddings today, and a future reader must apply the same filter.+The filter does not delete existing partitions or models. A re-run of `inbox_signal_embeddings` stays additive, so rows that an older run wrote remain in that partition. The `report_sources` side input reads only its metadata columns, and the example rows it joins to already drop the teams without training consent.## Deletion and retentiondiff --git a/products/signals/dags/inbox_ranking/tests/test_experiments.py b/products/signals/dags/inbox_ranking/tests/test_experiments.pyindex 5a530ed9895e..e1b7669874ae 100644--- a/products/signals/dags/inbox_ranking/tests/test_experiments.py+++ b/products/signals/dags/inbox_ranking/tests/test_experiments.py@@ -5,12 +5,21 @@ import pytestimport numpy as npimport pandas as pdimport dagster+import pyarrow as pa+import pyarrow.parquet as pqfrom posthog import settings-from products.signals.backend.ranking.features import REPORT_EMBEDDINGS_FEATURE_SET+from products.signals.backend.ranking.features import (+ EMBEDDING_DIMENSIONS,+ REPORT_EMBEDDINGS_EXTRA,+ REPORT_EMBEDDINGS_FEATURE_SET,+ REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,+ REPORT_SOURCES_EXTRA,+)from products.signals.dags.inbox_ranking.common import partition_object_key-from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable+from products.signals.dags.inbox_ranking.dataset.dag import SIGNAL_EMBEDDINGS_SCHEMA, SIGNAL_EMBEDDINGS_TABLE+from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable, report_matrixfrom products.signals.dags.inbox_ranking.nn import experimentsfrom products.signals.dags.inbox_ranking.nn.dag import NN_EXAMPLES_TABLEfrom products.signals.dags.inbox_ranking.nn.examples import NN_HEAD_NAMES@@ -180,3 +189,59 @@ def test_vs_baseline_pairs_each_seed_inside_each_team_on_the_same_holdout_rows()assert block["heads"][first]["pooled"] == pairedassert block["heads"][third]["within_team"]["paired_seeds"] == 0assert block["heads"][third]["within_team"]["mean_delta_auc"] is None+++def _signal_log(signals: list[tuple[str, str, str, str]]) -> bytes:+ table = pa.Table.from_pylist(+ [+ {+ "signal_id": signal_id,+ "team_id": 1,+ "report_id": report_id,+ "source_product": product,+ "embedding_inserted_at": datetime.datetime.fromisoformat(inserted_at),+ "is_deleted": False,+ }+ for signal_id, report_id, product, inserted_at in signals+ ],+ schema=SIGNAL_EMBEDDINGS_SCHEMA,+ )+ sink = pa.BufferOutputStream()+ pq.write_table(table, sink)+ return sink.getvalue().to_pybytes()+++def test_a_late_signal_does_not_leak_into_an_earlier_row():+ day_before, day = datetime.date(2026, 10, 19), datetime.date(2026, 10, 20)+ objects = {+ partition_object_key(PREFIX, SIGNAL_EMBEDDINGS_TABLE, day_before.isoformat()): _signal_log(+ [("s1", "r1", "error_tracking", "2026-10-19T10:00:00+00:00")]+ ),+ partition_object_key(PREFIX, SIGNAL_EMBEDDINGS_TABLE, day.isoformat()): _signal_log(+ [("s2", "r1", "github", "2026-10-20T10:00:00+00:00")]+ ),+ }+ vectors = pd.DataFrame(+ {+ "embedding_small": [np.zeros(EMBEDDING_DIMENSIONS, dtype=np.float32)] * 2,+ "embedding_inserted_at": pd.to_datetime(["2026-10-19T00:00:00"] * 2, utc=True),+ },+ index=pd.Index(["r1", "r2"], name="report_id"),+ )+ extras = {+ REPORT_EMBEDDINGS_EXTRA: vectors,+ **load_side_inputs(_ModelStoreS3(objects), "bucket", PREFIX, PARTITION, [REPORT_SOURCES_EXTRA]),+ }+ rows = pd.DataFrame(+ {"snapshot_date": [day_before, day, day_before], "report_id": ["r1", "r1", "r2"], "priority": [None] * 3}+ )++ matrix = pd.DataFrame(+ report_matrix(rows, extras, REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET),+ columns=list(REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET.feature_names),+ )++ columns = ["source_error_tracking", "source_github", "source_other", "source_count", "has_sources"]+ # The github signal landed on dt=D, after the dt=D-1 row's moment, so only the dt=D row sees it.+ # A report with no signals has zero in every source column.+ assert matrix[columns].values.tolist() == [[1, 0, 0, 1, 1], [1, 1, 0, 2, 1], [0, 0, 0, 0, 0]]diff --git a/products/signals/dags/inbox_ranking/training/families.py b/products/signals/dags/inbox_ranking/training/families.pyindex 528a8d49cf64..7df45b995129 100644--- a/products/signals/dags/inbox_ranking/training/families.py+++ b/products/signals/dags/inbox_ranking/training/families.py@@ -16,6 +16,7 @@ from posthog.dataclasses import frozenfrom products.signals.backend.ranking.features import (REPORT_EMBEDDINGS_FEATURE_SET,REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,+ REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,FeatureSet,)from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND, ONNX_MODEL_KIND@@ -26,6 +27,7 @@ from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND# servable by key, and their online grades keep their own series.NN_MODEL_NAME = "nn_v3"NN_WITH_PRIO_MODEL_NAME = "nn_with_prio_v2"+NN_WITH_PRIO_SOURCES_MODEL_NAME = "nn_with_prio_sources_v1"# The Dagster code locations that train a family: `signals` on the main image, and `signals_ml` on# the image that carries torch.MAIN_LOCATION = "signals"@@ -65,6 +67,8 @@ class ModelFamily:# torch, so the `signals_ml` location trains it. The main location fits no family. `nn` declares# the report embeddings set, because that set builds its report input in training and at serving.# `nn_with_prio_v2` is the same network with the report's priority levels as extra report inputs.+# `nn_with_prio_sources_v1` is an experiment: `nn_with_prio_v2` plus the source products of the+# report's signals as of the moment, graded against `nn_with_prio_v2`.MODEL_FAMILIES: tuple[ModelFamily, ...] = (ModelFamily(name=NN_MODEL_NAME, feature_set=REPORT_EMBEDDINGS_FEATURE_SET, learner=ONNX_MODEL_KIND, location=ML_LOCATION@@ -75,6 +79,15 @@ MODEL_FAMILIES: tuple[ModelFamily, ...] = (learner=ONNX_MODEL_KIND,location=ML_LOCATION,),+ ModelFamily(+ name=NN_WITH_PRIO_SOURCES_MODEL_NAME,+ feature_set=REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,+ learner=ONNX_MODEL_KIND,+ location=ML_LOCATION,+ experiment=True,+ baseline=NN_WITH_PRIO_MODEL_NAME,+ expires=datetime.date(2026, 11, 11),+ ),)NN_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_MODEL_NAME)NN_WITH_PRIO_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_WITH_PRIO_MODEL_NAME)diff --git a/products/signals/dags/inbox_ranking/training/side_inputs.py b/products/signals/dags/inbox_ranking/training/side_inputs.pyindex 1e97240ce662..1df2a545d802 100644--- a/products/signals/dags/inbox_ranking/training/side_inputs.py+++ b/products/signals/dags/inbox_ranking/training/side_inputs.py@@ -16,10 +16,17 @@ from collections.abc import Callable, Iterableimport pandas as pd+from posthog import settingsfrom posthog.dataclasses import frozen-from products.signals.backend.ranking.features import Extras-from products.signals.dags.inbox_ranking.common import snapshot_bounds+from products.signals.backend.ranking.features import (+ REPORT_SOURCES_COLUMNS,+ REPORT_SOURCES_EXTRA,+ SIGNAL_INSERTED_AT_COLUMN,+ Extras,+)+from products.signals.dags.inbox_ranking.common import partition_object_key, read_parquet_if_exists, snapshot_bounds+from products.signals.dags.inbox_ranking.dataset.dag import SIGNAL_EMBEDDINGS_TABLE# Reads the dt=D object of one side input. Returns None when the object was never written.LoadFrame = Callable[[object, str, str, str], pd.DataFrame | None]@@ -82,3 +89,37 @@ def load_side_inputs(client, bucket: str, prefix: str, partition_key: str, keys:if frame is not None:loaded[key] = _known_by(frame, loader, snapshot_end)return loaded+++# A report groups signals that can arrive before it is born, so the sources read starts this many+# days before the training lookback.+REPORT_SOURCES_MARGIN_DAYS = 30+++def load_report_sources(client, bucket: str, prefix: str, partition_key: str) -> pd.DataFrame | None:+ """The metadata columns of every `inbox_signal_embeddings` emission from the start of the+ training lookback, less `REPORT_SOURCES_MARGIN_DAYS`, to dt=D. The log holds only each day's+ inserts, so the frame is the union of the days. The vector column is never read. A missing+ earlier day is left out, but a missing dt=D returns None, because that day's sources are unknown."""+ end = datetime.date.fromisoformat(partition_key)+ days = settings.INBOX_RANKING_TRAINING_LOOKBACK_DAYS + REPORT_SOURCES_MARGIN_DAYS+ frames: list[pd.DataFrame] = []+ for offset in range(days + 1):+ day = (end - datetime.timedelta(days=offset)).isoformat()+ table = read_parquet_if_exists(+ client,+ bucket,+ partition_object_key(prefix, SIGNAL_EMBEDDINGS_TABLE, day),+ columns=list(REPORT_SOURCES_COLUMNS),+ )+ if table is None:+ if offset == 0:+ return None+ continue+ frames.append(table.to_pandas())+ return pd.concat(frames, ignore_index=True)+++register_side_input(+ SideInputLoader(key=REPORT_SOURCES_EXTRA, as_of_column=SIGNAL_INSERTED_AT_COLUMN, load=load_report_sources)+)