Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): add a report sources experiment for inbox ranking (#121623)

posthog[bot]17f60c6486b3
diff --git a/products/signals/backend/ranking/features.py b/products/signals/backend/ranking/features.py
index 21ed05b718d7..b26dc26f83b9 100644
--- a/products/signals/backend/ranking/features.py
+++ b/products/signals/backend/ranking/features.py
@@ -6,13 +6,14 @@ matrix. A model records the set it was fit on in its `metadata.json`, so a run c
sets at once: the examples Parquet, the training matrix and the scoring matrix are all built per
set, and a model whose declared set this build cannot produce is left unscored.
-Two sets live here. `report_embeddings` is the report's combined-text embedding vector, and
+Three sets live here. `report_embeddings` is the report's combined-text embedding vector, and
`report_embeddings_priority` is that vector plus the report's priority, one column per level. They
are the served contract: training (`products/signals/dags/inbox_ranking/training/`) and serving
(the scoring sweep in this package) build features through the same `build_matrix`, so the model's
`feature_names` match the serving matrix by construction. The sweep reads the vector at scoring
time, and reads the latest priority judgment only for a model whose set declares the `priority`
-state column.
+state column. `report_embeddings_priority_sources` adds the source products of the report's signals.
+Only an experiment family trains on it, and the sweep cannot build it yet.
"""
import abc
@@ -47,6 +48,38 @@ EMBEDDING_FEATURE_NAMES = tuple(f"emb_{index}" for index in range(EMBEDDING_DIME
PRIORITY_COLUMN = "priority"
PRIORITY_LEVELS = ("P0", "P1", "P2", "P3", "P4")
+# The source products of a report's signals, under this key in `extras`: the metadata columns of the
+# dataset dag's `inbox_signal_embeddings` emission log, one row per signal emission, with the moment
+# the emission landed in `SIGNAL_INSERTED_AT_COLUMN`.
+REPORT_SOURCES_EXTRA = "report_sources"
+SIGNAL_INSERTED_AT_COLUMN = "embedding_inserted_at"
+REPORT_SOURCES_COLUMNS = (
+ "team_id",
+ "signal_id",
+ "report_id",
+ "source_product",
+ "is_deleted",
+ SIGNAL_INSERTED_AT_COLUMN,
+)
+# A fixed vocabulary, so a new source product does not change the feature names. A product outside
+# it sets `source_other`.
+SOURCE_PRODUCTS = (
+ "signals_scout",
+ "error_tracking",
+ "session_replay",
+ "replay_vision",
+ "conversations",
+ "zendesk",
+ "github",
+ "linear",
+ "jira",
+ "llm_analytics",
+ "logs",
+ "health_checks",
+ "signals_check",
+)
+SOURCE_FEATURE_NAMES = (*(f"source_{product}" for product in SOURCE_PRODUCTS), "source_other")
+
# The team's project-vector kinds, in the order a network that reads them takes them. The dataset
# dag's `inbox_project_vectors` writes one row per (team, kind), and the training and serving
# inputs both read these names, so the two sides cannot order the kinds differently.
@@ -232,10 +265,73 @@ class ReportEmbeddingsPriorityFeatureSet(ReportEmbeddingsFeatureSet):
return matrix
+class ReportEmbeddingsPrioritySourcesFeatureSet(ReportEmbeddingsPriorityFeatureSet):
+ """The priority set, then the source products of the report's signals as of the moment.
+
+ One column per product in `SOURCE_PRODUCTS`, `source_other` for any other or a missing product,
+ `source_count` for the number of distinct products, and `has_sources`. The signals come from the
+ `REPORT_SOURCES_EXTRA` side input, an emission log with one row per signal emission. A moment
+ reads only the emissions inserted at or before it, takes the latest emission per
+ (team_id, signal_id), and drops a signal whose latest emission is a retraction. A signal can
+ move to another report when it is regrouped, so the report comes from that latest emission.
+
+ The impression labels also carry a `source_products` column, but it holds the value of the
+ latest impression, so it can postdate the moment. This set never reads it.
+
+ A report with no live signal at the moment gets zero in every source column, `has_sources`
+ included. `buildable` is the parent's, so this set keeps the same rows as its baseline.
+ """
+
+ schema_version = 1
+ feature_names = (
+ *ReportEmbeddingsPriorityFeatureSet.feature_names,
+ *SOURCE_FEATURE_NAMES,
+ "source_count",
+ "has_sources",
+ )
+
+ def __init__(self, *, name: str, extras_key: str, sources_key: str) -> None:
+ super().__init__(name=name, extras_key=extras_key)
+ self.extras_keys = (extras_key, sources_key)
+
+ def build_matrix(
+ self, rows: pd.DataFrame, extras: Extras = NO_EXTRAS, *, as_of: datetime.datetime | None = None
+ ) -> pd.DataFrame:
+ if as_of is None:
+ raise ValueError(f"the {self.name} feature set reads its sources only as of a moment")
+ matrix = super().build_matrix(rows, extras, as_of=as_of)
+ sources = self._sources_as_of(extras[self.extras_keys[1]], rows.index, as_of)
+ for column in (*SOURCE_FEATURE_NAMES, "source_count", "has_sources"):
+ matrix[column] = sources[column].to_numpy(dtype=np.float32)
+ return matrix
+
+ @staticmethod
+ def _sources_as_of(signals: pd.DataFrame, report_ids: pd.Index, as_of: datetime.datetime) -> pd.DataFrame:
+ known = signals.loc[signals[SIGNAL_INSERTED_AT_COLUMN] <= as_of]
+ latest = known.sort_values(SIGNAL_INSERTED_AT_COLUMN, kind="stable").drop_duplicates(
+ ["team_id", "signal_id"], keep="last"
+ )
+ live = latest.loc[~latest["is_deleted"].astype(bool) & latest["report_id"].isin(report_ids)]
+ product = live["source_product"].where(live["source_product"].isin(SOURCE_PRODUCTS), "other")
+ one_hot = (
+ pd.crosstab(live["report_id"], "source_" + product.astype(str))
+ .clip(upper=1)
+ .reindex(index=report_ids, columns=list(SOURCE_FEATURE_NAMES), fill_value=0)
+ )
+ # A missing product counts as one source of its own, so `has_sources` implies a count of 1 or more.
+ distinct = live["source_product"].fillna("").groupby(live["report_id"]).nunique()
+ one_hot["source_count"] = distinct.reindex(report_ids, fill_value=0).to_numpy()
+ one_hot["has_sources"] = (one_hot["source_count"] > 0).astype(int).to_numpy()
+ return one_hot
+
+
REPORT_EMBEDDINGS_FEATURE_SET = ReportEmbeddingsFeatureSet(name="report_embeddings", extras_key=REPORT_EMBEDDINGS_EXTRA)
REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET = ReportEmbeddingsPriorityFeatureSet(
name="report_embeddings_priority", extras_key=REPORT_EMBEDDINGS_EXTRA
)
+REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET = ReportEmbeddingsPrioritySourcesFeatureSet(
+ name="report_embeddings_priority_sources", extras_key=REPORT_EMBEDDINGS_EXTRA, sources_key=REPORT_SOURCES_EXTRA
+)
# Every set this build can produce, by name. A model that names a set absent from here cannot be
# scored, the same way a model whose feature names have moved on cannot.
@@ -243,6 +339,7 @@ FEATURE_SETS: Mapping[str, FeatureSet] = MappingProxyType(
{
REPORT_EMBEDDINGS_FEATURE_SET.name: REPORT_EMBEDDINGS_FEATURE_SET,
REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET.name: REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
+ REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET.name: REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
}
)
diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md
index 3495dc77ad8a..f75b42065b50 100644
--- a/products/signals/dags/inbox_ranking/README.md
+++ b/products/signals/dags/inbox_ranking/README.md
@@ -396,7 +396,9 @@ An experiment is a `MODEL_FAMILIES` entry with `experiment=True`, a `baseline` s
- **No new examples.** It reads the day's `inbox_ranking_nn_examples` object. A day without that object is skipped, and so is a family after its `expires` day.
- **Never served.** The serving manifest reads only `serving_families()`. The pin check refuses an experiment key, so the promote job, the served pointer and the pins cannot name one. To serve the recipe, register it as a serving family under a new `model_name`.
- **Paired read.** The experiment and its baseline each fit once per seed in `EXPERIMENT_SEEDS`, on the same train rows. `metadata.json` holds a `vs_baseline` block per head: each seed's within-team and pooled holdout AUC for both arms, and the mean, min and max of the gaps. One `inbox_ranking_experiment_compared` event per head carries the gaps.
-- **Side inputs.** A side input beyond the report vectors needs a loader in `training/side_inputs.py`. The registry refuses a frame without a timezone-aware `as_of_column` and drops the rows known after the partition end. A feature set reads the frame through `rows_as_of`.
+- **Side inputs.** A side input beyond the report vectors needs a loader in `training/side_inputs.py`. The registry refuses a frame without a timezone-aware `as_of_column` and drops the rows known after the partition end. A feature set in the dags reads the frame through `rows_as_of`. A set in `features.py` cannot import the dags, so it applies the same `<= as_of` cut on the loader's column itself.
+
+The live experiment is `nn_with_prio_sources_v1`, graded against `nn_with_prio_v2` until 2026-11-11. Its set, `report_embeddings_priority_sources`, adds the source products of the report's signals: one column per product in a fixed vocabulary, `source_other`, `source_count` and `has_sources`. The `report_sources` loader reads only the metadata columns of `inbox_signal_embeddings`, from 30 days before the training lookback to dt=D. A moment takes the latest emission per `(team_id, signal_id)` inserted at or before it and drops a retracted signal. It never reads the impression labels' `source_products` column, which holds the latest impression's value. The scoring sweep cannot build this set, so the recipe needs serving work before it can become a serving family.
The schedule is a fallback. To read an experiment on a chosen day, launch the job for that partition from the Dagster UI or the CLI:
@@ -447,7 +449,7 @@ The dags train only on reports from organizations whose `Organization.is_ai_trai
The filter follows the current setting, so an organization that opts back in is trained on again.
The state asset and the `nn` examples asset record `excluded_no_training_consent_reports` and `excluded_no_training_consent_teams` as metadata. They are counts, never ids.
The online grade reads the dt=D state snapshot for its newborn pool, so that pool shrinks to consenting teams too. Scoring in the sweep does not change.
-The filter does not delete existing partitions or models. A re-run of `inbox_signal_embeddings` stays additive, so rows that an older run wrote remain in that partition. Nothing trains on signal embeddings today, and a future reader must apply the same filter.
+The filter does not delete existing partitions or models. A re-run of `inbox_signal_embeddings` stays additive, so rows that an older run wrote remain in that partition. The `report_sources` side input reads only its metadata columns, and the example rows it joins to already drop the teams without training consent.
## Deletion and retention
diff --git a/products/signals/dags/inbox_ranking/tests/test_experiments.py b/products/signals/dags/inbox_ranking/tests/test_experiments.py
index 5a530ed9895e..e1b7669874ae 100644
--- a/products/signals/dags/inbox_ranking/tests/test_experiments.py
+++ b/products/signals/dags/inbox_ranking/tests/test_experiments.py
@@ -5,12 +5,21 @@ import pytest
import numpy as np
import pandas as pd
import dagster
+import pyarrow as pa
+import pyarrow.parquet as pq
from posthog import settings
-from products.signals.backend.ranking.features import REPORT_EMBEDDINGS_FEATURE_SET
+from products.signals.backend.ranking.features import (
+ EMBEDDING_DIMENSIONS,
+ REPORT_EMBEDDINGS_EXTRA,
+ REPORT_EMBEDDINGS_FEATURE_SET,
+ REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
+ REPORT_SOURCES_EXTRA,
+)
from products.signals.dags.inbox_ranking.common import partition_object_key
-from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable
+from products.signals.dags.inbox_ranking.dataset.dag import SIGNAL_EMBEDDINGS_SCHEMA, SIGNAL_EMBEDDINGS_TABLE
+from products.signals.dags.inbox_ranking.multihead.prototype import ExampleTable, report_matrix
from products.signals.dags.inbox_ranking.nn import experiments
from products.signals.dags.inbox_ranking.nn.dag import NN_EXAMPLES_TABLE
from products.signals.dags.inbox_ranking.nn.examples import NN_HEAD_NAMES
@@ -180,3 +189,59 @@ def test_vs_baseline_pairs_each_seed_inside_each_team_on_the_same_holdout_rows()
assert block["heads"][first]["pooled"] == paired
assert block["heads"][third]["within_team"]["paired_seeds"] == 0
assert block["heads"][third]["within_team"]["mean_delta_auc"] is None
+
+
+def _signal_log(signals: list[tuple[str, str, str, str]]) -> bytes:
+ table = pa.Table.from_pylist(
+ [
+ {
+ "signal_id": signal_id,
+ "team_id": 1,
+ "report_id": report_id,
+ "source_product": product,
+ "embedding_inserted_at": datetime.datetime.fromisoformat(inserted_at),
+ "is_deleted": False,
+ }
+ for signal_id, report_id, product, inserted_at in signals
+ ],
+ schema=SIGNAL_EMBEDDINGS_SCHEMA,
+ )
+ sink = pa.BufferOutputStream()
+ pq.write_table(table, sink)
+ return sink.getvalue().to_pybytes()
+
+
+def test_a_late_signal_does_not_leak_into_an_earlier_row():
+ day_before, day = datetime.date(2026, 10, 19), datetime.date(2026, 10, 20)
+ objects = {
+ partition_object_key(PREFIX, SIGNAL_EMBEDDINGS_TABLE, day_before.isoformat()): _signal_log(
+ [("s1", "r1", "error_tracking", "2026-10-19T10:00:00+00:00")]
+ ),
+ partition_object_key(PREFIX, SIGNAL_EMBEDDINGS_TABLE, day.isoformat()): _signal_log(
+ [("s2", "r1", "github", "2026-10-20T10:00:00+00:00")]
+ ),
+ }
+ vectors = pd.DataFrame(
+ {
+ "embedding_small": [np.zeros(EMBEDDING_DIMENSIONS, dtype=np.float32)] * 2,
+ "embedding_inserted_at": pd.to_datetime(["2026-10-19T00:00:00"] * 2, utc=True),
+ },
+ index=pd.Index(["r1", "r2"], name="report_id"),
+ )
+ extras = {
+ REPORT_EMBEDDINGS_EXTRA: vectors,
+ **load_side_inputs(_ModelStoreS3(objects), "bucket", PREFIX, PARTITION, [REPORT_SOURCES_EXTRA]),
+ }
+ rows = pd.DataFrame(
+ {"snapshot_date": [day_before, day, day_before], "report_id": ["r1", "r1", "r2"], "priority": [None] * 3}
+ )
+
+ matrix = pd.DataFrame(
+ report_matrix(rows, extras, REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET),
+ columns=list(REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET.feature_names),
+ )
+
+ columns = ["source_error_tracking", "source_github", "source_other", "source_count", "has_sources"]
+ # The github signal landed on dt=D, after the dt=D-1 row's moment, so only the dt=D row sees it.
+ # A report with no signals has zero in every source column.
+ assert matrix[columns].values.tolist() == [[1, 0, 0, 1, 1], [1, 1, 0, 2, 1], [0, 0, 0, 0, 0]]
diff --git a/products/signals/dags/inbox_ranking/training/families.py b/products/signals/dags/inbox_ranking/training/families.py
index 528a8d49cf64..7df45b995129 100644
--- a/products/signals/dags/inbox_ranking/training/families.py
+++ b/products/signals/dags/inbox_ranking/training/families.py
@@ -16,6 +16,7 @@ from posthog.dataclasses import frozen
from products.signals.backend.ranking.features import (
REPORT_EMBEDDINGS_FEATURE_SET,
REPORT_EMBEDDINGS_PRIORITY_FEATURE_SET,
+ REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
FeatureSet,
)
from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND, ONNX_MODEL_KIND
@@ -26,6 +27,7 @@ from products.signals.backend.ranking.serving_manifest import DEFAULT_MODEL_KIND
# servable by key, and their online grades keep their own series.
NN_MODEL_NAME = "nn_v3"
NN_WITH_PRIO_MODEL_NAME = "nn_with_prio_v2"
+NN_WITH_PRIO_SOURCES_MODEL_NAME = "nn_with_prio_sources_v1"
# The Dagster code locations that train a family: `signals` on the main image, and `signals_ml` on
# the image that carries torch.
MAIN_LOCATION = "signals"
@@ -65,6 +67,8 @@ class ModelFamily:
# torch, so the `signals_ml` location trains it. The main location fits no family. `nn` declares
# the report embeddings set, because that set builds its report input in training and at serving.
# `nn_with_prio_v2` is the same network with the report's priority levels as extra report inputs.
+# `nn_with_prio_sources_v1` is an experiment: `nn_with_prio_v2` plus the source products of the
+# report's signals as of the moment, graded against `nn_with_prio_v2`.
MODEL_FAMILIES: tuple[ModelFamily, ...] = (
ModelFamily(
name=NN_MODEL_NAME, feature_set=REPORT_EMBEDDINGS_FEATURE_SET, learner=ONNX_MODEL_KIND, location=ML_LOCATION
@@ -75,6 +79,15 @@ MODEL_FAMILIES: tuple[ModelFamily, ...] = (
learner=ONNX_MODEL_KIND,
location=ML_LOCATION,
),
+ ModelFamily(
+ name=NN_WITH_PRIO_SOURCES_MODEL_NAME,
+ feature_set=REPORT_EMBEDDINGS_PRIORITY_SOURCES_FEATURE_SET,
+ learner=ONNX_MODEL_KIND,
+ location=ML_LOCATION,
+ experiment=True,
+ baseline=NN_WITH_PRIO_MODEL_NAME,
+ expires=datetime.date(2026, 11, 11),
+ ),
)
NN_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_MODEL_NAME)
NN_WITH_PRIO_FAMILY = next(family for family in MODEL_FAMILIES if family.name == NN_WITH_PRIO_MODEL_NAME)
diff --git a/products/signals/dags/inbox_ranking/training/side_inputs.py b/products/signals/dags/inbox_ranking/training/side_inputs.py
index 1e97240ce662..1df2a545d802 100644
--- a/products/signals/dags/inbox_ranking/training/side_inputs.py
+++ b/products/signals/dags/inbox_ranking/training/side_inputs.py
@@ -16,10 +16,17 @@ from collections.abc import Callable, Iterable
import pandas as pd
+from posthog import settings
from posthog.dataclasses import frozen
-from products.signals.backend.ranking.features import Extras
-from products.signals.dags.inbox_ranking.common import snapshot_bounds
+from products.signals.backend.ranking.features import (
+ REPORT_SOURCES_COLUMNS,
+ REPORT_SOURCES_EXTRA,
+ SIGNAL_INSERTED_AT_COLUMN,
+ Extras,
+)
+from products.signals.dags.inbox_ranking.common import partition_object_key, read_parquet_if_exists, snapshot_bounds
+from products.signals.dags.inbox_ranking.dataset.dag import SIGNAL_EMBEDDINGS_TABLE
# Reads the dt=D object of one side input. Returns None when the object was never written.
LoadFrame = Callable[[object, str, str, str], pd.DataFrame | None]
@@ -82,3 +89,37 @@ def load_side_inputs(client, bucket: str, prefix: str, partition_key: str, keys:
if frame is not None:
loaded[key] = _known_by(frame, loader, snapshot_end)
return loaded
+
+
+# A report groups signals that can arrive before it is born, so the sources read starts this many
+# days before the training lookback.
+REPORT_SOURCES_MARGIN_DAYS = 30
+
+
+def load_report_sources(client, bucket: str, prefix: str, partition_key: str) -> pd.DataFrame | None:
+ """The metadata columns of every `inbox_signal_embeddings` emission from the start of the
+ training lookback, less `REPORT_SOURCES_MARGIN_DAYS`, to dt=D. The log holds only each day's
+ inserts, so the frame is the union of the days. The vector column is never read. A missing
+ earlier day is left out, but a missing dt=D returns None, because that day's sources are unknown."""
+ end = datetime.date.fromisoformat(partition_key)
+ days = settings.INBOX_RANKING_TRAINING_LOOKBACK_DAYS + REPORT_SOURCES_MARGIN_DAYS
+ frames: list[pd.DataFrame] = []
+ for offset in range(days + 1):
+ day = (end - datetime.timedelta(days=offset)).isoformat()
+ table = read_parquet_if_exists(
+ client,
+ bucket,
+ partition_object_key(prefix, SIGNAL_EMBEDDINGS_TABLE, day),
+ columns=list(REPORT_SOURCES_COLUMNS),
+ )
+ if table is None:
+ if offset == 0:
+ return None
+ continue
+ frames.append(table.to_pandas())
+ return pd.concat(frames, ignore_index=True)
+
+
+register_side_input(
+ SideInputLoader(key=REPORT_SOURCES_EXTRA, as_of_column=SIGNAL_INSERTED_AT_COLUMN, load=load_report_sources)
+)