Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): record report content columns in the inbox ranking state (#129318)

posthog[bot]4266c3a4215c
diff --git a/products/signals/dags/inbox_ranking/README.md b/products/signals/dags/inbox_ranking/README.md
index ac4973492210..a658b5d17369 100644
--- a/products/signals/dags/inbox_ranking/README.md
+++ b/products/signals/dags/inbox_ranking/README.md
@@ -49,6 +49,14 @@ s3://<bucket>/<prefix>/
The first four are report grain and land in one table. `inbox_signal_embeddings` is signal grain, feeds the group-level model, and is read on its own — training joins it to `inbox_report_model_data` by `report_id`.
+`inbox_report_state` also records the report's content columns, and `inbox_report_model_data` passes them through: `chart_count`, `metric_count`, `suggested_prompt_count` and `tag_count`, `content_revision_count` and `corroboration_count`, `user_authored`, and `already_addressed`.
+No model reads them yet.
+
+- The list lengths are computed in Postgres, so the job never loads chart or metric definitions. A stored value that is not a list counts as 0. A null counter is stored as 0.
+- `already_addressed` comes from the same actionability artefact as `actionability`, with the same cutoff rules. It is null when that artefact is missing, was edited after the cutoff, or holds no boolean.
+- Partitions written before these columns landed do not have them. A missing column means "not recorded", not zero, so a reader must not fill it with 0. They have no backfill, because a backfill would record the report as it is today.
+- The columns do not bump `FEATURE_SCHEMA_VERSION`. A bump sets off the labels refresh sensor, which rewrites every labels partition in the lookback, and no label changed. A reader tells the partitions apart by the presence of the columns.
+
`inbox_report_title_embeddings` is a report-grain leaf. It snapshots the `title_v1` rendering the same way `inbox_report_embeddings` snapshots `title_summary_v1`, and nothing joins it. No model family trains on it since `title_embeddings` was retired; the snapshot stays for offline reads. It carries its own `embedding_inserted_at`, because a summary-only edit re-emits only `title_summary_v1`. Its dependency on `inbox_report_model_data` is for ordering, not data — it holds a vector per live report, so it runs last and alone in the run pod. That edge has a cost: a failed join, or a run that hits the job's runtime cap, skips the title snapshot for the day, and the schedule never revisits a day. Repair such a gap with a single-asset backfill while the source rows are inside their 3-month TTL.
`inbox_report_reviewers` is a (report, reviewer) leaf for user–report affinity modeling. It holds one row per entry of each spine report's current `SignalReportSuggestedReviewer` index, for teams that allow AI training. Nothing joins it, and its object carries its own schema version (`REVIEWERS_SCHEMA_VERSION`), so a failure here does not fail the training table.
diff --git a/products/signals/dags/inbox_ranking/dataset/dag.py b/products/signals/dags/inbox_ranking/dataset/dag.py
index 0ee3f17813e1..ea8dc79609e3 100644
--- a/products/signals/dags/inbox_ranking/dataset/dag.py
+++ b/products/signals/dags/inbox_ranking/dataset/dag.py
@@ -74,7 +74,9 @@ from collections.abc import Collection, Iterator, Mapping
from typing import Any, cast
from uuid import UUID
-from django.db.models import Count, Min
+from django.db.models import Case, CharField, Count, F, Func, IntegerField, Min, Value, When
+from django.db.models.functions import Coalesce
+from django.db.models.lookups import Exact
import dagster
import pyarrow as pa
@@ -251,11 +253,18 @@ def tag_dagster_queries(context: dagster.AssetExecutionContext, query_type: str)
_TIMESTAMP = pa.timestamp("us", tz="UTC")
-_STATE_FIELDS: list[tuple[str, pa.DataType]] = [
- ("snapshot_date", pa.date32()),
- ("report_id", pa.string()),
- ("report_team_id", pa.int64()),
- ("region", pa.string()),
+_REPORT_CONTENT_FIELDS: list[tuple[str, pa.DataType]] = [
+ ("chart_count", pa.int32()),
+ ("metric_count", pa.int32()),
+ ("suggested_prompt_count", pa.int32()),
+ ("tag_count", pa.int32()),
+ ("content_revision_count", pa.int32()),
+ ("corroboration_count", pa.int32()),
+ ("user_authored", pa.bool_()),
+ ("already_addressed", pa.bool_()),
+]
+
+_REPORT_STATE_FIELDS: list[tuple[str, pa.DataType]] = [
("status", pa.string()),
("billing_exempt_reason", pa.string()),
("report_created_at", _TIMESTAMP),
@@ -271,6 +280,15 @@ _STATE_FIELDS: list[tuple[str, pa.DataType]] = [
("actionability", pa.string()),
("title_chars", pa.int32()),
("summary_chars", pa.int32()),
+ *_REPORT_CONTENT_FIELDS,
+]
+
+_STATE_FIELDS: list[tuple[str, pa.DataType]] = [
+ ("snapshot_date", pa.date32()),
+ ("report_id", pa.string()),
+ ("report_team_id", pa.int64()),
+ ("region", pa.string()),
+ *_REPORT_STATE_FIELDS,
("features_observed_at", _TIMESTAMP),
]
STATE_SCHEMA = pa.schema(_STATE_FIELDS)
@@ -438,21 +456,7 @@ _MODEL_DATA_FIELDS: list[tuple[str, pa.DataType]] = [
("built_at", _TIMESTAMP),
("run_id", pa.string()),
("features_observed_at", _TIMESTAMP),
- ("status", pa.string()),
- ("billing_exempt_reason", pa.string()),
- ("report_created_at", _TIMESTAMP),
- ("promoted_at", _TIMESTAMP),
- ("last_run_at", _TIMESTAMP),
- ("pg_updated_at", _TIMESTAMP),
- ("report_age_hours", pa.float32()),
- ("hours_since_promotion", pa.float32()),
- ("signal_count", pa.int32()),
- ("total_weight", pa.float32()),
- ("run_count", pa.int32()),
- ("priority", pa.string()),
- ("actionability", pa.string()),
- ("title_chars", pa.int32()),
- ("summary_chars", pa.int32()),
+ *_REPORT_STATE_FIELDS,
("embedding_small", pa.list_(pa.float32())),
("embedding_inserted_at", _TIMESTAMP),
("embedding_rendering", pa.string()),
@@ -554,6 +558,7 @@ _STATE_PASSTHROUGH_COLUMNS = (
"actionability",
"title_chars",
"summary_chars",
+ *(name for name, _ in _REPORT_CONTENT_FIELDS),
)
@@ -573,9 +578,30 @@ def _judgment_value(parsed: dict[str, Any], key: str) -> str | None:
return value if isinstance(value, str) else None
-def _artefact_judgments(report_ids: list[str], snapshot_end: datetime.datetime) -> dict[str, dict[str, str | None]]:
+def _judgment_flag(parsed: dict[str, Any], key: str) -> bool | None:
+ value = parsed.get(key)
+ return value if isinstance(value, bool) else None
+
+
+def _json_array_length(field: str) -> Case:
+ """Length of a JSON list column, computed in Postgres so the list itself is never loaded.
+ `jsonb_array_length` raises on a value that is not an array, so any other stored value counts as 0."""
+ return Case(
+ When(
+ Exact(Func(F(field), function="jsonb_typeof", output_field=CharField()), Value("array")),
+ then=Func(F(field), function="jsonb_array_length", output_field=IntegerField()),
+ ),
+ default=Value(0),
+ output_field=IntegerField(),
+ )
+
+
+def _artefact_judgments(
+ report_ids: list[str], snapshot_end: datetime.datetime
+) -> dict[str, dict[str, str | bool | None]]:
"""Latest priority/actionability judgment per report as of the snapshot cutoff, parsed from
- the artefact content JSON.
+ the artefact content JSON. `already_addressed` comes from the same actionability artefact, so
+ it follows the same cutoff rules rather than the current `SignalReport.latest_already_addressed`.
Artefacts are appended in normal operation but the API permits editing one in place
(`update_content` rewrites content and bumps updated_at, leaving created_at alone), so a row's
@@ -584,7 +610,7 @@ def _artefact_judgments(report_ids: list[str], snapshot_end: datetime.datetime)
selection instead would hand back the judgment they superseded, which was already stale at the
cutoff — silently wrong where null is merely unknown. The genuinely immutable classification is
status_event_priority/actionability, snapshotted onto each transition in the label stream."""
- judgments: dict[str, dict[str, str | None]] = {}
+ judgments: dict[str, dict[str, str | bool | None]] = {}
for chunk in _chunked(report_ids):
artefacts = (
SignalReportArtefact.objects.filter(
@@ -610,11 +636,14 @@ def _artefact_judgments(report_ids: list[str], snapshot_end: datetime.datetime)
# Legacy artefact rows can hold JSON that is not an object; readers tolerate them.
if not isinstance(parsed, dict):
continue
- entry = judgments.setdefault(str(report_id), {"priority": None, "actionability": None})
+ entry = judgments.setdefault(
+ str(report_id), {"priority": None, "actionability": None, "already_addressed": None}
+ )
if artefact_type == SignalReportArtefact.ArtefactType.PRIORITY_JUDGMENT:
entry["priority"] = _judgment_value(parsed, "priority")
else:
entry["actionability"] = _judgment_value(parsed, "actionability")
+ entry["already_addressed"] = _judgment_flag(parsed, "already_addressed")
return judgments
@@ -875,6 +904,12 @@ def inbox_report_state(context: dagster.AssetExecutionContext) -> None:
for spine_chunk in _chunked(ordered_spine_ids):
for report in (
SignalReport.objects.filter(id__in=spine_chunk)
+ .annotate(
+ chart_count=_json_array_length("charts"),
+ metric_count=_json_array_length("metrics"),
+ suggested_prompt_count=_json_array_length("suggested_prompts"),
+ tag_count=Coalesce(Func(F("tags"), function="cardinality"), Value(0), output_field=IntegerField()),
+ )
.values(
"id",
"team_id",
@@ -889,6 +924,13 @@ def inbox_report_state(context: dagster.AssetExecutionContext) -> None:
"run_count",
"title",
"summary",
+ "chart_count",
+ "metric_count",
+ "suggested_prompt_count",
+ "tag_count",
+ "content_revision_count",
+ "corroboration_count",
+ "user_authored",
)
.iterator(chunk_size=2000)
):
@@ -919,6 +961,14 @@ def inbox_report_state(context: dagster.AssetExecutionContext) -> None:
"actionability": judgment.get("actionability"),
"title_chars": len(report["title"] or ""),
"summary_chars": len(report["summary"] or ""),
+ "chart_count": report["chart_count"],
+ "metric_count": report["metric_count"],
+ "suggested_prompt_count": report["suggested_prompt_count"],
+ "tag_count": report["tag_count"],
+ "content_revision_count": report["content_revision_count"] or 0,
+ "corroboration_count": report["corroboration_count"] or 0,
+ "user_authored": report["user_authored"],
+ "already_addressed": judgment.get("already_addressed"),
"features_observed_at": features_observed_at,
}
)
diff --git a/products/signals/dags/inbox_ranking/tests/test_dataset.py b/products/signals/dags/inbox_ranking/tests/test_dataset.py
index f07fe919f2b3..ebf4a519f7ab 100644
--- a/products/signals/dags/inbox_ranking/tests/test_dataset.py
+++ b/products/signals/dags/inbox_ranking/tests/test_dataset.py
@@ -1,3 +1,4 @@
+import json
import datetime
from typing import Any
@@ -404,8 +405,20 @@ def _embedding_row(report_id: str, **overrides):
}
+REPORT_CONTENT_COLUMNS = {
+ "chart_count": 2,
+ "metric_count": 1,
+ "suggested_prompt_count": 3,
+ "tag_count": 4,
+ "content_revision_count": 5,
+ "corroboration_count": 6,
+ "user_authored": True,
+ "already_addressed": False,
+}
+
+
def test_assemble_model_rows_spine_and_join():
- state = [_state_row("r1"), _state_row("r2")]
+ state = [_state_row("r1", **REPORT_CONTENT_COLUMNS), _state_row("r2")]
embeddings = [
_embedding_row("r1"),
_embedding_row("r2", embedding_small=None, is_tombstone=True),
@@ -433,6 +446,7 @@ def test_assemble_model_rows_spine_and_join():
assert rows["r1"]["has_embedding"] is True
assert rows["r1"]["open_count"] == 4
assert rows["r1"]["label_provenance_ok"] is True
+ assert {column: rows["r1"][column] for column in REPORT_CONTENT_COLUMNS} == REPORT_CONTENT_COLUMNS
assert rows["r2"]["has_embedding"] is False
assert rows["r2"]["embedding_small"] is None
assert rows["r2"]["open_count"] == 0
@@ -588,6 +602,82 @@ class TestSpineInclusion(BaseTest):
assert metadata["excluded_no_training_consent_reports"].value == 2
assert metadata["excluded_no_training_consent_teams"].value == 2
+ def _state_snapshot(self) -> dict[str, dict[str, Any]]:
+ self.organization.is_ai_training_opted_in = True
+ self.organization.save()
+ written: dict[str, Any] = {}
+ with (
+ patch.object(dag, "skip_unconfigured", lambda context: False),
+ patch.object(dag, "tag_dagster_queries", lambda context, query_type: None),
+ patch.object(dag, "labels_team", lambda: self.team),
+ patch.object(dag, "hogql_rows", lambda *args, **kwargs: []),
+ patch.object(dag, "s3_client", lambda: None),
+ patch.object(dag, "write_parquet", lambda client, bucket, key, table: written.update(table=table)),
+ dagster.build_asset_context(partition_key=SNAPSHOT_DATE.isoformat()) as context,
+ ):
+ dag.inbox_report_state(context)
+ return {row["report_id"]: row for row in written["table"].to_pylist()}
+
+ def test_state_snapshot_counts_report_content_without_failing_on_a_value_that_is_not_a_list(self):
+ populated = self._report(SignalReport.Status.READY)
+ SignalReport.objects.filter(id=populated).update(
+ charts=[{"id": "c1"}, {"id": "c2"}],
+ metrics=[{"id": "m1"}],
+ suggested_prompts=["a", "b", "c"],
+ tags=["x", "y"],
+ content_revision_count=3,
+ corroboration_count=4,
+ user_authored=True,
+ )
+ empty = self._report(SignalReport.Status.READY)
+ not_a_list = self._report(SignalReport.Status.READY)
+ SignalReport.objects.filter(id=not_a_list).update(charts={"id": "c1"}, metrics="metric", suggested_prompts=7)
+
+ rows = self._state_snapshot()
+
+ counts = ("chart_count", "metric_count", "suggested_prompt_count", "tag_count")
+ counters = ("content_revision_count", "corroboration_count")
+ assert {column: rows[populated][column] for column in (*counts, *counters, "user_authored")} == {
+ "chart_count": 2,
+ "metric_count": 1,
+ "suggested_prompt_count": 3,
+ "tag_count": 2,
+ "content_revision_count": 3,
+ "corroboration_count": 4,
+ "user_authored": True,
+ }
+ for report_id in (empty, not_a_list):
+ assert {column: rows[report_id][column] for column in (*counts, *counters)} == dict.fromkeys(
+ (*counts, *counters), 0
+ )
+ assert rows[report_id]["user_authored"] is False
+
+ @parameterized.expand(
+ [
+ ("latest_pre_cutoff_artefact_wins", [(True, T1, None), (False, T2, None)], False),
+ ("edited_after_the_cutoff_is_unknown", [(True, T1, None), (False, T2, AFTER_SNAPSHOT_END)], None),
+ ("created_after_the_cutoff_is_ignored", [(True, T1, None), (False, AFTER_SNAPSHOT_END, None)], True),
+ ("a_value_that_is_not_a_boolean_is_unknown", [("yes", T1, None)], None),
+ ("no_artefact_is_unknown", [], None),
+ ]
+ )
+ def test_state_snapshot_reads_already_addressed_as_of_the_cutoff(self, _name, artefacts, expected):
+ report_id = self._report(SignalReport.Status.READY)
+ # The current value must never leak into a past snapshot.
+ SignalReport.objects.filter(id=report_id).update(latest_already_addressed=True)
+ for already_addressed, created_at, updated_at in artefacts:
+ artefact = SignalReportArtefact.objects.create(
+ team=self.team,
+ report_id=report_id,
+ type=SignalReportArtefact.ArtefactType.ACTIONABILITY_JUDGMENT,
+ content=json.dumps({"actionability": "immediately_actionable", "already_addressed": already_addressed}),
+ )
+ SignalReportArtefact.objects.filter(id=artefact.id).update(
+ created_at=created_at, updated_at=updated_at or created_at
+ )
+
+ assert self._state_snapshot()[report_id]["already_addressed"] is expected
+
class TestReviewerSnapshot(BaseTest):
def _report_with_reviewers(self, team: Team, entries: list[tuple[str | None, str | None]]) -> str: