Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): add command to measure activity summary source coverage (#121316)

posthog[bot]5a845bd8d269
diff --git a/products/signals/backend/activity_summary/measure.py b/products/signals/backend/activity_summary/measure.py
new file mode 100644
index 000000000000..2b8c6a2096ab
--- /dev/null
+++ b/products/signals/backend/activity_summary/measure.py
@@ -0,0 +1,316 @@
+"""Measure what each input source adds to an activity summary. Nothing here writes a summary row.
+
+Coverage runs the same reads and filters as the tick and keeps only counts, so no user text leaves
+the process. Leave one source out calls the model once with all sources and once per removed
+source, and returns the summaries for rating by hand.
+"""
+
+import random
+import datetime
+from collections import Counter
+from collections.abc import Iterable, Sequence
+from enum import StrEnum
+from typing import Any
+
+from posthog.dataclasses import frozen
+from posthog.models import ActivityLog, Team
+
+from products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS, user_activity, user_tasks
+from products.signals.backend.activity_summary.queue import ActivitySummaryPair
+from products.signals.backend.activity_summary.render import (
+ ACTIVITY_LINES,
+ TASK_LINES,
+ ActivityItem,
+ Drop,
+ RenderedInput,
+ TaskItem,
+ build_input,
+ sampling_seed,
+)
+from products.signals.backend.activity_summary.summaries import (
+ DEFAULT_MODEL,
+ MODEL_PRICES,
+ cost_usd,
+ gateway_client,
+ summarize,
+)
+from products.tasks.backend.facade.user_task_summaries import list_user_task_times
+
+# Teams per pair query, so a run over a whole region reads in bounded batches.
+TEAM_BATCH_SIZE = 500
+PERCENTILES = (50, 90, 99)
+
+
+class Coverage(StrEnum):
+ """Which sources still have units after the filters and caps."""
+
+ TASKS_ONLY = "tasks_only"
+ ACTIVITY_ONLY = "activity_only"
+ BOTH = "both"
+ NEITHER = "neither"
+
+
+class Variant(StrEnum):
+ ALL = "all"
+ WITHOUT_TASKS = "without_tasks"
+ WITHOUT_ACTIVITY = "without_activity"
+
+
+@frozen
+class PairCoverage:
+ coverage: Coverage
+ raw_tasks: int
+ raw_activity: int
+ task_units: int
+ activity_units: int
+ task_chars: int
+ activity_chars: int
+ task_lines_capped: bool
+ activity_lines_capped: bool
+ sections_budget_capped: bool
+ drops: dict[str, int]
+
+ @property
+ def units(self) -> int:
+ return self.task_units + self.activity_units
+
+
+@frozen
+class CoverageReport:
+ pairs: int
+ coverage: dict[str, int]
+ # Per metric, the value at each of `PERCENTILES`.
+ percentiles: dict[str, dict[str, int]]
+ caps: dict[str, int]
+ drops: dict[str, int]
+ min_units: int
+ pass_min_units: int
+
+ def as_dict(self) -> dict[str, Any]:
+ return {
+ "pairs": self.pairs,
+ "coverage": self.coverage,
+ "percentiles": self.percentiles,
+ "caps": self.caps,
+ "drops": self.drops,
+ "min_units": self.min_units,
+ "pass_min_units": self.pass_min_units,
+ }
+
+
+def _classify(rendered: RenderedInput) -> Coverage:
+ tasks = rendered.n_tasks > 0
+ activity = rendered.n_activity_lines > 0
+ if tasks and activity:
+ return Coverage.BOTH
+ if tasks:
+ return Coverage.TASKS_ONLY
+ if activity:
+ return Coverage.ACTIVITY_ONLY
+ return Coverage.NEITHER
+
+
+def measure_pair(
+ tasks: Sequence[TaskItem], activity: Sequence[ActivityItem], *, as_of: datetime.datetime, seed: int
+) -> PairCoverage:
+ rendered = build_input(tasks, activity, as_of=as_of, seed=seed)
+ if rendered is None:
+ return PairCoverage(
+ coverage=Coverage.NEITHER,
+ raw_tasks=len(tasks),
+ raw_activity=len(activity),
+ task_units=0,
+ activity_units=0,
+ task_chars=0,
+ activity_chars=0,
+ task_lines_capped=False,
+ activity_lines_capped=False,
+ sections_budget_capped=False,
+ drops={},
+ )
+ return PairCoverage(
+ coverage=_classify(rendered),
+ raw_tasks=len(tasks),
+ raw_activity=len(activity),
+ task_units=rendered.n_tasks,
+ activity_units=rendered.n_activity_lines,
+ task_chars=rendered.task_chars,
+ activity_chars=rendered.activity_chars,
+ task_lines_capped=rendered.task_pool > TASK_LINES,
+ activity_lines_capped=rendered.activity_pool > ACTIVITY_LINES,
+ sections_budget_capped=Drop.SECTIONS_BUDGET in rendered.drops,
+ drops=rendered.drops,
+ )
+
+
+def _percentile(sorted_values: Sequence[int], percent: int) -> int:
+ """Nearest rank, so the value is one that a pair has."""
+ if not sorted_values:
+ return 0
+ rank = max(1, -(-percent * len(sorted_values) // 100))
+ return sorted_values[rank - 1]
+
+
+def coverage_report(pairs: Iterable[PairCoverage], *, min_units: int) -> CoverageReport:
+ coverage: Counter[str] = Counter()
+ caps: Counter[str] = Counter()
+ drops: Counter[str] = Counter()
+ metrics: dict[str, list[int]] = {
+ "raw_tasks": [],
+ "raw_activity": [],
+ "task_units": [],
+ "activity_units": [],
+ "task_chars": [],
+ "activity_chars": [],
+ }
+ count = 0
+ passed = 0
+ for pair in pairs:
+ count += 1
+ coverage[pair.coverage] += 1
+ caps["task_lines"] += pair.task_lines_capped
+ caps["activity_lines"] += pair.activity_lines_capped
+ caps["sections_budget"] += pair.sections_budget_capped
+ drops.update(pair.drops)
+ passed += pair.units >= min_units
+ for name, values in metrics.items():
+ values.append(getattr(pair, name))
+ return CoverageReport(
+ pairs=count,
+ coverage={str(kind): coverage[kind] for kind in Coverage},
+ percentiles={
+ name: {f"p{percent}": _percentile(sorted(values), percent) for percent in PERCENTILES}
+ for name, values in metrics.items()
+ },
+ caps=dict(caps),
+ drops=dict(sorted(drops.items())),
+ min_units=min_units,
+ pass_min_units=passed,
+ )
+
+
+def scope_team_ids(organization_id: str | None, team_id: int | None) -> list[int]:
+ teams = Team.objects.all()
+ if organization_id is not None:
+ teams = teams.filter(organization_id=organization_id)
+ if team_id is not None:
+ teams = teams.filter(id=team_id)
+ return list(teams.order_by("id").values_list("id", flat=True))
+
+
+def candidate_pairs(
+ team_ids: Sequence[int], start: datetime.datetime, end: datetime.datetime
+) -> list[ActivitySummaryPair]:
+ """Every (team, user) pair with a task or an activity log row of their own in [start, end).
+
+ The same row filters as `inputs.py`. The render filters are not applied here, so a pair whose
+ rows all drop counts as `Coverage.NEITHER`."""
+ pairs: set[ActivitySummaryPair] = set()
+ for offset in range(0, len(team_ids), TEAM_BATCH_SIZE):
+ batch = list(team_ids[offset : offset + TEAM_BATCH_SIZE])
+ rows = (
+ ActivityLog.objects.filter(
+ team_id__in=batch, user_id__isnull=False, created_at__gte=start, created_at__lt=end
+ )
+ .exclude(is_system=True)
+ .exclude(was_impersonated=True)
+ .values_list("team_id", "user_id")
+ .distinct()
+ )
+ pairs.update(ActivitySummaryPair(team_id=team, user_id=user) for team, user in rows if team is not None)
+ pairs.update(
+ ActivitySummaryPair(team_id=task.team_id, user_id=task.user_id)
+ for task in list_user_task_times(team_ids=batch, start=start, end=end, origin_products=TASK_ORIGIN_PRODUCTS)
+ )
+ return sorted(pairs)
+
+
+def sample_pairs(
+ pairs: Sequence[ActivitySummaryPair], sample: int | None, seed: int | None
+) -> list[ActivitySummaryPair]:
+ if sample is None or sample >= len(pairs):
+ return list(pairs)
+ return sorted(random.Random(seed).sample(list(pairs), sample))
+
+
+def measure_coverage(
+ pairs: Iterable[ActivitySummaryPair], *, start: datetime.datetime, now: datetime.datetime
+) -> Iterable[PairCoverage]:
+ """One pair at a time, so memory holds the rows of one pair only."""
+ for pair in pairs:
+ yield measure_pair(
+ user_tasks(pair.team_id, pair.user_id, start, now),
+ user_activity(pair.team_id, pair.user_id, start, now),
+ as_of=now,
+ seed=sampling_seed(now.date().isoformat(), pair.team_id, pair.user_id),
+ )
+
+
+def leave_one_out_inputs(
+ tasks: Sequence[TaskItem], activity: Sequence[ActivityItem], *, as_of: datetime.datetime, seed: int
+) -> dict[Variant, RenderedInput]:
+ """The full input, and one input per source that has units in it. A variant with no units left is omitted."""
+ full = build_input(tasks, activity, as_of=as_of, seed=seed)
+ if full is None:
+ return {}
+ variants = {Variant.ALL: full}
+ if full.n_tasks:
+ without_tasks = build_input([], activity, as_of=as_of, seed=seed)
+ if without_tasks is not None:
+ variants[Variant.WITHOUT_TASKS] = without_tasks
+ if full.n_activity_lines:
+ without_activity = build_input(tasks, [], as_of=as_of, seed=seed)
+ if without_activity is not None:
+ variants[Variant.WITHOUT_ACTIVITY] = without_activity
+ return variants
+
+
+def leave_one_out_pair(
+ pair: ActivitySummaryPair, *, start: datetime.datetime, now: datetime.datetime, min_units: int
+) -> list[dict[str, Any]]:
+ """One record per variant. With no previous summary, so each summary reflects its input only."""
+ variants = leave_one_out_inputs(
+ user_tasks(pair.team_id, pair.user_id, start, now),
+ user_activity(pair.team_id, pair.user_id, start, now),
+ as_of=now,
+ seed=sampling_seed(now.date().isoformat(), pair.team_id, pair.user_id),
+ )
+ if Variant.ALL not in variants or variants[Variant.ALL].units < min_units:
+ return []
+ client = gateway_client(pair.team_id, pair.user_id, now.date())
+ records: list[dict[str, Any]] = []
+ for variant, rendered in variants.items():
+ outcome = summarize(
+ client, rendered_input=rendered.text, previous_summary=None, model=DEFAULT_MODEL, as_of=now.date()
+ )
+ records.append(
+ {
+ "team_id": pair.team_id,
+ "user_id": pair.user_id,
+ "variant": str(variant),
+ "tasks": rendered.n_tasks,
+ "activity_lines": rendered.n_activity_lines,
+ "input_chars": rendered.input_chars,
+ "tokens_in": outcome.tokens_in,
+ "tokens_out": outcome.tokens_out,
+ "cost_usd": cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(DEFAULT_MODEL)),
+ "latency_ms": outcome.latency_ms,
+ "error": outcome.error,
+ "summary": outcome.summary,
+ }
+ )
+ return records
+
+
+def leave_one_out_totals(records: Iterable[dict[str, Any]]) -> dict[str, dict[str, float]]:
+ totals: dict[str, dict[str, float]] = {}
+ for record in records:
+ total = totals.setdefault(
+ record["variant"], {"calls": 0, "ok": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0}
+ )
+ total["calls"] += 1
+ total["ok"] += record["summary"] is not None
+ total["tokens_in"] += record["tokens_in"] or 0
+ total["tokens_out"] += record["tokens_out"] or 0
+ total["cost_usd"] += record["cost_usd"] or 0.0
+ return totals
diff --git a/products/signals/backend/activity_summary/render.py b/products/signals/backend/activity_summary/render.py
index bb29f45c05ce..6d036b20ef97 100644
--- a/products/signals/backend/activity_summary/render.py
+++ b/products/signals/backend/activity_summary/render.py
@@ -139,6 +139,9 @@ class RenderedInput:
input_chars: int
input_tokens_estimate: int
drops: dict[str, int]
+ # Units of each source before its line cap, so a caller can tell when the cap binds.
+ task_pool: int
+ activity_pool: int
@property
def units(self) -> int:
@@ -371,6 +374,8 @@ def build_input(
drops: Counter[str] = Counter()
task_pool = burst_cap(dedupe_latest(task_units(tasks, drops), drops), drops)
activity_lines = activity_units(activity, drops)
+ # Only `activity_units` has counted a sample cap drop so far.
+ activity_pool = len(activity_lines) + drops[Drop.SAMPLE_CAP]
if not task_pool and not activity_lines:
return None
chosen_tasks = sample_units(task_pool, limit=TASK_LINES, as_of=as_of, seed=seed)
@@ -388,4 +393,6 @@ def build_input(
input_chars=len(text),
input_tokens_estimate=estimate_tokens(len(text)),
drops={str(reason): count for reason, count in sorted(drops.items()) if count},
+ task_pool=len(task_pool),
+ activity_pool=activity_pool,
)
diff --git a/products/signals/backend/management/commands/measure_activity_summary_sources.py b/products/signals/backend/management/commands/measure_activity_summary_sources.py
new file mode 100644
index 000000000000..0f9c7277cff7
--- /dev/null
+++ b/products/signals/backend/management/commands/measure_activity_summary_sources.py
@@ -0,0 +1,83 @@
+"""Measure the coverage and the signal of each activity summary input source. Writes no summary rows.
+
+`--mode coverage` prints aggregate counts only. `--mode leave-one-out` calls the model and writes the
+summaries to a local JSONL file, so it runs only for one organization that approved AI data processing.
+"""
+
+import json
+import datetime
+from pathlib import Path
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from posthog.models import Organization
+
+from products.signals.backend.activity_summary.logic import MIN_UNITS
+from products.signals.backend.activity_summary.measure import (
+ candidate_pairs,
+ coverage_report,
+ leave_one_out_pair,
+ leave_one_out_totals,
+ measure_coverage,
+ sample_pairs,
+ scope_team_ids,
+)
+from products.signals.backend.activity_summary.render import WINDOW_DAYS
+
+MODE_COVERAGE = "coverage"
+MODE_LEAVE_ONE_OUT = "leave-one-out"
+# Bounds the model calls of a leave-one-out run: up to three per pair.
+DEFAULT_LEAVE_ONE_OUT_SAMPLE = 20
+
+
+class Command(BaseCommand):
+ help = "Measure how much the tasks and the activity log sources contribute to activity summaries."
+
+ def add_arguments(self, parser):
+ parser.add_argument("--mode", choices=[MODE_COVERAGE, MODE_LEAVE_ONE_OUT], default=MODE_COVERAGE)
+ parser.add_argument("--organization-id", type=str, help="Required for leave-one-out.")
+ parser.add_argument("--team-id", type=int)
+ parser.add_argument("--sample", type=int, help="Measure a random sample of this many pairs.")
+ parser.add_argument("--seed", type=int, help="Seed of the pair sample, for a repeatable run.")
+ parser.add_argument("--window-days", type=int, default=WINDOW_DAYS)
+ parser.add_argument("--min-units", type=int, default=MIN_UNITS)
+ parser.add_argument(
+ "--output",
+ type=Path,
+ default=Path("activity_summary_leave_one_out.jsonl"),
+ help="The local JSONL file of leave-one-out.",
+ )
+
+ def handle(self, *args, **options):
+ mode = options["mode"]
+ organization_id = options["organization_id"]
+ if mode == MODE_LEAVE_ONE_OUT:
+ if not organization_id:
+ raise CommandError("leave-one-out sends real input to the model and needs --organization-id.")
+ organization = Organization.objects.filter(id=organization_id).first()
+ if organization is None or organization.is_ai_data_processing_approved is not True:
+ raise CommandError("leave-one-out needs an organization that approved AI data processing.")
+
+ now = timezone.now()
+ start = now - datetime.timedelta(days=options["window_days"])
+ team_ids = scope_team_ids(organization_id, options["team_id"])
+ sample = options["sample"]
+ if mode == MODE_LEAVE_ONE_OUT and sample is None:
+ sample = DEFAULT_LEAVE_ONE_OUT_SAMPLE
+ pairs = sample_pairs(candidate_pairs(team_ids, start, now), sample, options["seed"])
+ self.stderr.write(f"teams: {len(team_ids)}, pairs: {len(pairs)}")
+
+ if mode == MODE_COVERAGE:
+ report = coverage_report(measure_coverage(pairs, start=start, now=now), min_units=options["min_units"])
+ self.stdout.write(json.dumps(report.as_dict(), indent=2, sort_keys=True))
+ return
+
+ records = []
+ with options["output"].open("w") as output:
+ for pair in pairs:
+ for record in leave_one_out_pair(pair, start=start, now=now, min_units=options["min_units"]):
+ output.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
+ records.append(record)
+ self.stderr.write(f"wrote {len(records)} records to {options['output']}")
+ self.stdout.write(json.dumps(leave_one_out_totals(records), indent=2, sort_keys=True))
diff --git a/products/signals/backend/test/test_activity_summary_measure.py b/products/signals/backend/test/test_activity_summary_measure.py
new file mode 100644
index 000000000000..63b0d8e7a4da
--- /dev/null
+++ b/products/signals/backend/test/test_activity_summary_measure.py
@@ -0,0 +1,128 @@
+import json
+import datetime
+from io import StringIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import pytest
+import time_machine
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.core.management import CommandError, call_command
+
+from parameterized import parameterized
+
+from posthog.models import ActivityLog
+
+from products.signals.backend.activity_summary.measure import Coverage, coverage_report, measure_pair
+from products.signals.backend.activity_summary.render import ACTIVITY_LINES, ActivityItem, TaskItem
+from products.signals.backend.models import UserActivitySummary
+from products.signals.backend.test.test_activity_summary_render import FakeLLM
+
+NOW = datetime.datetime(2026, 10, 19, 12, tzinfo=datetime.UTC)
+ANSWER = json.dumps(
+ {
+ "summary": "Builds the signup funnel.",
+ "focus_areas": [{"topic": "signup funnel", "strength": "high", "last_seen": "2026-10-18"}],
+ "products": ["product_analytics"],
+ "recent_shift": None,
+ }
+)
+TASK = TaskItem(
+ at=NOW - datetime.timedelta(days=1), title="why", summary="Goal: find why weekly signups dropped.", tags=()
+)
+
+
+def _change(name: str, client: str | None = None) -> ActivityItem:
+ return ActivityItem(
+ at=NOW - datetime.timedelta(days=1), scope="Insight", activity="created", name=name, client=client
+ )
+
+
+ [
+ ("tasks only", [TASK], [], Coverage.TASKS_ONLY, False),
+ ("activity only", [], [_change("Signup funnel")], Coverage.ACTIVITY_ONLY, False),
+ ("both", [TASK], [_change("Signup funnel")], Coverage.BOTH, False),
+ ("neither once the filters drop every row", [], [_change("Scout funnel", "scout:x")], Coverage.NEITHER, False),
+ (
+ "the activity line cap binds",
+ [],
+ [_change(f"Funnel {index}") for index in range(ACTIVITY_LINES + 1)],
+ Coverage.ACTIVITY_ONLY,
+ True,
+ ),
+ ]
+)
+def test_measure_pair_classifies_coverage(
+ _name: str, tasks: list[TaskItem], activity: list[ActivityItem], coverage: Coverage, capped: bool
+) -> None:
+ pair = measure_pair(tasks, activity, as_of=NOW, seed=1)
+
+ assert (pair.coverage, pair.activity_lines_capped) == (coverage, capped)
+ report = coverage_report([pair], min_units=1)
+ assert report.coverage[coverage] == 1
+ assert report.caps["activity_lines"] == int(capped)
+
+
+@time_machine.travel(NOW, tick=False)
+class TestMeasureActivitySummarySources(BaseTest):
+ def _log(self, name: str) -> None:
+ ActivityLog.objects.create(
+ team_id=self.team.id,
+ organization_id=self.organization.id,
+ user=self.user,
+ scope="Insight",
+ activity="created",
+ item_id=name,
+ detail={"name": name},
+ created_at=NOW - datetime.timedelta(days=1),
+ )
+
+ def _run(self, *args: str) -> str:
+ out = StringIO()
+ with patch("products.signals.backend.activity_summary.measure.list_user_task_times", return_value=[]):
+ call_command("measure_activity_summary_sources", *args, stdout=out, stderr=StringIO())
+ return out.getvalue()
+
+ def test_coverage_prints_counts_without_rendered_text(self) -> None:
+ self._log("Signup funnel")
+
+ with patch("products.signals.backend.activity_summary.measure.user_tasks", return_value=[TASK]):
+ out = self._run()
+
+ assert "Signup funnel" not in out and "signups" not in out
+ report = json.loads(out)
+ assert (report["pairs"], report["coverage"][Coverage.BOTH]) == (1, 1)
+
+ def test_leave_one_out_refuses_without_an_organization(self) -> None:
+ with pytest.raises(CommandError):
+ self._run("--mode", "leave-one-out")
+
+ def test_leave_one_out_builds_one_variant_per_source_and_writes_no_summary(self) -> None:
+ for index in range(3):
+ self._log(f"Funnel {index}")
+ llm = FakeLLM([ANSWER])
+
+ with (
+ TemporaryDirectory() as directory,
+ patch("products.signals.backend.activity_summary.measure.user_tasks", return_value=[TASK]),
+ patch("products.signals.backend.activity_summary.measure.gateway_client", return_value=llm),
+ ):
+ output = Path(directory) / "out.jsonl"
+ totals = json.loads(
+ self._run(
+ "--mode", "leave-one-out", "--organization-id", str(self.organization.id), "--output", str(output)
+ )
+ )
+ records = [json.loads(line) for line in output.read_text().splitlines()]
+
+ assert [(r["variant"], r["tasks"], r["activity_lines"]) for r in records] == [
+ ("all", 1, 3),
+ ("without_tasks", 0, 3),
+ ("without_activity", 1, 0),
+ ]
+ assert len(llm.requests) == 3
+ assert set(totals) == {"all", "without_tasks", "without_activity"}
+ assert not UserActivitySummary.objects.for_team(self.team.id).exists()