:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): add command to measure activity summary source coverage (#121316)
posthog[bot]5a845bd8d269
diff --git a/products/signals/backend/activity_summary/measure.py b/products/signals/backend/activity_summary/measure.pynew file mode 100644index 000000000000..2b8c6a2096ab--- /dev/null+++ b/products/signals/backend/activity_summary/measure.py@@ -0,0 +1,316 @@+"""Measure what each input source adds to an activity summary. Nothing here writes a summary row.++Coverage runs the same reads and filters as the tick and keeps only counts, so no user text leaves+the process. Leave one source out calls the model once with all sources and once per removed+source, and returns the summaries for rating by hand.+"""++import random+import datetime+from collections import Counter+from collections.abc import Iterable, Sequence+from enum import StrEnum+from typing import Any++from posthog.dataclasses import frozen+from posthog.models import ActivityLog, Team++from products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS, user_activity, user_tasks+from products.signals.backend.activity_summary.queue import ActivitySummaryPair+from products.signals.backend.activity_summary.render import (+ ACTIVITY_LINES,+ TASK_LINES,+ ActivityItem,+ Drop,+ RenderedInput,+ TaskItem,+ build_input,+ sampling_seed,+)+from products.signals.backend.activity_summary.summaries import (+ DEFAULT_MODEL,+ MODEL_PRICES,+ cost_usd,+ gateway_client,+ summarize,+)+from products.tasks.backend.facade.user_task_summaries import list_user_task_times++# Teams per pair query, so a run over a whole region reads in bounded batches.+TEAM_BATCH_SIZE = 500+PERCENTILES = (50, 90, 99)+++class Coverage(StrEnum):+ """Which sources still have units after the filters and caps."""++ TASKS_ONLY = "tasks_only"+ ACTIVITY_ONLY = "activity_only"+ BOTH = "both"+ NEITHER = "neither"+++class Variant(StrEnum):+ ALL = "all"+ WITHOUT_TASKS = "without_tasks"+ WITHOUT_ACTIVITY = "without_activity"+++@frozen+class PairCoverage:+ coverage: Coverage+ raw_tasks: int+ raw_activity: int+ task_units: int+ activity_units: int+ task_chars: int+ activity_chars: int+ task_lines_capped: bool+ activity_lines_capped: bool+ sections_budget_capped: bool+ drops: dict[str, int]++ @property+ def units(self) -> int:+ return self.task_units + self.activity_units+++@frozen+class CoverageReport:+ pairs: int+ coverage: dict[str, int]+ # Per metric, the value at each of `PERCENTILES`.+ percentiles: dict[str, dict[str, int]]+ caps: dict[str, int]+ drops: dict[str, int]+ min_units: int+ pass_min_units: int++ def as_dict(self) -> dict[str, Any]:+ return {+ "pairs": self.pairs,+ "coverage": self.coverage,+ "percentiles": self.percentiles,+ "caps": self.caps,+ "drops": self.drops,+ "min_units": self.min_units,+ "pass_min_units": self.pass_min_units,+ }+++def _classify(rendered: RenderedInput) -> Coverage:+ tasks = rendered.n_tasks > 0+ activity = rendered.n_activity_lines > 0+ if tasks and activity:+ return Coverage.BOTH+ if tasks:+ return Coverage.TASKS_ONLY+ if activity:+ return Coverage.ACTIVITY_ONLY+ return Coverage.NEITHER+++def measure_pair(+ tasks: Sequence[TaskItem], activity: Sequence[ActivityItem], *, as_of: datetime.datetime, seed: int+) -> PairCoverage:+ rendered = build_input(tasks, activity, as_of=as_of, seed=seed)+ if rendered is None:+ return PairCoverage(+ coverage=Coverage.NEITHER,+ raw_tasks=len(tasks),+ raw_activity=len(activity),+ task_units=0,+ activity_units=0,+ task_chars=0,+ activity_chars=0,+ task_lines_capped=False,+ activity_lines_capped=False,+ sections_budget_capped=False,+ drops={},+ )+ return PairCoverage(+ coverage=_classify(rendered),+ raw_tasks=len(tasks),+ raw_activity=len(activity),+ task_units=rendered.n_tasks,+ activity_units=rendered.n_activity_lines,+ task_chars=rendered.task_chars,+ activity_chars=rendered.activity_chars,+ task_lines_capped=rendered.task_pool > TASK_LINES,+ activity_lines_capped=rendered.activity_pool > ACTIVITY_LINES,+ sections_budget_capped=Drop.SECTIONS_BUDGET in rendered.drops,+ drops=rendered.drops,+ )+++def _percentile(sorted_values: Sequence[int], percent: int) -> int:+ """Nearest rank, so the value is one that a pair has."""+ if not sorted_values:+ return 0+ rank = max(1, -(-percent * len(sorted_values) // 100))+ return sorted_values[rank - 1]+++def coverage_report(pairs: Iterable[PairCoverage], *, min_units: int) -> CoverageReport:+ coverage: Counter[str] = Counter()+ caps: Counter[str] = Counter()+ drops: Counter[str] = Counter()+ metrics: dict[str, list[int]] = {+ "raw_tasks": [],+ "raw_activity": [],+ "task_units": [],+ "activity_units": [],+ "task_chars": [],+ "activity_chars": [],+ }+ count = 0+ passed = 0+ for pair in pairs:+ count += 1+ coverage[pair.coverage] += 1+ caps["task_lines"] += pair.task_lines_capped+ caps["activity_lines"] += pair.activity_lines_capped+ caps["sections_budget"] += pair.sections_budget_capped+ drops.update(pair.drops)+ passed += pair.units >= min_units+ for name, values in metrics.items():+ values.append(getattr(pair, name))+ return CoverageReport(+ pairs=count,+ coverage={str(kind): coverage[kind] for kind in Coverage},+ percentiles={+ name: {f"p{percent}": _percentile(sorted(values), percent) for percent in PERCENTILES}+ for name, values in metrics.items()+ },+ caps=dict(caps),+ drops=dict(sorted(drops.items())),+ min_units=min_units,+ pass_min_units=passed,+ )+++def scope_team_ids(organization_id: str | None, team_id: int | None) -> list[int]:+ teams = Team.objects.all()+ if organization_id is not None:+ teams = teams.filter(organization_id=organization_id)+ if team_id is not None:+ teams = teams.filter(id=team_id)+ return list(teams.order_by("id").values_list("id", flat=True))+++def candidate_pairs(+ team_ids: Sequence[int], start: datetime.datetime, end: datetime.datetime+) -> list[ActivitySummaryPair]:+ """Every (team, user) pair with a task or an activity log row of their own in [start, end).++ The same row filters as `inputs.py`. The render filters are not applied here, so a pair whose+ rows all drop counts as `Coverage.NEITHER`."""+ pairs: set[ActivitySummaryPair] = set()+ for offset in range(0, len(team_ids), TEAM_BATCH_SIZE):+ batch = list(team_ids[offset : offset + TEAM_BATCH_SIZE])+ rows = (+ ActivityLog.objects.filter(+ team_id__in=batch, user_id__isnull=False, created_at__gte=start, created_at__lt=end+ )+ .exclude(is_system=True)+ .exclude(was_impersonated=True)+ .values_list("team_id", "user_id")+ .distinct()+ )+ pairs.update(ActivitySummaryPair(team_id=team, user_id=user) for team, user in rows if team is not None)+ pairs.update(+ ActivitySummaryPair(team_id=task.team_id, user_id=task.user_id)+ for task in list_user_task_times(team_ids=batch, start=start, end=end, origin_products=TASK_ORIGIN_PRODUCTS)+ )+ return sorted(pairs)+++def sample_pairs(+ pairs: Sequence[ActivitySummaryPair], sample: int | None, seed: int | None+) -> list[ActivitySummaryPair]:+ if sample is None or sample >= len(pairs):+ return list(pairs)+ return sorted(random.Random(seed).sample(list(pairs), sample))+++def measure_coverage(+ pairs: Iterable[ActivitySummaryPair], *, start: datetime.datetime, now: datetime.datetime+) -> Iterable[PairCoverage]:+ """One pair at a time, so memory holds the rows of one pair only."""+ for pair in pairs:+ yield measure_pair(+ user_tasks(pair.team_id, pair.user_id, start, now),+ user_activity(pair.team_id, pair.user_id, start, now),+ as_of=now,+ seed=sampling_seed(now.date().isoformat(), pair.team_id, pair.user_id),+ )+++def leave_one_out_inputs(+ tasks: Sequence[TaskItem], activity: Sequence[ActivityItem], *, as_of: datetime.datetime, seed: int+) -> dict[Variant, RenderedInput]:+ """The full input, and one input per source that has units in it. A variant with no units left is omitted."""+ full = build_input(tasks, activity, as_of=as_of, seed=seed)+ if full is None:+ return {}+ variants = {Variant.ALL: full}+ if full.n_tasks:+ without_tasks = build_input([], activity, as_of=as_of, seed=seed)+ if without_tasks is not None:+ variants[Variant.WITHOUT_TASKS] = without_tasks+ if full.n_activity_lines:+ without_activity = build_input(tasks, [], as_of=as_of, seed=seed)+ if without_activity is not None:+ variants[Variant.WITHOUT_ACTIVITY] = without_activity+ return variants+++def leave_one_out_pair(+ pair: ActivitySummaryPair, *, start: datetime.datetime, now: datetime.datetime, min_units: int+) -> list[dict[str, Any]]:+ """One record per variant. With no previous summary, so each summary reflects its input only."""+ variants = leave_one_out_inputs(+ user_tasks(pair.team_id, pair.user_id, start, now),+ user_activity(pair.team_id, pair.user_id, start, now),+ as_of=now,+ seed=sampling_seed(now.date().isoformat(), pair.team_id, pair.user_id),+ )+ if Variant.ALL not in variants or variants[Variant.ALL].units < min_units:+ return []+ client = gateway_client(pair.team_id, pair.user_id, now.date())+ records: list[dict[str, Any]] = []+ for variant, rendered in variants.items():+ outcome = summarize(+ client, rendered_input=rendered.text, previous_summary=None, model=DEFAULT_MODEL, as_of=now.date()+ )+ records.append(+ {+ "team_id": pair.team_id,+ "user_id": pair.user_id,+ "variant": str(variant),+ "tasks": rendered.n_tasks,+ "activity_lines": rendered.n_activity_lines,+ "input_chars": rendered.input_chars,+ "tokens_in": outcome.tokens_in,+ "tokens_out": outcome.tokens_out,+ "cost_usd": cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(DEFAULT_MODEL)),+ "latency_ms": outcome.latency_ms,+ "error": outcome.error,+ "summary": outcome.summary,+ }+ )+ return records+++def leave_one_out_totals(records: Iterable[dict[str, Any]]) -> dict[str, dict[str, float]]:+ totals: dict[str, dict[str, float]] = {}+ for record in records:+ total = totals.setdefault(+ record["variant"], {"calls": 0, "ok": 0, "tokens_in": 0, "tokens_out": 0, "cost_usd": 0.0}+ )+ total["calls"] += 1+ total["ok"] += record["summary"] is not None+ total["tokens_in"] += record["tokens_in"] or 0+ total["tokens_out"] += record["tokens_out"] or 0+ total["cost_usd"] += record["cost_usd"] or 0.0+ return totalsdiff --git a/products/signals/backend/activity_summary/render.py b/products/signals/backend/activity_summary/render.pyindex bb29f45c05ce..6d036b20ef97 100644--- a/products/signals/backend/activity_summary/render.py+++ b/products/signals/backend/activity_summary/render.py@@ -139,6 +139,9 @@ class RenderedInput:input_chars: intinput_tokens_estimate: intdrops: dict[str, int]+ # Units of each source before its line cap, so a caller can tell when the cap binds.+ task_pool: int+ activity_pool: int@propertydef units(self) -> int:@@ -371,6 +374,8 @@ def build_input(drops: Counter[str] = Counter()task_pool = burst_cap(dedupe_latest(task_units(tasks, drops), drops), drops)activity_lines = activity_units(activity, drops)+ # Only `activity_units` has counted a sample cap drop so far.+ activity_pool = len(activity_lines) + drops[Drop.SAMPLE_CAP]if not task_pool and not activity_lines:return Nonechosen_tasks = sample_units(task_pool, limit=TASK_LINES, as_of=as_of, seed=seed)@@ -388,4 +393,6 @@ def build_input(input_chars=len(text),input_tokens_estimate=estimate_tokens(len(text)),drops={str(reason): count for reason, count in sorted(drops.items()) if count},+ task_pool=len(task_pool),+ activity_pool=activity_pool,)diff --git a/products/signals/backend/management/commands/measure_activity_summary_sources.py b/products/signals/backend/management/commands/measure_activity_summary_sources.pynew file mode 100644index 000000000000..0f9c7277cff7--- /dev/null+++ b/products/signals/backend/management/commands/measure_activity_summary_sources.py@@ -0,0 +1,83 @@+"""Measure the coverage and the signal of each activity summary input source. Writes no summary rows.++`--mode coverage` prints aggregate counts only. `--mode leave-one-out` calls the model and writes the+summaries to a local JSONL file, so it runs only for one organization that approved AI data processing.+"""++import json+import datetime+from pathlib import Path++from django.core.management.base import BaseCommand, CommandError+from django.utils import timezone++from posthog.models import Organization++from products.signals.backend.activity_summary.logic import MIN_UNITS+from products.signals.backend.activity_summary.measure import (+ candidate_pairs,+ coverage_report,+ leave_one_out_pair,+ leave_one_out_totals,+ measure_coverage,+ sample_pairs,+ scope_team_ids,+)+from products.signals.backend.activity_summary.render import WINDOW_DAYS++MODE_COVERAGE = "coverage"+MODE_LEAVE_ONE_OUT = "leave-one-out"+# Bounds the model calls of a leave-one-out run: up to three per pair.+DEFAULT_LEAVE_ONE_OUT_SAMPLE = 20+++class Command(BaseCommand):+ help = "Measure how much the tasks and the activity log sources contribute to activity summaries."++ def add_arguments(self, parser):+ parser.add_argument("--mode", choices=[MODE_COVERAGE, MODE_LEAVE_ONE_OUT], default=MODE_COVERAGE)+ parser.add_argument("--organization-id", type=str, help="Required for leave-one-out.")+ parser.add_argument("--team-id", type=int)+ parser.add_argument("--sample", type=int, help="Measure a random sample of this many pairs.")+ parser.add_argument("--seed", type=int, help="Seed of the pair sample, for a repeatable run.")+ parser.add_argument("--window-days", type=int, default=WINDOW_DAYS)+ parser.add_argument("--min-units", type=int, default=MIN_UNITS)+ parser.add_argument(+ "--output",+ type=Path,+ default=Path("activity_summary_leave_one_out.jsonl"),+ help="The local JSONL file of leave-one-out.",+ )++ def handle(self, *args, **options):+ mode = options["mode"]+ organization_id = options["organization_id"]+ if mode == MODE_LEAVE_ONE_OUT:+ if not organization_id:+ raise CommandError("leave-one-out sends real input to the model and needs --organization-id.")+ organization = Organization.objects.filter(id=organization_id).first()+ if organization is None or organization.is_ai_data_processing_approved is not True:+ raise CommandError("leave-one-out needs an organization that approved AI data processing.")++ now = timezone.now()+ start = now - datetime.timedelta(days=options["window_days"])+ team_ids = scope_team_ids(organization_id, options["team_id"])+ sample = options["sample"]+ if mode == MODE_LEAVE_ONE_OUT and sample is None:+ sample = DEFAULT_LEAVE_ONE_OUT_SAMPLE+ pairs = sample_pairs(candidate_pairs(team_ids, start, now), sample, options["seed"])+ self.stderr.write(f"teams: {len(team_ids)}, pairs: {len(pairs)}")++ if mode == MODE_COVERAGE:+ report = coverage_report(measure_coverage(pairs, start=start, now=now), min_units=options["min_units"])+ self.stdout.write(json.dumps(report.as_dict(), indent=2, sort_keys=True))+ return++ records = []+ with options["output"].open("w") as output:+ for pair in pairs:+ for record in leave_one_out_pair(pair, start=start, now=now, min_units=options["min_units"]):+ output.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")+ records.append(record)+ self.stderr.write(f"wrote {len(records)} records to {options['output']}")+ self.stdout.write(json.dumps(leave_one_out_totals(records), indent=2, sort_keys=True))diff --git a/products/signals/backend/test/test_activity_summary_measure.py b/products/signals/backend/test/test_activity_summary_measure.pynew file mode 100644index 000000000000..63b0d8e7a4da--- /dev/null+++ b/products/signals/backend/test/test_activity_summary_measure.py@@ -0,0 +1,128 @@+import json+import datetime+from io import StringIO+from pathlib import Path+from tempfile import TemporaryDirectory++import pytest+import time_machine+from posthog.test.base import BaseTest+from unittest.mock import patch++from django.core.management import CommandError, call_command++from parameterized import parameterized++from posthog.models import ActivityLog++from products.signals.backend.activity_summary.measure import Coverage, coverage_report, measure_pair+from products.signals.backend.activity_summary.render import ACTIVITY_LINES, ActivityItem, TaskItem+from products.signals.backend.models import UserActivitySummary+from products.signals.backend.test.test_activity_summary_render import FakeLLM++NOW = datetime.datetime(2026, 10, 19, 12, tzinfo=datetime.UTC)+ANSWER = json.dumps(+ {+ "summary": "Builds the signup funnel.",+ "focus_areas": [{"topic": "signup funnel", "strength": "high", "last_seen": "2026-10-18"}],+ "products": ["product_analytics"],+ "recent_shift": None,+ }+)+TASK = TaskItem(+ at=NOW - datetime.timedelta(days=1), title="why", summary="Goal: find why weekly signups dropped.", tags=()+)+++def _change(name: str, client: str | None = None) -> ActivityItem:+ return ActivityItem(+ at=NOW - datetime.timedelta(days=1), scope="Insight", activity="created", name=name, client=client+ )+++ [+ ("tasks only", [TASK], [], Coverage.TASKS_ONLY, False),+ ("activity only", [], [_change("Signup funnel")], Coverage.ACTIVITY_ONLY, False),+ ("both", [TASK], [_change("Signup funnel")], Coverage.BOTH, False),+ ("neither once the filters drop every row", [], [_change("Scout funnel", "scout:x")], Coverage.NEITHER, False),+ (+ "the activity line cap binds",+ [],+ [_change(f"Funnel {index}") for index in range(ACTIVITY_LINES + 1)],+ Coverage.ACTIVITY_ONLY,+ True,+ ),+ ]+)+def test_measure_pair_classifies_coverage(+ _name: str, tasks: list[TaskItem], activity: list[ActivityItem], coverage: Coverage, capped: bool+) -> None:+ pair = measure_pair(tasks, activity, as_of=NOW, seed=1)++ assert (pair.coverage, pair.activity_lines_capped) == (coverage, capped)+ report = coverage_report([pair], min_units=1)+ assert report.coverage[coverage] == 1+ assert report.caps["activity_lines"] == int(capped)+++@time_machine.travel(NOW, tick=False)+class TestMeasureActivitySummarySources(BaseTest):+ def _log(self, name: str) -> None:+ ActivityLog.objects.create(+ team_id=self.team.id,+ organization_id=self.organization.id,+ user=self.user,+ scope="Insight",+ activity="created",+ item_id=name,+ detail={"name": name},+ created_at=NOW - datetime.timedelta(days=1),+ )++ def _run(self, *args: str) -> str:+ out = StringIO()+ with patch("products.signals.backend.activity_summary.measure.list_user_task_times", return_value=[]):+ call_command("measure_activity_summary_sources", *args, stdout=out, stderr=StringIO())+ return out.getvalue()++ def test_coverage_prints_counts_without_rendered_text(self) -> None:+ self._log("Signup funnel")++ with patch("products.signals.backend.activity_summary.measure.user_tasks", return_value=[TASK]):+ out = self._run()++ assert "Signup funnel" not in out and "signups" not in out+ report = json.loads(out)+ assert (report["pairs"], report["coverage"][Coverage.BOTH]) == (1, 1)++ def test_leave_one_out_refuses_without_an_organization(self) -> None:+ with pytest.raises(CommandError):+ self._run("--mode", "leave-one-out")++ def test_leave_one_out_builds_one_variant_per_source_and_writes_no_summary(self) -> None:+ for index in range(3):+ self._log(f"Funnel {index}")+ llm = FakeLLM([ANSWER])++ with (+ TemporaryDirectory() as directory,+ patch("products.signals.backend.activity_summary.measure.user_tasks", return_value=[TASK]),+ patch("products.signals.backend.activity_summary.measure.gateway_client", return_value=llm),+ ):+ output = Path(directory) / "out.jsonl"+ totals = json.loads(+ self._run(+ "--mode", "leave-one-out", "--organization-id", str(self.organization.id), "--output", str(output)+ )+ )+ records = [json.loads(line) for line in output.read_text().splitlines()]++ assert [(r["variant"], r["tasks"], r["activity_lines"]) for r in records] == [+ ("all", 1, 3),+ ("without_tasks", 0, 3),+ ("without_activity", 1, 0),+ ]+ assert len(llm.requests) == 3+ assert set(totals) == {"all", "without_tasks", "without_activity"}+ assert not UserActivitySummary.objects.for_team(self.team.id).exists()