:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): backfill activity summaries as of past dates (#124481)
posthog[bot]6766379d4a28
diff --git a/products/signals/backend/activity_summary/backfill.py b/products/signals/backend/activity_summary/backfill.pynew file mode 100644index 000000000000..9179c22ec80d--- /dev/null+++ b/products/signals/backend/activity_summary/backfill.py@@ -0,0 +1,248 @@+"""Activity summaries built as of past dates, for the offline evaluation of activity summary vectors.++A backfill build reads only the inputs before its as-of moment, and writes only an embedding under its+own document type and an event. It never touches `UserActivitySummary`, its versions or the queue state.+"""++import datetime+from collections import Counter+from collections.abc import Callable, Iterable, Sequence+from concurrent.futures import ThreadPoolExecutor++from django.db import connection++import structlog++from posthog.dataclasses import frozen+from posthog.models import Team++from products.signals.backend.activity_summary.logic import (+ MIN_UNITS,+ ActivitySummaryUpdate,+ UpdateStatus,+ build_summary_as_of,+ render_as_of,+)+from products.signals.backend.activity_summary.prompt import PROMPT_VERSION, summary_text+from products.signals.backend.activity_summary.queue import max_update_cost_usd+from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, RENDER_VERSION, Product, RenderLimits+from products.signals.backend.activity_summary.summaries import DEFAULT_MODEL+from products.signals.backend.activity_summary.tick import capture_generated, emit_summary_embedding+from products.signals.backend.models import SignalReportAction++logger = structlog.get_logger(__name__)++# Separate from the live `activity_summary` type, so the live vectors stay clean.+BACKFILL_DOCUMENT_TYPE = "activity_summary_backfill"+SOURCE_BACKFILL = "backfill"+# The document id suffix of a build without inbox inputs, so both variants of one pair and day coexist.+EXCLUDE_INBOX_SUFFIX = ":no_inbox"+INBOX_PRODUCTS: frozenset[Product] = frozenset({Product.INBOX})+# The status of a build that raised. It can have called the model, so it counts at the highest cost.+STATUS_EXCEPTION = "exception"+++@frozen(order=True)+class TeamUser:+ team_id: int+ user_id: int+++@frozen+class BackfillJob:+ team_id: int+ user_id: int+ as_of: datetime.datetime+++@frozen+class BackfillOptions:+ limits: RenderLimits = DEFAULT_LIMITS+ min_units: int = MIN_UNITS+ exclude_inbox: bool = False+++@frozen+class BackfillPlan:+ jobs: int+ # Jobs whose rendered input clears `min_units`, so a real run would call the model for each.+ builds: int+ max_cost_usd: float+++@frozen+class BackfillResult:+ jobs: int+ attempted: int+ statuses: dict[str, int]+ cost_usd: float+ stopped_at_cost_cap: bool+++def eligible_team_ids(team_ids: Iterable[int]) -> set[int]:+ """The teams whose organization approved AI data processing and opted in to AI training."""+ return set(+ Team.objects.filter(+ id__in=list(team_ids),+ organization__is_ai_data_processing_approved=True,+ organization__is_ai_training_opted_in=True,+ ).values_list("id", flat=True)+ )+++def inbox_users(as_of: datetime.datetime) -> set[TeamUser]:+ """Every (team, user) with an inbox report action that started before `as_of`."""+ return {+ TeamUser(team_id=team_id, user_id=user_id)+ for team_id, user_id in (+ # nosemgrep: idor-lookup-without-team (an operator backfill over all teams, with no user input)+ SignalReportAction.all_teams.filter(first_at__lt=as_of).values_list("team_id", "user_id").distinct()+ )+ }+++def backfill_jobs(+ as_of_dates: Sequence[datetime.datetime],+ *,+ pair: TeamUser | None = None,+) -> list[BackfillJob]:+ """One job per as-of date and pair, for `pair` or for the inbox users before that date, on eligible teams only."""+ jobs: list[BackfillJob] = []+ for as_of in as_of_dates:+ pairs = {pair} if pair is not None else inbox_users(as_of)+ eligible = eligible_team_ids({p.team_id for p in pairs})+ jobs.extend(+ BackfillJob(team_id=p.team_id, user_id=p.user_id, as_of=as_of)+ for p in sorted(pairs)+ if p.team_id in eligible+ )+ return jobs+++def _excluded_products(options: BackfillOptions) -> frozenset[Product]:+ return INBOX_PRODUCTS if options.exclude_inbox else frozenset()+++def document_id(job: BackfillJob, exclude_inbox: bool) -> str:+ base = f"{job.user_id}:{job.as_of.date().isoformat()}"+ return f"{base}{EXCLUDE_INBOX_SUFFIX}" if exclude_inbox else base+++def _emit_embedding(job: BackfillJob, update: ActivitySummaryUpdate, exclude_inbox: bool) -> None:+ if update.summary is None:+ return+ emit_summary_embedding(+ job.team_id,+ content=summary_text(update.summary),+ document_type=BACKFILL_DOCUMENT_TYPE,+ document_id=document_id(job, exclude_inbox),+ # The as-of moment, so a rerun of the same job replaces its own row.+ timestamp=job.as_of,+ metadata={+ "prompt_version": PROMPT_VERSION,+ "render_version": RENDER_VERSION,+ "model": DEFAULT_MODEL,+ "source": SOURCE_BACKFILL,+ "exclude_inbox": exclude_inbox,+ },+ )+++def run_job(job: BackfillJob, options: BackfillOptions) -> ActivitySummaryUpdate:+ """Build one summary as of the job date, then emit its embedding and its event."""+ update = build_summary_as_of(+ job.team_id,+ job.user_id,+ job.as_of,+ limits=options.limits,+ min_units=options.min_units,+ exclude_products=_excluded_products(options),+ trace_variant=f"{SOURCE_BACKFILL}:{document_id(job, options.exclude_inbox)}",+ )+ if update.status == UpdateStatus.OK:+ try:+ _emit_embedding(job, update, options.exclude_inbox)+ except Exception:+ logger.exception("activity_summary_backfill_embedding_failed", team_id=job.team_id, user_id=job.user_id)+ if update.status in (UpdateStatus.OK, UpdateStatus.FAILED):+ try:+ capture_generated(job.team_id, job.user_id, update, None, source=SOURCE_BACKFILL)+ except Exception:+ logger.exception("activity_summary_backfill_capture_failed", team_id=job.team_id, user_id=job.user_id)+ return update+++def plan_backfill(jobs: Sequence[BackfillJob], options: BackfillOptions) -> BackfillPlan:+ """Count the jobs that would call the model, without a model call."""+ builds = 0+ for job in jobs:+ built = render_as_of(+ job.team_id, job.user_id, job.as_of, limits=options.limits, exclude_products=_excluded_products(options)+ )+ builds += built is not None and built.rendered.units >= options.min_units+ return BackfillPlan(jobs=len(jobs), builds=builds, max_cost_usd=builds * max_update_cost_usd(options.limits))+++def _run_job_in_thread(job: BackfillJob, options: BackfillOptions) -> ActivitySummaryUpdate:+ try:+ return run_job(job, options)+ finally:+ # Each worker thread opens its own connection, and nothing else closes it.+ connection.close()+++def _guarded(+ run: Callable[[BackfillJob, BackfillOptions], ActivitySummaryUpdate], job: BackfillJob, options: BackfillOptions+) -> ActivitySummaryUpdate | None:+ try:+ return run(job, options)+ except Exception:+ logger.exception("activity_summary_backfill_job_failed", team_id=job.team_id, user_id=job.user_id)+ return None+++def run_backfill(+ jobs: Sequence[BackfillJob],+ options: BackfillOptions,+ *,+ concurrency: int = 1,+ max_cost_usd: float,+ on_batch: Callable[[int, float], None] | None = None,+) -> BackfillResult:+ """Run the jobs in batches of `concurrency`, and stop between batches when the cost reaches `max_cost_usd`.++ A build without a cost, or a build that raised, counts at the highest cost of one update."""+ bound = max_update_cost_usd(options.limits)+ statuses: Counter[str] = Counter()+ cost = 0.0+ attempted = 0+ batch_size = max(1, concurrency)+ executor = ThreadPoolExecutor(max_workers=batch_size) if batch_size > 1 else None+ try:+ while attempted < len(jobs) and cost < max_cost_usd:+ batch = jobs[attempted : attempted + batch_size]+ attempted += len(batch)+ if executor is None:+ outcomes = [_guarded(run_job, job, options) for job in batch]+ else:+ outcomes = list(executor.map(lambda job: _guarded(_run_job_in_thread, job, options), batch))+ for update in outcomes:+ if update is None:+ statuses[STATUS_EXCEPTION] += 1+ cost += bound+ continue+ statuses[update.status.value] += 1+ if update.status in (UpdateStatus.OK, UpdateStatus.FAILED):+ cost += bound if update.cost_usd is None else update.cost_usd+ if on_batch is not None:+ on_batch(attempted, cost)+ finally:+ if executor is not None:+ executor.shutdown()+ return BackfillResult(+ jobs=len(jobs),+ attempted=attempted,+ statuses=dict(statuses),+ cost_usd=cost,+ stopped_at_cost_cap=attempted < len(jobs),+ )diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.pyindex c3b859908d7f..4d115f781568 100644--- a/products/signals/backend/activity_summary/logic.py+++ b/products/signals/backend/activity_summary/logic.py@@ -1,7 +1,10 @@-"""One activity summary update for one (team, user): gather, filter, render, summarize, persist."""+"""One activity summary update for one (team, user): gather, filter, render, summarize, persist.++`build_summary_as_of` runs the same steps as of a past moment, and persists nothing."""import datetimeimport dataclasses+from collections import Counterfrom enum import StrEnumfrom typing import Any, Protocol@@ -19,13 +22,20 @@ from products.signals.backend.activity_summary.prompt import PROMPT_VERSION, sumfrom products.signals.backend.activity_summary.render import (DEFAULT_LIMITS,RENDER_VERSION,+ ActivityItem,+ Product,+ RenderedInput,RenderLimits,+ TaskItem,+ ViewItem,build_input,sampling_seed,+ without_products,)from products.signals.backend.activity_summary.summaries import (DEFAULT_MODEL,MODEL_PRICES,+ SummaryOutcome,cost_usd,gateway_client,summarize,@@ -125,6 +135,77 @@ def _input_start(watermark: datetime.datetime | None, now: datetime.datetime, wireturn earliest if watermark is None else max(earliest, watermark + _WATERMARK_STEP)+@frozen+class _WindowInputs:+ tasks: list[TaskItem]+ activity: list[ActivityItem]+ views: list[ViewItem]+++def _read_inputs(team_id: int, user_id: int, start: datetime.datetime, end: datetime.datetime) -> _WindowInputs:+ return _WindowInputs(+ tasks=user_tasks(team_id, user_id, start, end),+ activity=user_activity(team_id, user_id, start, end),+ views=user_views(team_id, user_id, start, end),+ )+++def _render(+ team_id: int,+ user_id: int,+ tasks: list[TaskItem],+ activity: list[ActivityItem],+ views: list[ViewItem],+ *,+ as_of: datetime.datetime,+ limits: RenderLimits,+) -> RenderedInput | None:+ return build_input(+ tasks,+ activity,+ as_of=as_of,+ seed=sampling_seed(as_of.date().isoformat(), team_id, user_id),+ views=views,+ limits=limits,+ parents=activity_parents(team_id, activity),+ )+++def _inputs_used(rendered: RenderedInput, limits: RenderLimits) -> dict[str, Any]:+ return {+ "tasks": rendered.n_tasks,+ "activity_lines": rendered.n_activity_lines,+ "view_lines": rendered.n_view_lines,+ "input_chars": rendered.input_chars,+ "drops": rendered.drops,+ "limits": dataclasses.asdict(limits),+ }+++def _call_model(+ team_id: int,+ user_id: int,+ rendered: RenderedInput,+ computed: dict[str, Any],+ *,+ previous: dict[str, Any] | None,+ as_of: datetime.datetime,+ limits: RenderLimits,+ trace_variant: str | None = None,+) -> tuple[SummaryOutcome, float | None]:+ outcome = summarize(+ gateway_client(team_id, user_id, as_of.date(), variant=trace_variant),+ rendered_input=rendered.text,+ previous_summary=previous,+ model=DEFAULT_MODEL,+ as_of=as_of.date(),+ max_completion_tokens=limits.max_completion_tokens,+ profile_max_chars=limits.profile_max_chars,+ computed=computed,+ )+ return outcome, cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(DEFAULT_MODEL))++def _back_off(team_id: int,user_id: int,@@ -277,18 +358,9 @@ def update_activity_summary(rebuild = is_stale(stored, rebuild_before)window_start = _input_start(None, now, limits.window_days)start = _input_start(stored.input_watermark if stored and not rebuild else None, now, limits.window_days)- tasks = user_tasks(team_id, user_id, start, now)- activity = user_activity(team_id, user_id, start, now)- views = user_views(team_id, user_id, start, now)- rendered = build_input(- tasks,- activity,- as_of=now,- seed=sampling_seed(now.date().isoformat(), team_id, user_id),- views=views,- limits=limits,- parents=activity_parents(team_id, activity),- )+ window = _read_inputs(team_id, user_id, start, now)+ tasks, activity, views = window.tasks, window.activity, window.views+ rendered = _render(team_id, user_id, tasks, activity, views, as_of=now, limits=limits)if rendered is None or rendered.units < min_units:if not dry_run:_back_off(team_id, user_id, stored, now, max_skip_backoff)@@ -298,37 +370,16 @@ def update_activity_summary(inputs_used={"drops": rendered.drops} if rendered else None,)- inputs_used: dict[str, Any] = {- "tasks": rendered.n_tasks,- "activity_lines": rendered.n_activity_lines,- "view_lines": rendered.n_view_lines,- "input_chars": rendered.input_chars,- "drops": rendered.drops,- "limits": dataclasses.asdict(limits),- }+ inputs_used = _inputs_used(rendered, limits)watermark = max([task.at for task in tasks] + [item.at for item in activity] + [view.at for view in views])# An incremental update reads only the rows after the watermark, so the computed fields read the full window.if start == window_start:computed = compute_fields(tasks, activity, views).as_json()else:- computed = compute_fields(- user_tasks(team_id, user_id, window_start, now),- user_activity(team_id, user_id, window_start, now),- user_views(team_id, user_id, window_start, now),- ).as_json()+ full = _read_inputs(team_id, user_id, window_start, now)+ computed = compute_fields(full.tasks, full.activity, full.views).as_json()previous = stored.summary if stored is not None and not rebuild else None- model = DEFAULT_MODEL- outcome = summarize(- gateway_client(team_id, user_id, now.date()),- rendered_input=rendered.text,- previous_summary=previous,- model=model,- as_of=now.date(),- max_completion_tokens=limits.max_completion_tokens,- profile_max_chars=limits.profile_max_chars,- computed=computed,- )- cost = cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(model))+ outcome, cost = _call_model(team_id, user_id, rendered, computed, previous=previous, as_of=now, limits=limits)if outcome.summary is not None:status = UpdateStatus.OKelif outcome.empty_profile:@@ -399,7 +450,7 @@ def update_activity_summary("summary_text": summary_text(outcome.summary),"computed": computed,"generated_at": now,- "model": model,+ "model": DEFAULT_MODEL,"prompt_version": PROMPT_VERSION,"render_version": RENDER_VERSION,"input_watermark": watermark,@@ -411,3 +462,79 @@ def update_activity_summary(return resultexcept Exception as error:raise UpdateAfterModelCallError(cost) from error+++@frozen+class AsOfInput:+ rendered: RenderedInput+ computed: dict[str, Any]+ inputs_used: dict[str, Any]+++def render_as_of(+ team_id: int,+ user_id: int,+ as_of: datetime.datetime,+ *,+ limits: RenderLimits = DEFAULT_LIMITS,+ exclude_products: frozenset[Product] = frozenset(),+) -> AsOfInput | None:+ """The rendered input and computed fields of `[as_of - limits.window_days, as_of)`, or None when nothing renders.++ Views come from the latest view of each object, so an object opened again after `as_of` is missing."""+ window = _read_inputs(team_id, user_id, as_of - datetime.timedelta(days=limits.window_days), as_of)+ tasks, activity, views = window.tasks, window.activity, window.views+ excluded: Counter[str] = Counter()+ activity, views = without_products(activity, views, exclude_products, excluded)+ rendered = _render(team_id, user_id, tasks, activity, views, as_of=as_of, limits=limits)+ if rendered is None:+ return None+ inputs_used = _inputs_used(rendered, limits)+ inputs_used["drops"] = {**rendered.drops, **excluded}+ return AsOfInput(+ rendered=rendered, computed=compute_fields(tasks, activity, views).as_json(), inputs_used=inputs_used+ )+++def build_summary_as_of(+ team_id: int,+ user_id: int,+ as_of: datetime.datetime,+ *,+ limits: RenderLimits = DEFAULT_LIMITS,+ min_units: int = MIN_UNITS,+ exclude_products: frozenset[Product] = frozenset(),+ trace_variant: str | None = None,+) -> ActivitySummaryUpdate:+ """A summary of one user in one project as of `as_of`, from the inputs before it only.++ The model gets no previous summary, and the sample seed comes from the `as_of` date. Nothing is written."""+ built = render_as_of(team_id, user_id, as_of, limits=limits, exclude_products=exclude_products)+ if built is None or built.rendered.units < min_units:+ return ActivitySummaryUpdate(+ status=UpdateStatus.SKIPPED_TOO_FEW_UNITS,+ rendered_input=built.rendered.text if built else None,+ inputs_used=built.inputs_used if built else None,+ )+ outcome, cost = _call_model(+ team_id,+ user_id,+ built.rendered,+ built.computed,+ previous=None,+ as_of=as_of,+ limits=limits,+ trace_variant=trace_variant,+ )+ return ActivitySummaryUpdate(+ status=UpdateStatus.OK if outcome.summary is not None else UpdateStatus.FAILED,+ rendered_input=built.rendered.text,+ summary=outcome.summary,+ tokens_in=outcome.tokens_in,+ tokens_out=outcome.tokens_out,+ cost_usd=cost,+ latency_ms=outcome.latency_ms,+ error=outcome.error,+ inputs_used=built.inputs_used,+ computed=built.computed if outcome.summary is not None else None,+ )diff --git a/products/signals/backend/activity_summary/render.py b/products/signals/backend/activity_summary/render.pyindex 677f07a75bb3..828603a20ef7 100644--- a/products/signals/backend/activity_summary/render.py+++ b/products/signals/backend/activity_summary/render.py@@ -267,6 +267,7 @@ class Drop(StrEnum):BURST_CAP = "burst_cap"SAMPLE_CAP = "sample_cap"SECTIONS_BUDGET = "sections_budget"+ EXCLUDED_PRODUCT = "excluded_product"@frozen@@ -842,6 +843,34 @@ def scope_of(view_type: str) -> str:return "".join(word.capitalize() for word in view_type.split("/")[0].split("_"))+def view_product(view: ViewItem) -> Product | None:+ if view.type == SCENE_VIEW_TYPE:+ return scene_product(view.ref)+ return scope_product(scope_of(view.type))+++def without_products(+ activity: Iterable[ActivityItem],+ views: Iterable[ViewItem],+ products: frozenset[Product],+ drops: Counter[str],+) -> tuple[list[ActivityItem], list[ViewItem]]:+ """The activity rows and views whose product is not in `products`. Tasks have no product, so they stay."""+ kept_activity: list[ActivityItem] = []+ for item in activity:+ if scope_product(item.scope) in products:+ drops[Drop.EXCLUDED_PRODUCT] += 1+ else:+ kept_activity.append(item)+ kept_views: list[ViewItem] = []+ for view in views:+ if view_product(view) in products:+ drops[Drop.EXCLUDED_PRODUCT] += 1+ else:+ kept_views.append(view)+ return kept_activity, kept_views++def _days(count: int) -> str:return f"{count} day" if count == 1 else f"{count} days"diff --git a/products/signals/backend/activity_summary/summaries.py b/products/signals/backend/activity_summary/summaries.pyindex 5b5f4b4a6371..6a07c246d301 100644--- a/products/signals/backend/activity_summary/summaries.py+++ b/products/signals/backend/activity_summary/summaries.py@@ -69,11 +69,13 @@ class SummaryOutcome:empty_profile: bool = False-def gateway_client(team_id: int, user_id: int, as_of: datetime.date) -> OpenAI:+def gateway_client(team_id: int, user_id: int, as_of: datetime.date, *, variant: str | None = None) -> OpenAI:+ """`variant` keeps the trace of another build of the same pair and day, such as a backfill, apart."""+ seed = f"{AI_PRODUCT}:{team_id}:{user_id}:{as_of.isoformat()}"return build_openai_client("signals",ai_product=AI_PRODUCT,- trace_id=trace_id_from_seed(f"{AI_PRODUCT}:{team_id}:{user_id}:{as_of.isoformat()}"),+ trace_id=trace_id_from_seed(f"{seed}:{variant}" if variant else seed),properties={"team_id": str(team_id)},distinct_id=PIPELINE_DISTINCT_ID,)diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.pyindex 3a0f8933ff85..90c19e40bee7 100644--- a/products/signals/backend/activity_summary/tick.py+++ b/products/signals/backend/activity_summary/tick.py@@ -8,6 +8,7 @@ import asyncioimport datetimeimport dataclassesfrom collections import Counter+from typing import Anyfrom django.conf import settingsfrom django.utils import timezone@@ -62,6 +63,8 @@ logger = structlog.get_logger(__name__)TICK_WORKFLOW_NAME = "activity-summary-tick"EMBEDDING_DOCUMENT_TYPE = "activity_summary"EMBEDDING_RENDERING = "summary_text"+# Where a summary came from: the tick, or a build as of a past date.+SOURCE_LIVE = "live"# Stop reasons of a tick. Null when the tick updated every due pair.STOP_MAX_PAIRS_PER_TICK = "max_pairs_per_tick"@@ -229,23 +232,42 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP)-def _emit_embedding(team_id: int, user_id: int) -> None:+def emit_summary_embedding(+ team_id: int,+ *,+ content: str,+ document_type: str,+ document_id: str,+ timestamp: datetime.datetime,+ metadata: dict[str, Any],+) -> None:# Deferred so importing the tick does not load the Kafka and ClickHouse chain behind the producer.from posthog.api.embedding_worker import emit_embedding_request # noqa: PLC0415+ emit_embedding_request(+ content=content,+ team_id=team_id,+ product=EMBEDDING_PRODUCT,+ document_type=document_type,+ rendering=EMBEDDING_RENDERING,+ document_id=document_id,+ # The models of the report embeddings, so both share one vector space.+ models=[model.value for model in EmbeddingModelName],+ timestamp=timestamp,+ metadata=metadata,+ )+++def _emit_embedding(team_id: int, user_id: int) -> None:stored = UserActivitySummary.objects.for_team(team_id).filter(user_id=user_id).first()if stored is None or not stored.summary_text or stored.generated_at is None:return- emit_embedding_request(+ emit_summary_embedding(+ team_id,content=stored.summary_text,- team_id=team_id,- product=EMBEDDING_PRODUCT,document_type=EMBEDDING_DOCUMENT_TYPE,- rendering=EMBEDDING_RENDERING,# One vector per user per day, so an analysis can read the vector a user had on a given day.document_id=f"{user_id}:{stored.generated_at.date().isoformat()}",- # The models of the report embeddings, so both share one vector space.- models=[model.value for model in EmbeddingModelName],timestamp=stored.generated_at,metadata={"prompt_version": stored.prompt_version,@@ -255,7 +277,9 @@ def _emit_embedding(team_id: int, user_id: int) -> None:)-def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate, trigger: str | None) -> None:+def capture_generated(+ team_id: int, user_id: int, update: ActivitySummaryUpdate, trigger: str | None, *, source: str = SOURCE_LIVE+) -> None:team = Team.objects.only("uuid", "organization_id").get(id=team_id)# Counts and accounting only. The summary and the rendered input are user content.ph_background_capture()(@@ -277,6 +301,7 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate"inputs_used": update.inputs_used,"rebuilt": update.rebuilt,"trigger": trigger,+ "source": source,},groups={"organization": str(team.organization_id), "project": str(team.uuid)},)@@ -327,7 +352,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> Alogger.exception("activity_summary_embedding_failed", team_id=input.team_id, user_id=input.user_id)if update.status in CALLED_MODEL_STATUSES and input.capture_events:try:- _capture_generated(input.team_id, input.user_id, update, input.trigger)+ capture_generated(input.team_id, input.user_id, update, input.trigger)except Exception:logger.exception("activity_summary_capture_failed", team_id=input.team_id, user_id=input.user_id)return ActivitySummaryPairResult(diff --git a/products/signals/backend/management/commands/backfill_activity_summary_history.py b/products/signals/backend/management/commands/backfill_activity_summary_history.pynew file mode 100644index 000000000000..7cf5bcafe55f--- /dev/null+++ b/products/signals/backend/management/commands/backfill_activity_summary_history.py@@ -0,0 +1,119 @@+"""Build activity summaries as of past dates and emit their vectors, for the offline ranking evaluation.++Each build reads only the inputs before its as-of date and writes only an embedding under the+`activity_summary_backfill` document type. `document_embeddings` drops rows three months after their+timestamp, so keep the as-of dates inside that window and copy the vectors out soon after a run.+"""++import json+import datetime++from django.core.management.base import BaseCommand, CommandError+from django.utils import timezone++from products.signals.backend.activity_summary.backfill import (+ BackfillOptions,+ TeamUser,+ backfill_jobs,+ plan_backfill,+ run_backfill,+)+from products.signals.backend.activity_summary.config import parse_limits+from products.signals.backend.activity_summary.logic import MIN_UNITS+++def _date(value: str) -> datetime.date:+ return datetime.date.fromisoformat(value)+++def as_of_dates(+ dates: list[datetime.date] | None,+ start: datetime.date | None,+ end: datetime.date | None,+ every_days: int,+) -> list[datetime.datetime]:+ """Midnight UTC of each date, from `--as-of` or from `--start` to `--end` inclusive."""+ if dates:+ days = sorted(set(dates))+ elif start is not None and end is not None and start <= end:+ step = datetime.timedelta(days=max(1, every_days))+ days = []+ day = start+ while day <= end:+ days.append(day)+ day += step+ else:+ raise CommandError("Give --as-of, or --start and --end with --start on or before --end.")+ return [datetime.datetime.combine(day, datetime.time(), tzinfo=datetime.UTC) for day in days]+++class Command(BaseCommand):+ help = "Build activity summaries as of past dates and emit their embeddings. Writes no live summary state."++ def add_arguments(self, parser):+ parser.add_argument("--as-of", type=_date, nargs="+", help="As-of dates, YYYY-MM-DD. Each is midnight UTC.")+ parser.add_argument("--start", type=_date, help="The first as-of date, YYYY-MM-DD.")+ parser.add_argument("--end", type=_date, help="The last as-of date, YYYY-MM-DD.")+ parser.add_argument("--every-days", type=int, default=7, help="Days between as-of dates from --start.")+ parser.add_argument("--team-id", type=int)+ parser.add_argument("--user-id", type=int)+ parser.add_argument(+ "--inbox-users",+ action="store_true",+ help="Every (team, user) with an inbox report action before each as-of date.",+ )+ parser.add_argument("--exclude-inbox", action="store_true", help="Leave out inbox activity and views.")+ parser.add_argument("--min-units", type=int, default=MIN_UNITS)+ parser.add_argument(+ "--limits",+ type=json.loads,+ default=None,+ help="Render limits as JSON, with the clamps of the flag payload.",+ )+ parser.add_argument("--concurrency", type=int, default=4)+ parser.add_argument(+ "--max-cost-usd",+ type=float,+ required=True,+ help="Stop between batches when the cost of the run reaches this.",+ )+ parser.add_argument(+ "--dry-run", action="store_true", help="Count the builds and their highest cost. No model call."+ )++ def handle(self, *args, **options):+ has_pair = options["team_id"] is not None and options["user_id"] is not None+ has_id = options["team_id"] is not None or options["user_id"] is not None+ if options["inbox_users"] == has_id or (has_id and not has_pair):+ raise CommandError("Give both --team-id and --user-id, or --inbox-users alone.")+ dates = as_of_dates(options["as_of"], options["start"], options["end"], options["every_days"])+ if dates[-1] > timezone.now():+ raise CommandError("An as-of date is in the future.")++ pair = None if options["inbox_users"] else TeamUser(team_id=options["team_id"], user_id=options["user_id"])+ jobs = backfill_jobs(dates, pair=pair)+ backfill_options = BackfillOptions(+ limits=parse_limits(options["limits"]),+ min_units=options["min_units"],+ exclude_inbox=options["exclude_inbox"],+ )+ if options["dry_run"]:+ plan = plan_backfill(jobs, backfill_options)+ self.stdout.write(+ f"dry run: {plan.jobs} jobs on {len(dates)} dates, {plan.builds} clear min units, "+ f"at most {plan.max_cost_usd:.2f} usd"+ )+ return++ result = run_backfill(+ jobs,+ backfill_options,+ concurrency=options["concurrency"],+ max_cost_usd=options["max_cost_usd"],+ on_batch=lambda done, cost: self.stdout.write(f"{done}/{len(jobs)} jobs, {cost:.4f} usd"),+ )+ stopped = ", stopped at the cost cap" if result.stopped_at_cost_cap else ""+ self.stdout.write(+ f"done: {result.attempted}/{result.jobs} jobs, statuses {json.dumps(result.statuses, sort_keys=True)}, "+ f"{result.cost_usd:.4f} usd{stopped}"+ )diff --git a/products/signals/backend/test/test_activity_summary_backfill.py b/products/signals/backend/test/test_activity_summary_backfill.pynew file mode 100644index 000000000000..4e99da3d91fa--- /dev/null+++ b/products/signals/backend/test/test_activity_summary_backfill.py@@ -0,0 +1,124 @@+import datetime++import time_machine+from posthog.test.base import BaseTest+from unittest.mock import MagicMock, patch++from posthog.models import ActivityLog, Organization, Team, User++from products.signals.backend.activity_summary import tick+from products.signals.backend.activity_summary.backfill import (+ BACKFILL_DOCUMENT_TYPE,+ BackfillJob,+ BackfillOptions,+ backfill_jobs,+ plan_backfill,+ run_backfill,+)+from products.signals.backend.activity_summary.queue import max_update_cost_usd+from products.signals.backend.models import (+ SignalReport,+ SignalReportAction,+ UserActivitySummary,+ UserActivitySummaryVersion,+)+from products.signals.backend.test.test_activity_summary_render import FakeLLM+from products.signals.backend.test.test_activity_summary_update import ANSWER++NOW = datetime.datetime(2026, 10, 19, 12, tzinfo=datetime.UTC)+AS_OF = datetime.datetime(2026, 10, 10, tzinfo=datetime.UTC)+GATEWAY = "products.signals.backend.activity_summary.logic.gateway_client"+++@time_machine.travel(NOW, tick=False)+class TestActivitySummaryBackfill(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.organization.is_ai_data_processing_approved = True+ self.organization.is_ai_training_opted_in = True+ self.organization.save()++ def _changes(self, team: Team, user: User, count: int) -> None:+ for index in range(count):+ ActivityLog.objects.create(+ team_id=team.id,+ organization_id=team.organization_id,+ user=user,+ scope="Insight",+ activity="created",+ item_id=f"funnel-{index}",+ detail={"name": f"Funnel {index}"},+ created_at=AS_OF - datetime.timedelta(days=1, hours=index),+ )++ def _inbox_action(self, team: Team, user: User) -> None:+ report = SignalReport.objects.create(team=team, status="ready", title="Report", summary="Summary")+ with time_machine.travel(AS_OF - datetime.timedelta(days=2), tick=False):+ SignalReportAction.record(+ team_id=team.id,+ report_id=str(report.id),+ user_id=user.id,+ action_type=SignalReportAction.ActionType.VIEW,+ )++ def _jobs(self, count: int) -> list[BackfillJob]:+ return [+ BackfillJob(team_id=self.team.id, user_id=self.user.id, as_of=AS_OF - datetime.timedelta(days=n))+ for n in range(count)+ ]++ def test_inbox_users_on_teams_without_training_consent_get_no_job(self) -> None:+ other_org = Organization.objects.create(+ name="Other", is_ai_data_processing_approved=True, is_ai_training_opted_in=False+ )+ other_team = Team.objects.create(organization=other_org)+ other_user = User.objects.create_and_join(other_org, "[email protected]", None)+ self._inbox_action(self.team, self.user)+ self._inbox_action(other_team, other_user)++ jobs = backfill_jobs([AS_OF, AS_OF - datetime.timedelta(days=3)])++ assert jobs == [BackfillJob(team_id=self.team.id, user_id=self.user.id, as_of=AS_OF)]++ def test_a_run_emits_backfill_vectors_and_events_and_writes_no_live_state(self) -> None:+ self._changes(self.team, self.user, 3)+ capture = MagicMock()+ with (+ patch(GATEWAY, return_value=FakeLLM([ANSWER])),+ patch("posthog.api.embedding_worker.emit_embedding_request") as emit,+ patch.object(tick, "ph_background_capture", return_value=capture),+ ):+ result = run_backfill(self._jobs(1), BackfillOptions(exclude_inbox=True), max_cost_usd=10.0)++ assert result.statuses == {"ok": 1}+ embedding = emit.call_args.kwargs+ assert embedding["document_type"] == BACKFILL_DOCUMENT_TYPE+ assert embedding["document_id"] == f"{self.user.id}:2026-10-10:no_inbox"+ assert embedding["timestamp"] == AS_OF+ assert embedding["metadata"]["source"] == "backfill"+ assert embedding["metadata"]["exclude_inbox"] is True+ assert capture.call_args.kwargs["properties"]["source"] == "backfill"+ assert not UserActivitySummary.objects.for_team(self.team.id).exists()+ assert not UserActivitySummaryVersion.objects.for_team(self.team.id).exists()++ def test_a_dry_run_counts_builds_without_a_model_call(self) -> None:+ self._changes(self.team, self.user, 3)++ with patch(GATEWAY, side_effect=AssertionError("model called")):+ plan = plan_backfill(self._jobs(1) + self._jobs(40)[-1:], BackfillOptions())++ assert (plan.jobs, plan.builds) == (2, 1)+ assert plan.max_cost_usd == max_update_cost_usd()++ def test_the_cost_cap_stops_the_run_between_batches(self) -> None:+ self._changes(self.team, self.user, 3)+ llm = FakeLLM([ANSWER, ANSWER, ANSWER, ANSWER])+ with (+ patch(GATEWAY, return_value=llm),+ patch("posthog.api.embedding_worker.emit_embedding_request"),+ patch.object(tick, "ph_background_capture", return_value=MagicMock()),+ ):+ result = run_backfill(self._jobs(4), BackfillOptions(), max_cost_usd=1e-9)++ assert (result.attempted, result.stopped_at_cost_cap) == (1, True)+ assert len(llm.requests) == 1diff --git a/products/signals/backend/test/test_activity_summary_render.py b/products/signals/backend/test/test_activity_summary_render.pyindex 614a821f2a8b..77d14a144757 100644--- a/products/signals/backend/test/test_activity_summary_render.py+++ b/products/signals/backend/test/test_activity_summary_render.py@@ -33,6 +33,7 @@ from products.signals.backend.activity_summary.render import (ActivityItem,Drop,Parent,+ Product,RenderLimits,TaskItem,Unit,@@ -45,6 +46,7 @@ from products.signals.backend.activity_summary.render import (sampling_seed,task_units,view_units,+ without_products,)from products.signals.backend.activity_summary.summaries import PIPELINE_DISTINCT_ID, SummaryPrices, cost_usd, summarize@@ -426,6 +428,20 @@ def test_activity_units_spread_a_burst_across_days(_name: str, older_days: int,assert drops[Drop.BURST_CAP] + drops[Drop.SAMPLE_CAP] == len(burst) + len(older) - ACTIVITY_LINES+def test_without_products_drops_inbox_rows_and_views_and_counts_them() -> None:+ drops: Counter[str] = Counter()+ activity, views = without_products(+ [_change("SignalReport", "updated", "Weekly digest"), _change("Insight", "created", "Signup funnel")],+ [_view("scene", "Inbox"), _view("scene", "Dashboards"), _view("insight", "abc", "Signup funnel")],+ frozenset({Product.INBOX}),+ drops,+ )++ assert [item.scope for item in activity] == ["Insight"]+ assert [view.ref for view in views] == ["Dashboards", "abc"]+ assert drops == {Drop.EXCLUDED_PRODUCT: 2}++def test_burst_cap_keeps_the_newest_units_of_a_day() -> None:drops: Counter[str] = Counter()units = [Unit(at=_at(1, minutes=minute), text=f"unit {minute}") for minute in range(MAX_UNITS_PER_DAY + 2)]diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.pyindex 1e42a2f31da6..d584629e4018 100644--- a/products/signals/backend/test/test_activity_summary_update.py+++ b/products/signals/backend/test/test_activity_summary_update.py@@ -14,9 +14,14 @@ from products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.product_analytics.backend.facade.models import Insightfrom products.signals.backend.activity_summary.inputs import user_activity-from products.signals.backend.activity_summary.logic import ActivitySummaryUpdate, UpdateStatus, update_activity_summary+from products.signals.backend.activity_summary.logic import (+ ActivitySummaryUpdate,+ UpdateStatus,+ build_summary_as_of,+ update_activity_summary,+)from products.signals.backend.activity_summary.prompt import PROMPT_VERSION-from products.signals.backend.activity_summary.render import RENDER_VERSION, RenderLimits+from products.signals.backend.activity_summary.render import RENDER_VERSION, Product, RenderLimitsfrom products.signals.backend.models import UserActivitySummary, UserActivitySummaryVersionfrom products.signals.backend.test.test_activity_summary_render import FakeLLM@@ -363,3 +368,32 @@ class TestUpdateActivitySummary(BaseTest):assert self._update(llm).status == UpdateStatus.SKIPPED_AI_NOT_APPROVEDassert llm.requests == []++ def test_a_build_as_of_a_past_date_reads_only_the_rows_before_it_and_writes_nothing(self) -> None:+ self._changes(3, days_ago=11)+ self._change("Weekly digest", days_ago=10.5, scope="SignalReport")+ self._change("Funnel after as of", days_ago=9)+ self._change("Funnel at as of", days_ago=10)+ self._update(FakeLLM([ANSWER]))+ stored = self._stored()+ assert stored is not None+ llm = FakeLLM([ANSWER])++ with patch("products.signals.backend.activity_summary.logic.gateway_client", return_value=llm):+ result = build_summary_as_of(+ self.team.id,+ self.user.id,+ NOW - datetime.timedelta(days=10),+ exclude_products=frozenset({Product.INBOX}),+ )++ assert result.status == UpdateStatus.OK+ sent = json.dumps(llm.requests[0]["messages"])+ assert "Funnel 2" in sent+ assert "Funnel after as of" not in sent and "Funnel at as of" not in sent+ assert "Weekly digest" not in sent+ assert "Builds the signup funnel." not in sent+ assert result.inputs_used is not None and result.inputs_used["drops"]["excluded_product"] == 1+ after = self._stored()+ assert after is not None and (after.version, after.updated_at) == (stored.version, stored.updated_at)+ assert self._versions() == [1]