Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): backfill activity summaries as of past dates (#124481)

posthog[bot]6766379d4a28
diff --git a/products/signals/backend/activity_summary/backfill.py b/products/signals/backend/activity_summary/backfill.py
new file mode 100644
index 000000000000..9179c22ec80d
--- /dev/null
+++ b/products/signals/backend/activity_summary/backfill.py
@@ -0,0 +1,248 @@
+"""Activity summaries built as of past dates, for the offline evaluation of activity summary vectors.
+
+A backfill build reads only the inputs before its as-of moment, and writes only an embedding under its
+own document type and an event. It never touches `UserActivitySummary`, its versions or the queue state.
+"""
+
+import datetime
+from collections import Counter
+from collections.abc import Callable, Iterable, Sequence
+from concurrent.futures import ThreadPoolExecutor
+
+from django.db import connection
+
+import structlog
+
+from posthog.dataclasses import frozen
+from posthog.models import Team
+
+from products.signals.backend.activity_summary.logic import (
+ MIN_UNITS,
+ ActivitySummaryUpdate,
+ UpdateStatus,
+ build_summary_as_of,
+ render_as_of,
+)
+from products.signals.backend.activity_summary.prompt import PROMPT_VERSION, summary_text
+from products.signals.backend.activity_summary.queue import max_update_cost_usd
+from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, RENDER_VERSION, Product, RenderLimits
+from products.signals.backend.activity_summary.summaries import DEFAULT_MODEL
+from products.signals.backend.activity_summary.tick import capture_generated, emit_summary_embedding
+from products.signals.backend.models import SignalReportAction
+
+logger = structlog.get_logger(__name__)
+
+# Separate from the live `activity_summary` type, so the live vectors stay clean.
+BACKFILL_DOCUMENT_TYPE = "activity_summary_backfill"
+SOURCE_BACKFILL = "backfill"
+# The document id suffix of a build without inbox inputs, so both variants of one pair and day coexist.
+EXCLUDE_INBOX_SUFFIX = ":no_inbox"
+INBOX_PRODUCTS: frozenset[Product] = frozenset({Product.INBOX})
+# The status of a build that raised. It can have called the model, so it counts at the highest cost.
+STATUS_EXCEPTION = "exception"
+
+
+@frozen(order=True)
+class TeamUser:
+ team_id: int
+ user_id: int
+
+
+@frozen
+class BackfillJob:
+ team_id: int
+ user_id: int
+ as_of: datetime.datetime
+
+
+@frozen
+class BackfillOptions:
+ limits: RenderLimits = DEFAULT_LIMITS
+ min_units: int = MIN_UNITS
+ exclude_inbox: bool = False
+
+
+@frozen
+class BackfillPlan:
+ jobs: int
+ # Jobs whose rendered input clears `min_units`, so a real run would call the model for each.
+ builds: int
+ max_cost_usd: float
+
+
+@frozen
+class BackfillResult:
+ jobs: int
+ attempted: int
+ statuses: dict[str, int]
+ cost_usd: float
+ stopped_at_cost_cap: bool
+
+
+def eligible_team_ids(team_ids: Iterable[int]) -> set[int]:
+ """The teams whose organization approved AI data processing and opted in to AI training."""
+ return set(
+ Team.objects.filter(
+ id__in=list(team_ids),
+ organization__is_ai_data_processing_approved=True,
+ organization__is_ai_training_opted_in=True,
+ ).values_list("id", flat=True)
+ )
+
+
+def inbox_users(as_of: datetime.datetime) -> set[TeamUser]:
+ """Every (team, user) with an inbox report action that started before `as_of`."""
+ return {
+ TeamUser(team_id=team_id, user_id=user_id)
+ for team_id, user_id in (
+ # nosemgrep: idor-lookup-without-team (an operator backfill over all teams, with no user input)
+ SignalReportAction.all_teams.filter(first_at__lt=as_of).values_list("team_id", "user_id").distinct()
+ )
+ }
+
+
+def backfill_jobs(
+ as_of_dates: Sequence[datetime.datetime],
+ *,
+ pair: TeamUser | None = None,
+) -> list[BackfillJob]:
+ """One job per as-of date and pair, for `pair` or for the inbox users before that date, on eligible teams only."""
+ jobs: list[BackfillJob] = []
+ for as_of in as_of_dates:
+ pairs = {pair} if pair is not None else inbox_users(as_of)
+ eligible = eligible_team_ids({p.team_id for p in pairs})
+ jobs.extend(
+ BackfillJob(team_id=p.team_id, user_id=p.user_id, as_of=as_of)
+ for p in sorted(pairs)
+ if p.team_id in eligible
+ )
+ return jobs
+
+
+def _excluded_products(options: BackfillOptions) -> frozenset[Product]:
+ return INBOX_PRODUCTS if options.exclude_inbox else frozenset()
+
+
+def document_id(job: BackfillJob, exclude_inbox: bool) -> str:
+ base = f"{job.user_id}:{job.as_of.date().isoformat()}"
+ return f"{base}{EXCLUDE_INBOX_SUFFIX}" if exclude_inbox else base
+
+
+def _emit_embedding(job: BackfillJob, update: ActivitySummaryUpdate, exclude_inbox: bool) -> None:
+ if update.summary is None:
+ return
+ emit_summary_embedding(
+ job.team_id,
+ content=summary_text(update.summary),
+ document_type=BACKFILL_DOCUMENT_TYPE,
+ document_id=document_id(job, exclude_inbox),
+ # The as-of moment, so a rerun of the same job replaces its own row.
+ timestamp=job.as_of,
+ metadata={
+ "prompt_version": PROMPT_VERSION,
+ "render_version": RENDER_VERSION,
+ "model": DEFAULT_MODEL,
+ "source": SOURCE_BACKFILL,
+ "exclude_inbox": exclude_inbox,
+ },
+ )
+
+
+def run_job(job: BackfillJob, options: BackfillOptions) -> ActivitySummaryUpdate:
+ """Build one summary as of the job date, then emit its embedding and its event."""
+ update = build_summary_as_of(
+ job.team_id,
+ job.user_id,
+ job.as_of,
+ limits=options.limits,
+ min_units=options.min_units,
+ exclude_products=_excluded_products(options),
+ trace_variant=f"{SOURCE_BACKFILL}:{document_id(job, options.exclude_inbox)}",
+ )
+ if update.status == UpdateStatus.OK:
+ try:
+ _emit_embedding(job, update, options.exclude_inbox)
+ except Exception:
+ logger.exception("activity_summary_backfill_embedding_failed", team_id=job.team_id, user_id=job.user_id)
+ if update.status in (UpdateStatus.OK, UpdateStatus.FAILED):
+ try:
+ capture_generated(job.team_id, job.user_id, update, None, source=SOURCE_BACKFILL)
+ except Exception:
+ logger.exception("activity_summary_backfill_capture_failed", team_id=job.team_id, user_id=job.user_id)
+ return update
+
+
+def plan_backfill(jobs: Sequence[BackfillJob], options: BackfillOptions) -> BackfillPlan:
+ """Count the jobs that would call the model, without a model call."""
+ builds = 0
+ for job in jobs:
+ built = render_as_of(
+ job.team_id, job.user_id, job.as_of, limits=options.limits, exclude_products=_excluded_products(options)
+ )
+ builds += built is not None and built.rendered.units >= options.min_units
+ return BackfillPlan(jobs=len(jobs), builds=builds, max_cost_usd=builds * max_update_cost_usd(options.limits))
+
+
+def _run_job_in_thread(job: BackfillJob, options: BackfillOptions) -> ActivitySummaryUpdate:
+ try:
+ return run_job(job, options)
+ finally:
+ # Each worker thread opens its own connection, and nothing else closes it.
+ connection.close()
+
+
+def _guarded(
+ run: Callable[[BackfillJob, BackfillOptions], ActivitySummaryUpdate], job: BackfillJob, options: BackfillOptions
+) -> ActivitySummaryUpdate | None:
+ try:
+ return run(job, options)
+ except Exception:
+ logger.exception("activity_summary_backfill_job_failed", team_id=job.team_id, user_id=job.user_id)
+ return None
+
+
+def run_backfill(
+ jobs: Sequence[BackfillJob],
+ options: BackfillOptions,
+ *,
+ concurrency: int = 1,
+ max_cost_usd: float,
+ on_batch: Callable[[int, float], None] | None = None,
+) -> BackfillResult:
+ """Run the jobs in batches of `concurrency`, and stop between batches when the cost reaches `max_cost_usd`.
+
+ A build without a cost, or a build that raised, counts at the highest cost of one update."""
+ bound = max_update_cost_usd(options.limits)
+ statuses: Counter[str] = Counter()
+ cost = 0.0
+ attempted = 0
+ batch_size = max(1, concurrency)
+ executor = ThreadPoolExecutor(max_workers=batch_size) if batch_size > 1 else None
+ try:
+ while attempted < len(jobs) and cost < max_cost_usd:
+ batch = jobs[attempted : attempted + batch_size]
+ attempted += len(batch)
+ if executor is None:
+ outcomes = [_guarded(run_job, job, options) for job in batch]
+ else:
+ outcomes = list(executor.map(lambda job: _guarded(_run_job_in_thread, job, options), batch))
+ for update in outcomes:
+ if update is None:
+ statuses[STATUS_EXCEPTION] += 1
+ cost += bound
+ continue
+ statuses[update.status.value] += 1
+ if update.status in (UpdateStatus.OK, UpdateStatus.FAILED):
+ cost += bound if update.cost_usd is None else update.cost_usd
+ if on_batch is not None:
+ on_batch(attempted, cost)
+ finally:
+ if executor is not None:
+ executor.shutdown()
+ return BackfillResult(
+ jobs=len(jobs),
+ attempted=attempted,
+ statuses=dict(statuses),
+ cost_usd=cost,
+ stopped_at_cost_cap=attempted < len(jobs),
+ )
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index c3b859908d7f..4d115f781568 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -1,7 +1,10 @@
-"""One activity summary update for one (team, user): gather, filter, render, summarize, persist."""
+"""One activity summary update for one (team, user): gather, filter, render, summarize, persist.
+
+`build_summary_as_of` runs the same steps as of a past moment, and persists nothing."""
import datetime
import dataclasses
+from collections import Counter
from enum import StrEnum
from typing import Any, Protocol
@@ -19,13 +22,20 @@ from products.signals.backend.activity_summary.prompt import PROMPT_VERSION, sum
from products.signals.backend.activity_summary.render import (
DEFAULT_LIMITS,
RENDER_VERSION,
+ ActivityItem,
+ Product,
+ RenderedInput,
RenderLimits,
+ TaskItem,
+ ViewItem,
build_input,
sampling_seed,
+ without_products,
)
from products.signals.backend.activity_summary.summaries import (
DEFAULT_MODEL,
MODEL_PRICES,
+ SummaryOutcome,
cost_usd,
gateway_client,
summarize,
@@ -125,6 +135,77 @@ def _input_start(watermark: datetime.datetime | None, now: datetime.datetime, wi
return earliest if watermark is None else max(earliest, watermark + _WATERMARK_STEP)
+@frozen
+class _WindowInputs:
+ tasks: list[TaskItem]
+ activity: list[ActivityItem]
+ views: list[ViewItem]
+
+
+def _read_inputs(team_id: int, user_id: int, start: datetime.datetime, end: datetime.datetime) -> _WindowInputs:
+ return _WindowInputs(
+ tasks=user_tasks(team_id, user_id, start, end),
+ activity=user_activity(team_id, user_id, start, end),
+ views=user_views(team_id, user_id, start, end),
+ )
+
+
+def _render(
+ team_id: int,
+ user_id: int,
+ tasks: list[TaskItem],
+ activity: list[ActivityItem],
+ views: list[ViewItem],
+ *,
+ as_of: datetime.datetime,
+ limits: RenderLimits,
+) -> RenderedInput | None:
+ return build_input(
+ tasks,
+ activity,
+ as_of=as_of,
+ seed=sampling_seed(as_of.date().isoformat(), team_id, user_id),
+ views=views,
+ limits=limits,
+ parents=activity_parents(team_id, activity),
+ )
+
+
+def _inputs_used(rendered: RenderedInput, limits: RenderLimits) -> dict[str, Any]:
+ return {
+ "tasks": rendered.n_tasks,
+ "activity_lines": rendered.n_activity_lines,
+ "view_lines": rendered.n_view_lines,
+ "input_chars": rendered.input_chars,
+ "drops": rendered.drops,
+ "limits": dataclasses.asdict(limits),
+ }
+
+
+def _call_model(
+ team_id: int,
+ user_id: int,
+ rendered: RenderedInput,
+ computed: dict[str, Any],
+ *,
+ previous: dict[str, Any] | None,
+ as_of: datetime.datetime,
+ limits: RenderLimits,
+ trace_variant: str | None = None,
+) -> tuple[SummaryOutcome, float | None]:
+ outcome = summarize(
+ gateway_client(team_id, user_id, as_of.date(), variant=trace_variant),
+ rendered_input=rendered.text,
+ previous_summary=previous,
+ model=DEFAULT_MODEL,
+ as_of=as_of.date(),
+ max_completion_tokens=limits.max_completion_tokens,
+ profile_max_chars=limits.profile_max_chars,
+ computed=computed,
+ )
+ return outcome, cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(DEFAULT_MODEL))
+
+
def _back_off(
team_id: int,
user_id: int,
@@ -277,18 +358,9 @@ def update_activity_summary(
rebuild = is_stale(stored, rebuild_before)
window_start = _input_start(None, now, limits.window_days)
start = _input_start(stored.input_watermark if stored and not rebuild else None, now, limits.window_days)
- tasks = user_tasks(team_id, user_id, start, now)
- activity = user_activity(team_id, user_id, start, now)
- views = user_views(team_id, user_id, start, now)
- rendered = build_input(
- tasks,
- activity,
- as_of=now,
- seed=sampling_seed(now.date().isoformat(), team_id, user_id),
- views=views,
- limits=limits,
- parents=activity_parents(team_id, activity),
- )
+ window = _read_inputs(team_id, user_id, start, now)
+ tasks, activity, views = window.tasks, window.activity, window.views
+ rendered = _render(team_id, user_id, tasks, activity, views, as_of=now, limits=limits)
if rendered is None or rendered.units < min_units:
if not dry_run:
_back_off(team_id, user_id, stored, now, max_skip_backoff)
@@ -298,37 +370,16 @@ def update_activity_summary(
inputs_used={"drops": rendered.drops} if rendered else None,
)
- inputs_used: dict[str, Any] = {
- "tasks": rendered.n_tasks,
- "activity_lines": rendered.n_activity_lines,
- "view_lines": rendered.n_view_lines,
- "input_chars": rendered.input_chars,
- "drops": rendered.drops,
- "limits": dataclasses.asdict(limits),
- }
+ inputs_used = _inputs_used(rendered, limits)
watermark = max([task.at for task in tasks] + [item.at for item in activity] + [view.at for view in views])
# An incremental update reads only the rows after the watermark, so the computed fields read the full window.
if start == window_start:
computed = compute_fields(tasks, activity, views).as_json()
else:
- computed = compute_fields(
- user_tasks(team_id, user_id, window_start, now),
- user_activity(team_id, user_id, window_start, now),
- user_views(team_id, user_id, window_start, now),
- ).as_json()
+ full = _read_inputs(team_id, user_id, window_start, now)
+ computed = compute_fields(full.tasks, full.activity, full.views).as_json()
previous = stored.summary if stored is not None and not rebuild else None
- model = DEFAULT_MODEL
- outcome = summarize(
- gateway_client(team_id, user_id, now.date()),
- rendered_input=rendered.text,
- previous_summary=previous,
- model=model,
- as_of=now.date(),
- max_completion_tokens=limits.max_completion_tokens,
- profile_max_chars=limits.profile_max_chars,
- computed=computed,
- )
- cost = cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(model))
+ outcome, cost = _call_model(team_id, user_id, rendered, computed, previous=previous, as_of=now, limits=limits)
if outcome.summary is not None:
status = UpdateStatus.OK
elif outcome.empty_profile:
@@ -399,7 +450,7 @@ def update_activity_summary(
"summary_text": summary_text(outcome.summary),
"computed": computed,
"generated_at": now,
- "model": model,
+ "model": DEFAULT_MODEL,
"prompt_version": PROMPT_VERSION,
"render_version": RENDER_VERSION,
"input_watermark": watermark,
@@ -411,3 +462,79 @@ def update_activity_summary(
return result
except Exception as error:
raise UpdateAfterModelCallError(cost) from error
+
+
+@frozen
+class AsOfInput:
+ rendered: RenderedInput
+ computed: dict[str, Any]
+ inputs_used: dict[str, Any]
+
+
+def render_as_of(
+ team_id: int,
+ user_id: int,
+ as_of: datetime.datetime,
+ *,
+ limits: RenderLimits = DEFAULT_LIMITS,
+ exclude_products: frozenset[Product] = frozenset(),
+) -> AsOfInput | None:
+ """The rendered input and computed fields of `[as_of - limits.window_days, as_of)`, or None when nothing renders.
+
+ Views come from the latest view of each object, so an object opened again after `as_of` is missing."""
+ window = _read_inputs(team_id, user_id, as_of - datetime.timedelta(days=limits.window_days), as_of)
+ tasks, activity, views = window.tasks, window.activity, window.views
+ excluded: Counter[str] = Counter()
+ activity, views = without_products(activity, views, exclude_products, excluded)
+ rendered = _render(team_id, user_id, tasks, activity, views, as_of=as_of, limits=limits)
+ if rendered is None:
+ return None
+ inputs_used = _inputs_used(rendered, limits)
+ inputs_used["drops"] = {**rendered.drops, **excluded}
+ return AsOfInput(
+ rendered=rendered, computed=compute_fields(tasks, activity, views).as_json(), inputs_used=inputs_used
+ )
+
+
+def build_summary_as_of(
+ team_id: int,
+ user_id: int,
+ as_of: datetime.datetime,
+ *,
+ limits: RenderLimits = DEFAULT_LIMITS,
+ min_units: int = MIN_UNITS,
+ exclude_products: frozenset[Product] = frozenset(),
+ trace_variant: str | None = None,
+) -> ActivitySummaryUpdate:
+ """A summary of one user in one project as of `as_of`, from the inputs before it only.
+
+ The model gets no previous summary, and the sample seed comes from the `as_of` date. Nothing is written."""
+ built = render_as_of(team_id, user_id, as_of, limits=limits, exclude_products=exclude_products)
+ if built is None or built.rendered.units < min_units:
+ return ActivitySummaryUpdate(
+ status=UpdateStatus.SKIPPED_TOO_FEW_UNITS,
+ rendered_input=built.rendered.text if built else None,
+ inputs_used=built.inputs_used if built else None,
+ )
+ outcome, cost = _call_model(
+ team_id,
+ user_id,
+ built.rendered,
+ built.computed,
+ previous=None,
+ as_of=as_of,
+ limits=limits,
+ trace_variant=trace_variant,
+ )
+ return ActivitySummaryUpdate(
+ status=UpdateStatus.OK if outcome.summary is not None else UpdateStatus.FAILED,
+ rendered_input=built.rendered.text,
+ summary=outcome.summary,
+ tokens_in=outcome.tokens_in,
+ tokens_out=outcome.tokens_out,
+ cost_usd=cost,
+ latency_ms=outcome.latency_ms,
+ error=outcome.error,
+ inputs_used=built.inputs_used,
+ computed=built.computed if outcome.summary is not None else None,
+ )
diff --git a/products/signals/backend/activity_summary/render.py b/products/signals/backend/activity_summary/render.py
index 677f07a75bb3..828603a20ef7 100644
--- a/products/signals/backend/activity_summary/render.py
+++ b/products/signals/backend/activity_summary/render.py
@@ -267,6 +267,7 @@ class Drop(StrEnum):
BURST_CAP = "burst_cap"
SAMPLE_CAP = "sample_cap"
SECTIONS_BUDGET = "sections_budget"
+ EXCLUDED_PRODUCT = "excluded_product"
@frozen
@@ -842,6 +843,34 @@ def scope_of(view_type: str) -> str:
return "".join(word.capitalize() for word in view_type.split("/")[0].split("_"))
+def view_product(view: ViewItem) -> Product | None:
+ if view.type == SCENE_VIEW_TYPE:
+ return scene_product(view.ref)
+ return scope_product(scope_of(view.type))
+
+
+def without_products(
+ activity: Iterable[ActivityItem],
+ views: Iterable[ViewItem],
+ products: frozenset[Product],
+ drops: Counter[str],
+) -> tuple[list[ActivityItem], list[ViewItem]]:
+ """The activity rows and views whose product is not in `products`. Tasks have no product, so they stay."""
+ kept_activity: list[ActivityItem] = []
+ for item in activity:
+ if scope_product(item.scope) in products:
+ drops[Drop.EXCLUDED_PRODUCT] += 1
+ else:
+ kept_activity.append(item)
+ kept_views: list[ViewItem] = []
+ for view in views:
+ if view_product(view) in products:
+ drops[Drop.EXCLUDED_PRODUCT] += 1
+ else:
+ kept_views.append(view)
+ return kept_activity, kept_views
+
+
def _days(count: int) -> str:
return f"{count} day" if count == 1 else f"{count} days"
diff --git a/products/signals/backend/activity_summary/summaries.py b/products/signals/backend/activity_summary/summaries.py
index 5b5f4b4a6371..6a07c246d301 100644
--- a/products/signals/backend/activity_summary/summaries.py
+++ b/products/signals/backend/activity_summary/summaries.py
@@ -69,11 +69,13 @@ class SummaryOutcome:
empty_profile: bool = False
-def gateway_client(team_id: int, user_id: int, as_of: datetime.date) -> OpenAI:
+def gateway_client(team_id: int, user_id: int, as_of: datetime.date, *, variant: str | None = None) -> OpenAI:
+ """`variant` keeps the trace of another build of the same pair and day, such as a backfill, apart."""
+ seed = f"{AI_PRODUCT}:{team_id}:{user_id}:{as_of.isoformat()}"
return build_openai_client(
"signals",
ai_product=AI_PRODUCT,
- trace_id=trace_id_from_seed(f"{AI_PRODUCT}:{team_id}:{user_id}:{as_of.isoformat()}"),
+ trace_id=trace_id_from_seed(f"{seed}:{variant}" if variant else seed),
properties={"team_id": str(team_id)},
distinct_id=PIPELINE_DISTINCT_ID,
)
diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.py
index 3a0f8933ff85..90c19e40bee7 100644
--- a/products/signals/backend/activity_summary/tick.py
+++ b/products/signals/backend/activity_summary/tick.py
@@ -8,6 +8,7 @@ import asyncio
import datetime
import dataclasses
from collections import Counter
+from typing import Any
from django.conf import settings
from django.utils import timezone
@@ -62,6 +63,8 @@ logger = structlog.get_logger(__name__)
TICK_WORKFLOW_NAME = "activity-summary-tick"
EMBEDDING_DOCUMENT_TYPE = "activity_summary"
EMBEDDING_RENDERING = "summary_text"
+# Where a summary came from: the tick, or a build as of a past date.
+SOURCE_LIVE = "live"
# Stop reasons of a tick. Null when the tick updated every due pair.
STOP_MAX_PAIRS_PER_TICK = "max_pairs_per_tick"
@@ -229,23 +232,42 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
)
-def _emit_embedding(team_id: int, user_id: int) -> None:
+def emit_summary_embedding(
+ team_id: int,
+ *,
+ content: str,
+ document_type: str,
+ document_id: str,
+ timestamp: datetime.datetime,
+ metadata: dict[str, Any],
+) -> None:
# Deferred so importing the tick does not load the Kafka and ClickHouse chain behind the producer.
from posthog.api.embedding_worker import emit_embedding_request # noqa: PLC0415
+ emit_embedding_request(
+ content=content,
+ team_id=team_id,
+ product=EMBEDDING_PRODUCT,
+ document_type=document_type,
+ rendering=EMBEDDING_RENDERING,
+ document_id=document_id,
+ # The models of the report embeddings, so both share one vector space.
+ models=[model.value for model in EmbeddingModelName],
+ timestamp=timestamp,
+ metadata=metadata,
+ )
+
+
+def _emit_embedding(team_id: int, user_id: int) -> None:
stored = UserActivitySummary.objects.for_team(team_id).filter(user_id=user_id).first()
if stored is None or not stored.summary_text or stored.generated_at is None:
return
- emit_embedding_request(
+ emit_summary_embedding(
+ team_id,
content=stored.summary_text,
- team_id=team_id,
- product=EMBEDDING_PRODUCT,
document_type=EMBEDDING_DOCUMENT_TYPE,
- rendering=EMBEDDING_RENDERING,
# One vector per user per day, so an analysis can read the vector a user had on a given day.
document_id=f"{user_id}:{stored.generated_at.date().isoformat()}",
- # The models of the report embeddings, so both share one vector space.
- models=[model.value for model in EmbeddingModelName],
timestamp=stored.generated_at,
metadata={
"prompt_version": stored.prompt_version,
@@ -255,7 +277,9 @@ def _emit_embedding(team_id: int, user_id: int) -> None:
)
-def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate, trigger: str | None) -> None:
+def capture_generated(
+ team_id: int, user_id: int, update: ActivitySummaryUpdate, trigger: str | None, *, source: str = SOURCE_LIVE
+) -> None:
team = Team.objects.only("uuid", "organization_id").get(id=team_id)
# Counts and accounting only. The summary and the rendered input are user content.
ph_background_capture()(
@@ -277,6 +301,7 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate
"inputs_used": update.inputs_used,
"rebuilt": update.rebuilt,
"trigger": trigger,
+ "source": source,
},
groups={"organization": str(team.organization_id), "project": str(team.uuid)},
)
@@ -327,7 +352,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A
logger.exception("activity_summary_embedding_failed", team_id=input.team_id, user_id=input.user_id)
if update.status in CALLED_MODEL_STATUSES and input.capture_events:
try:
- _capture_generated(input.team_id, input.user_id, update, input.trigger)
+ capture_generated(input.team_id, input.user_id, update, input.trigger)
except Exception:
logger.exception("activity_summary_capture_failed", team_id=input.team_id, user_id=input.user_id)
return ActivitySummaryPairResult(
diff --git a/products/signals/backend/management/commands/backfill_activity_summary_history.py b/products/signals/backend/management/commands/backfill_activity_summary_history.py
new file mode 100644
index 000000000000..7cf5bcafe55f
--- /dev/null
+++ b/products/signals/backend/management/commands/backfill_activity_summary_history.py
@@ -0,0 +1,119 @@
+"""Build activity summaries as of past dates and emit their vectors, for the offline ranking evaluation.
+
+Each build reads only the inputs before its as-of date and writes only an embedding under the
+`activity_summary_backfill` document type. `document_embeddings` drops rows three months after their
+timestamp, so keep the as-of dates inside that window and copy the vectors out soon after a run.
+"""
+
+import json
+import datetime
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from products.signals.backend.activity_summary.backfill import (
+ BackfillOptions,
+ TeamUser,
+ backfill_jobs,
+ plan_backfill,
+ run_backfill,
+)
+from products.signals.backend.activity_summary.config import parse_limits
+from products.signals.backend.activity_summary.logic import MIN_UNITS
+
+
+def _date(value: str) -> datetime.date:
+ return datetime.date.fromisoformat(value)
+
+
+def as_of_dates(
+ dates: list[datetime.date] | None,
+ start: datetime.date | None,
+ end: datetime.date | None,
+ every_days: int,
+) -> list[datetime.datetime]:
+ """Midnight UTC of each date, from `--as-of` or from `--start` to `--end` inclusive."""
+ if dates:
+ days = sorted(set(dates))
+ elif start is not None and end is not None and start <= end:
+ step = datetime.timedelta(days=max(1, every_days))
+ days = []
+ day = start
+ while day <= end:
+ days.append(day)
+ day += step
+ else:
+ raise CommandError("Give --as-of, or --start and --end with --start on or before --end.")
+ return [datetime.datetime.combine(day, datetime.time(), tzinfo=datetime.UTC) for day in days]
+
+
+class Command(BaseCommand):
+ help = "Build activity summaries as of past dates and emit their embeddings. Writes no live summary state."
+
+ def add_arguments(self, parser):
+ parser.add_argument("--as-of", type=_date, nargs="+", help="As-of dates, YYYY-MM-DD. Each is midnight UTC.")
+ parser.add_argument("--start", type=_date, help="The first as-of date, YYYY-MM-DD.")
+ parser.add_argument("--end", type=_date, help="The last as-of date, YYYY-MM-DD.")
+ parser.add_argument("--every-days", type=int, default=7, help="Days between as-of dates from --start.")
+ parser.add_argument("--team-id", type=int)
+ parser.add_argument("--user-id", type=int)
+ parser.add_argument(
+ "--inbox-users",
+ action="store_true",
+ help="Every (team, user) with an inbox report action before each as-of date.",
+ )
+ parser.add_argument("--exclude-inbox", action="store_true", help="Leave out inbox activity and views.")
+ parser.add_argument("--min-units", type=int, default=MIN_UNITS)
+ parser.add_argument(
+ "--limits",
+ type=json.loads,
+ default=None,
+ help="Render limits as JSON, with the clamps of the flag payload.",
+ )
+ parser.add_argument("--concurrency", type=int, default=4)
+ parser.add_argument(
+ "--max-cost-usd",
+ type=float,
+ required=True,
+ help="Stop between batches when the cost of the run reaches this.",
+ )
+ parser.add_argument(
+ "--dry-run", action="store_true", help="Count the builds and their highest cost. No model call."
+ )
+
+ def handle(self, *args, **options):
+ has_pair = options["team_id"] is not None and options["user_id"] is not None
+ has_id = options["team_id"] is not None or options["user_id"] is not None
+ if options["inbox_users"] == has_id or (has_id and not has_pair):
+ raise CommandError("Give both --team-id and --user-id, or --inbox-users alone.")
+ dates = as_of_dates(options["as_of"], options["start"], options["end"], options["every_days"])
+ if dates[-1] > timezone.now():
+ raise CommandError("An as-of date is in the future.")
+
+ pair = None if options["inbox_users"] else TeamUser(team_id=options["team_id"], user_id=options["user_id"])
+ jobs = backfill_jobs(dates, pair=pair)
+ backfill_options = BackfillOptions(
+ limits=parse_limits(options["limits"]),
+ min_units=options["min_units"],
+ exclude_inbox=options["exclude_inbox"],
+ )
+ if options["dry_run"]:
+ plan = plan_backfill(jobs, backfill_options)
+ self.stdout.write(
+ f"dry run: {plan.jobs} jobs on {len(dates)} dates, {plan.builds} clear min units, "
+ f"at most {plan.max_cost_usd:.2f} usd"
+ )
+ return
+
+ result = run_backfill(
+ jobs,
+ backfill_options,
+ concurrency=options["concurrency"],
+ max_cost_usd=options["max_cost_usd"],
+ on_batch=lambda done, cost: self.stdout.write(f"{done}/{len(jobs)} jobs, {cost:.4f} usd"),
+ )
+ stopped = ", stopped at the cost cap" if result.stopped_at_cost_cap else ""
+ self.stdout.write(
+ f"done: {result.attempted}/{result.jobs} jobs, statuses {json.dumps(result.statuses, sort_keys=True)}, "
+ f"{result.cost_usd:.4f} usd{stopped}"
+ )
diff --git a/products/signals/backend/test/test_activity_summary_backfill.py b/products/signals/backend/test/test_activity_summary_backfill.py
new file mode 100644
index 000000000000..4e99da3d91fa
--- /dev/null
+++ b/products/signals/backend/test/test_activity_summary_backfill.py
@@ -0,0 +1,124 @@
+import datetime
+
+import time_machine
+from posthog.test.base import BaseTest
+from unittest.mock import MagicMock, patch
+
+from posthog.models import ActivityLog, Organization, Team, User
+
+from products.signals.backend.activity_summary import tick
+from products.signals.backend.activity_summary.backfill import (
+ BACKFILL_DOCUMENT_TYPE,
+ BackfillJob,
+ BackfillOptions,
+ backfill_jobs,
+ plan_backfill,
+ run_backfill,
+)
+from products.signals.backend.activity_summary.queue import max_update_cost_usd
+from products.signals.backend.models import (
+ SignalReport,
+ SignalReportAction,
+ UserActivitySummary,
+ UserActivitySummaryVersion,
+)
+from products.signals.backend.test.test_activity_summary_render import FakeLLM
+from products.signals.backend.test.test_activity_summary_update import ANSWER
+
+NOW = datetime.datetime(2026, 10, 19, 12, tzinfo=datetime.UTC)
+AS_OF = datetime.datetime(2026, 10, 10, tzinfo=datetime.UTC)
+GATEWAY = "products.signals.backend.activity_summary.logic.gateway_client"
+
+
+@time_machine.travel(NOW, tick=False)
+class TestActivitySummaryBackfill(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.organization.is_ai_data_processing_approved = True
+ self.organization.is_ai_training_opted_in = True
+ self.organization.save()
+
+ def _changes(self, team: Team, user: User, count: int) -> None:
+ for index in range(count):
+ ActivityLog.objects.create(
+ team_id=team.id,
+ organization_id=team.organization_id,
+ user=user,
+ scope="Insight",
+ activity="created",
+ item_id=f"funnel-{index}",
+ detail={"name": f"Funnel {index}"},
+ created_at=AS_OF - datetime.timedelta(days=1, hours=index),
+ )
+
+ def _inbox_action(self, team: Team, user: User) -> None:
+ report = SignalReport.objects.create(team=team, status="ready", title="Report", summary="Summary")
+ with time_machine.travel(AS_OF - datetime.timedelta(days=2), tick=False):
+ SignalReportAction.record(
+ team_id=team.id,
+ report_id=str(report.id),
+ user_id=user.id,
+ action_type=SignalReportAction.ActionType.VIEW,
+ )
+
+ def _jobs(self, count: int) -> list[BackfillJob]:
+ return [
+ BackfillJob(team_id=self.team.id, user_id=self.user.id, as_of=AS_OF - datetime.timedelta(days=n))
+ for n in range(count)
+ ]
+
+ def test_inbox_users_on_teams_without_training_consent_get_no_job(self) -> None:
+ other_org = Organization.objects.create(
+ name="Other", is_ai_data_processing_approved=True, is_ai_training_opted_in=False
+ )
+ other_team = Team.objects.create(organization=other_org)
+ other_user = User.objects.create_and_join(other_org, "[email protected]", None)
+ self._inbox_action(self.team, self.user)
+ self._inbox_action(other_team, other_user)
+
+ jobs = backfill_jobs([AS_OF, AS_OF - datetime.timedelta(days=3)])
+
+ assert jobs == [BackfillJob(team_id=self.team.id, user_id=self.user.id, as_of=AS_OF)]
+
+ def test_a_run_emits_backfill_vectors_and_events_and_writes_no_live_state(self) -> None:
+ self._changes(self.team, self.user, 3)
+ capture = MagicMock()
+ with (
+ patch(GATEWAY, return_value=FakeLLM([ANSWER])),
+ patch("posthog.api.embedding_worker.emit_embedding_request") as emit,
+ patch.object(tick, "ph_background_capture", return_value=capture),
+ ):
+ result = run_backfill(self._jobs(1), BackfillOptions(exclude_inbox=True), max_cost_usd=10.0)
+
+ assert result.statuses == {"ok": 1}
+ embedding = emit.call_args.kwargs
+ assert embedding["document_type"] == BACKFILL_DOCUMENT_TYPE
+ assert embedding["document_id"] == f"{self.user.id}:2026-10-10:no_inbox"
+ assert embedding["timestamp"] == AS_OF
+ assert embedding["metadata"]["source"] == "backfill"
+ assert embedding["metadata"]["exclude_inbox"] is True
+ assert capture.call_args.kwargs["properties"]["source"] == "backfill"
+ assert not UserActivitySummary.objects.for_team(self.team.id).exists()
+ assert not UserActivitySummaryVersion.objects.for_team(self.team.id).exists()
+
+ def test_a_dry_run_counts_builds_without_a_model_call(self) -> None:
+ self._changes(self.team, self.user, 3)
+
+ with patch(GATEWAY, side_effect=AssertionError("model called")):
+ plan = plan_backfill(self._jobs(1) + self._jobs(40)[-1:], BackfillOptions())
+
+ assert (plan.jobs, plan.builds) == (2, 1)
+ assert plan.max_cost_usd == max_update_cost_usd()
+
+ def test_the_cost_cap_stops_the_run_between_batches(self) -> None:
+ self._changes(self.team, self.user, 3)
+ llm = FakeLLM([ANSWER, ANSWER, ANSWER, ANSWER])
+ with (
+ patch(GATEWAY, return_value=llm),
+ patch("posthog.api.embedding_worker.emit_embedding_request"),
+ patch.object(tick, "ph_background_capture", return_value=MagicMock()),
+ ):
+ result = run_backfill(self._jobs(4), BackfillOptions(), max_cost_usd=1e-9)
+
+ assert (result.attempted, result.stopped_at_cost_cap) == (1, True)
+ assert len(llm.requests) == 1
diff --git a/products/signals/backend/test/test_activity_summary_render.py b/products/signals/backend/test/test_activity_summary_render.py
index 614a821f2a8b..77d14a144757 100644
--- a/products/signals/backend/test/test_activity_summary_render.py
+++ b/products/signals/backend/test/test_activity_summary_render.py
@@ -33,6 +33,7 @@ from products.signals.backend.activity_summary.render import (
ActivityItem,
Drop,
Parent,
+ Product,
RenderLimits,
TaskItem,
Unit,
@@ -45,6 +46,7 @@ from products.signals.backend.activity_summary.render import (
sampling_seed,
task_units,
view_units,
+ without_products,
)
from products.signals.backend.activity_summary.summaries import PIPELINE_DISTINCT_ID, SummaryPrices, cost_usd, summarize
@@ -426,6 +428,20 @@ def test_activity_units_spread_a_burst_across_days(_name: str, older_days: int,
assert drops[Drop.BURST_CAP] + drops[Drop.SAMPLE_CAP] == len(burst) + len(older) - ACTIVITY_LINES
+def test_without_products_drops_inbox_rows_and_views_and_counts_them() -> None:
+ drops: Counter[str] = Counter()
+ activity, views = without_products(
+ [_change("SignalReport", "updated", "Weekly digest"), _change("Insight", "created", "Signup funnel")],
+ [_view("scene", "Inbox"), _view("scene", "Dashboards"), _view("insight", "abc", "Signup funnel")],
+ frozenset({Product.INBOX}),
+ drops,
+ )
+
+ assert [item.scope for item in activity] == ["Insight"]
+ assert [view.ref for view in views] == ["Dashboards", "abc"]
+ assert drops == {Drop.EXCLUDED_PRODUCT: 2}
+
+
def test_burst_cap_keeps_the_newest_units_of_a_day() -> None:
drops: Counter[str] = Counter()
units = [Unit(at=_at(1, minutes=minute), text=f"unit {minute}") for minute in range(MAX_UNITS_PER_DAY + 2)]
diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.py
index 1e42a2f31da6..d584629e4018 100644
--- a/products/signals/backend/test/test_activity_summary_update.py
+++ b/products/signals/backend/test/test_activity_summary_update.py
@@ -14,9 +14,14 @@ from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.product_analytics.backend.facade.models import Insight
from products.signals.backend.activity_summary.inputs import user_activity
-from products.signals.backend.activity_summary.logic import ActivitySummaryUpdate, UpdateStatus, update_activity_summary
+from products.signals.backend.activity_summary.logic import (
+ ActivitySummaryUpdate,
+ UpdateStatus,
+ build_summary_as_of,
+ update_activity_summary,
+)
from products.signals.backend.activity_summary.prompt import PROMPT_VERSION
-from products.signals.backend.activity_summary.render import RENDER_VERSION, RenderLimits
+from products.signals.backend.activity_summary.render import RENDER_VERSION, Product, RenderLimits
from products.signals.backend.models import UserActivitySummary, UserActivitySummaryVersion
from products.signals.backend.test.test_activity_summary_render import FakeLLM
@@ -363,3 +368,32 @@ class TestUpdateActivitySummary(BaseTest):
assert self._update(llm).status == UpdateStatus.SKIPPED_AI_NOT_APPROVED
assert llm.requests == []
+
+ def test_a_build_as_of_a_past_date_reads_only_the_rows_before_it_and_writes_nothing(self) -> None:
+ self._changes(3, days_ago=11)
+ self._change("Weekly digest", days_ago=10.5, scope="SignalReport")
+ self._change("Funnel after as of", days_ago=9)
+ self._change("Funnel at as of", days_ago=10)
+ self._update(FakeLLM([ANSWER]))
+ stored = self._stored()
+ assert stored is not None
+ llm = FakeLLM([ANSWER])
+
+ with patch("products.signals.backend.activity_summary.logic.gateway_client", return_value=llm):
+ result = build_summary_as_of(
+ self.team.id,
+ self.user.id,
+ NOW - datetime.timedelta(days=10),
+ exclude_products=frozenset({Product.INBOX}),
+ )
+
+ assert result.status == UpdateStatus.OK
+ sent = json.dumps(llm.requests[0]["messages"])
+ assert "Funnel 2" in sent
+ assert "Funnel after as of" not in sent and "Funnel at as of" not in sent
+ assert "Weekly digest" not in sent
+ assert "Builds the signup funnel." not in sent
+ assert result.inputs_used is not None and result.inputs_used["drops"]["excluded_product"] == 1
+ after = self._stored()
+ assert after is not None and (after.version, after.updated_at) == (stored.version, stored.updated_at)
+ assert self._versions() == [1]