Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): move activity summary size limits into the flag payload (#121323)

posthog[bot]8a1d015eed15
diff --git a/products/signals/backend/activity_summary/config.py b/products/signals/backend/activity_summary/config.py
index a9c5552a7542..23283d95ed49 100644
--- a/products/signals/backend/activity_summary/config.py
+++ b/products/signals/backend/activity_summary/config.py
@@ -4,6 +4,7 @@ The parse fails closed: a missing, malformed or unreadable payload turns the tic
field falls back to its default, and no default is unbounded.
"""
+import dataclasses
from typing import Any
import structlog
@@ -11,6 +12,7 @@ import structlog
from posthog.dataclasses import frozen
from products.signals.backend.activity_summary.logic import MIN_UNITS, MIN_UPDATE_INTERVAL
+from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, RenderLimits
from products.signals.backend.scout_harness.team_limits import read_flag_payload
logger = structlog.get_logger(__name__)
@@ -34,6 +36,7 @@ class ActivitySummaryConfig:
min_update_interval_hours: float = MIN_UPDATE_INTERVAL.total_seconds() / 3600
emit_embeddings: bool = True
capture_events: bool = True
+ limits: RenderLimits = DEFAULT_LIMITS
DISABLED = ActivitySummaryConfig()
@@ -64,6 +67,20 @@ def _organization_ids(value: Any) -> tuple[str, ...]:
return tuple(item for item in value if isinstance(item, str) and item) if isinstance(value, list) else ()
+def parse_limits(payload: Any) -> RenderLimits:
+ """The `limits` object of the payload. An invalid field gets its default, a large one its `max_value`."""
+ if not isinstance(payload, dict):
+ return DEFAULT_LIMITS
+ return RenderLimits(
+ **{
+ field.name: min(
+ _positive_int(payload.get(field.name), getattr(DEFAULT_LIMITS, field.name)), field.metadata["max_value"]
+ )
+ for field in dataclasses.fields(RenderLimits)
+ }
+ )
+
+
def parse_config(payload: Any) -> ActivitySummaryConfig:
if not isinstance(payload, dict) or payload.get("enabled") is not True:
return DISABLED
@@ -82,6 +99,7 @@ def parse_config(payload: Any) -> ActivitySummaryConfig:
),
emit_embeddings=_bool(payload.get("emit_embeddings"), defaults.emit_embeddings),
capture_events=_bool(payload.get("capture_events"), defaults.capture_events),
+ limits=parse_limits(payload.get("limits")),
)
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index 108488d6315a..8380ce2160ce 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -1,6 +1,7 @@
"""One activity summary update for one (team, user): gather, filter, render, summarize, persist."""
import datetime
+import dataclasses
from enum import StrEnum
from typing import Any
@@ -13,7 +14,13 @@ from posthog.models import Team
from products.signals.backend.activity_summary.inputs import user_activity, user_tasks
from products.signals.backend.activity_summary.prompt import PROMPT_VERSION, summary_text
-from products.signals.backend.activity_summary.render import RENDER_VERSION, WINDOW_DAYS, build_input, sampling_seed
+from products.signals.backend.activity_summary.render import (
+ DEFAULT_LIMITS,
+ RENDER_VERSION,
+ RenderLimits,
+ build_input,
+ sampling_seed,
+)
from products.signals.backend.activity_summary.summaries import (
DEFAULT_MODEL,
MODEL_PRICES,
@@ -65,8 +72,8 @@ class ActivitySummaryUpdate:
input_watermark: datetime.datetime | None = None
-def _input_start(watermark: datetime.datetime | None, now: datetime.datetime) -> datetime.datetime:
- earliest = now - datetime.timedelta(days=WINDOW_DAYS)
+def _input_start(watermark: datetime.datetime | None, now: datetime.datetime, window_days: int) -> datetime.datetime:
+ earliest = now - datetime.timedelta(days=window_days)
return earliest if watermark is None else max(earliest, watermark + _WATERMARK_STEP)
@@ -142,10 +149,11 @@ def update_activity_summary(
force: bool = False,
min_units: int = MIN_UNITS,
min_update_interval: datetime.timedelta = MIN_UPDATE_INTERVAL,
+ limits: RenderLimits = DEFAULT_LIMITS,
) -> ActivitySummaryUpdate:
"""Update the stored activity summary of one user in one project.
- Reads only the inputs newer than the stored watermark, or the last `WINDOW_DAYS` on a first
+ Reads only the inputs newer than the stored watermark, or the last `limits.window_days` on a first
run. A failed update records the error and keeps the stored summary and the watermark. With
`dry_run`, the model is still called but nothing is written. `force` ignores `next_eligible_at`.
A skip for too few units and a failed update set `next_eligible_at` to `RETRY_BACKOFF` from now.
@@ -159,10 +167,12 @@ def update_activity_summary(
if not force and stored is not None and stored.next_eligible_at is not None and stored.next_eligible_at > now:
return ActivitySummaryUpdate(status=UpdateStatus.SKIPPED_NOT_ELIGIBLE_YET)
- start = _input_start(stored.input_watermark if stored else None, now)
+ start = _input_start(stored.input_watermark if stored else None, now, limits.window_days)
tasks = user_tasks(team_id, user_id, start, now)
activity = user_activity(team_id, user_id, start, now)
- rendered = build_input(tasks, activity, as_of=now, seed=sampling_seed(now.date().isoformat(), team_id, user_id))
+ rendered = build_input(
+ tasks, activity, as_of=now, seed=sampling_seed(now.date().isoformat(), team_id, user_id), limits=limits
+ )
if rendered is None or rendered.units < min_units:
if not dry_run:
_back_off(team_id, user_id, stored, now + RETRY_BACKOFF)
@@ -177,6 +187,7 @@ def update_activity_summary(
"activity_lines": rendered.n_activity_lines,
"input_chars": rendered.input_chars,
"drops": rendered.drops,
+ "limits": dataclasses.asdict(limits),
}
watermark = max([task.at for task in tasks] + [item.at for item in activity])
previous = stored.summary if stored is not None else None
@@ -187,6 +198,8 @@ def update_activity_summary(
previous_summary=previous,
model=model,
as_of=now.date(),
+ max_completion_tokens=limits.max_completion_tokens,
+ summary_max_words=limits.summary_max_words,
)
cost = cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(model))
result = ActivitySummaryUpdate(
diff --git a/products/signals/backend/activity_summary/prompt.py b/products/signals/backend/activity_summary/prompt.py
index 8f8a6d325548..a76e36b04f4d 100644
--- a/products/signals/backend/activity_summary/prompt.py
+++ b/products/signals/backend/activity_summary/prompt.py
@@ -23,14 +23,19 @@ DEFAULT_STRENGTH = "low"
# Strings a model writes in place of a JSON null.
_NULL_STRINGS = frozenset({"", "none", "null", "n/a", "na", "nil"})
-SYSTEM_PROMPT = """You maintain a short summary of what one person has been working on in one PostHog project.
+_SYSTEM_PROMPT = """You maintain a short summary of what one person has been working on in one PostHog project.
You get their previous summary, if any, and a sample of their recent activity in up to two sections: tasks they gave to PostHog AI or the Slack agent, and things they created or changed in PostHog, with the reason their coding agent gave when an agent made the change.
Update the summary to reflect what they are focused on now. Keep what is still true, drop what has gone quiet, and add what is new. Weigh themes that repeat across sections above one-off items. Treat bulk changes of the same kind as one theme, not a list.
Name concrete products, features, metrics, pipelines and workflows. Never write generic phrases like "data analysis".
Write about the work, not the person: no names, emails, IDs, secrets, or quotes from the input. Open with the work itself.
-Return JSON: `summary` (at most 60 words), `focus_areas` (up to 6, each with `topic`, `strength` high, medium or low, and `last_seen` as YYYY-MM-DD), `products`, and `recent_shift` (one sentence, or null).
+Return JSON: `summary` (at most {max_words} words), `focus_areas` (up to 6, each with `topic`, `strength` high, medium or low, and `last_seen` as YYYY-MM-DD), `products`, and `recent_shift` (one sentence, or null).
Set `recent_shift` only when there is a previous summary and the activity shows a real change from it. Otherwise it is null."""
+
+def system_prompt(max_words: int = SUMMARY_MAX_WORDS) -> str:
+ return _SYSTEM_PROMPT.format(max_words=max_words)
+
+
# Strict structured output. The normalization below still checks every rule, because a provider
# behind the gateway may not enforce the schema.
RESPONSE_FORMAT: Any = {
@@ -141,7 +146,9 @@ def _products(value: Any) -> list[str]:
return products[:MAX_PRODUCTS]
-def normalize_summary(content: str | None, *, had_previous_summary: bool) -> dict[str, Any]:
+def normalize_summary(
+ content: str | None, *, had_previous_summary: bool, max_words: int = SUMMARY_MAX_WORDS
+) -> dict[str, Any]:
"""The model's JSON answer under the output rules, or `MalformedSummaryError`.
Only a missing or empty `summary` makes an answer malformed. Every other field degrades to an
@@ -159,7 +166,7 @@ def normalize_summary(content: str | None, *, had_previous_summary: bool) -> dic
raise MalformedSummaryError("response has no summary")
recent_shift = _clean(parsed.get("recent_shift"), MAX_RECENT_SHIFT_CHARS) if had_previous_summary else None
return {
- "summary": cap_words(summary),
+ "summary": cap_words(summary, max_words),
"focus_areas": _focus_areas(parsed.get("focus_areas")),
"products": _products(parsed.get("products")),
"recent_shift": recent_shift,
diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.py
index 4202500f4c7a..f5016d87a970 100644
--- a/products/signals/backend/activity_summary/queue.py
+++ b/products/signals/backend/activity_summary/queue.py
@@ -16,20 +16,15 @@ from posthog.models import ActivityLog, Team
from products.signals.backend.activity_summary.config import ActivitySummaryConfig
from products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS
from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX
-from products.signals.backend.activity_summary.prompt import SYSTEM_PROMPT
+from products.signals.backend.activity_summary.prompt import system_prompt
from products.signals.backend.activity_summary.render import (
+ DEFAULT_LIMITS,
+ MAX_LIMITS,
MAX_SUMMARY_CHARS,
SCOUT_CLIENT_PREFIX,
- SECTIONS_MAX_CHARS,
- WINDOW_DAYS,
-)
-from products.signals.backend.activity_summary.summaries import (
- DEFAULT_MODEL,
- MAX_ATTEMPTS,
- MAX_COMPLETION_TOKENS,
- MODEL_PRICES,
- cost_usd,
+ RenderLimits,
)
+from products.signals.backend.activity_summary.summaries import DEFAULT_MODEL, MAX_ATTEMPTS, MODEL_PRICES, cost_usd
from products.signals.backend.models import UserActivitySummary
from products.tasks.backend.facade.user_task_summaries import list_user_task_times
@@ -60,12 +55,12 @@ class DailyUsage:
cost_usd: float
-def max_update_cost_usd(model: str = DEFAULT_MODEL) -> float:
+def max_update_cost_usd(limits: RenderLimits = DEFAULT_LIMITS, model: str = DEFAULT_MODEL) -> float:
"""The most one update can cost: every attempt with the largest input and output."""
- input_chars = SECTIONS_MAX_CHARS + MAX_SUMMARY_CHARS + len(SYSTEM_PROMPT)
+ input_chars = limits.sections_max_chars + MAX_SUMMARY_CHARS + len(system_prompt(limits.summary_max_words))
bound = cost_usd(
MAX_ATTEMPTS * input_chars // _BOUND_CHARS_PER_TOKEN,
- MAX_ATTEMPTS * MAX_COMPLETION_TOKENS,
+ MAX_ATTEMPTS * limits.max_completion_tokens,
MODEL_PRICES.get(model),
)
return _UNPRICED_UPDATE_COST_USD if bound is None else bound
@@ -76,7 +71,8 @@ def daily_usage(now: datetime.datetime) -> DailyUsage:
One row holds the latest update of a pair, so an earlier update of the same pair in the same 24
hours is not counted. With the minimum update interval that happens only after a failure.
- A row without a cost counts as the most one update can cost, so missing usage cannot pass the cap.
+ A row without a cost counts as the most one update can cost under the largest limits the flag allows,
+ because the row can come from a tick with larger limits than now. So missing usage cannot pass the cap.
An update that raised before it called the model is not counted, unless the row holds an earlier paid update.
"""
rows = (
@@ -91,7 +87,7 @@ def daily_usage(now: datetime.datetime) -> DailyUsage:
)
return DailyUsage(
updates=rows["updates"],
- cost_usd=(rows["cost"] or 0.0) + rows["unpriced"] * max_update_cost_usd(),
+ cost_usd=(rows["cost"] or 0.0) + rows["unpriced"] * max_update_cost_usd(MAX_LIMITS),
)
@@ -171,13 +167,13 @@ def select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: i
"""The due pairs of the enrolled teams, highest priority first, cut to `limit`.
A pair is due when it has a task or an activity log row after its watermark, or in the last
- `WINDOW_DAYS` with no watermark, and its `next_eligible_at` is empty or past. Pairs with no
+ `window_days` with no watermark, and its `next_eligible_at` is empty or past. Pairs with no
summary come first, then the pairs with the most new rows, then the oldest summaries.
"""
team_ids = _enrolled_team_ids(config)
if not team_ids or limit <= 0:
return []
- window_start = now - datetime.timedelta(days=WINDOW_DAYS)
+ window_start = now - datetime.timedelta(days=config.limits.window_days)
stored = _stored_rows(team_ids)
new_rows = Counter(_new_activity_rows(team_ids, window_start, now)) + _new_tasks(
team_ids, window_start, now, stored
diff --git a/products/signals/backend/activity_summary/render.py b/products/signals/backend/activity_summary/render.py
index 7c169ab95af0..30f1d27db8c4 100644
--- a/products/signals/backend/activity_summary/render.py
+++ b/products/signals/backend/activity_summary/render.py
@@ -13,14 +13,19 @@ import math
import random
import hashlib
import datetime
+import dataclasses
from collections import Counter
from collections.abc import Iterable, Sequence
from enum import StrEnum
+from typing import Any
from urllib.parse import urlsplit
from posthog.dataclasses import frozen
-# Bump on any change to the filters, the caps, the sampling or the rendered layout, so summaries stay comparable.
+from products.signals.backend.activity_summary.prompt import SUMMARY_MAX_WORDS
+
+# Bump on any code change to the filters, the default caps, the sampling or the rendered layout, so summaries
+# stay comparable. Limits from the flag payload go into `inputs_used` instead.
RENDER_VERSION = 4
WINDOW_DAYS = 28
@@ -36,8 +41,8 @@ MAX_ACTIVITY_ROWS_PER_PERSON = 1000
MAX_UNITS_PER_DAY = 3
TASK_LINES = 12
ACTIVITY_LINES = 15
-# Per-line caps, so one runaway title or intent cannot take the budget. With them a full prompt
-# section stays under SECTIONS_MAX_CHARS; the hard stop is the backstop if the caps change.
+# Per-line caps, so one runaway title or intent cannot take the budget. With the default caps a full
+# prompt section stays under SECTIONS_MAX_CHARS; the hard stop is the backstop when the caps change.
TASK_LINE_CHARS = 260
TASK_GOAL_CHARS = 180
TASK_TITLE_CHARS = 100
@@ -47,6 +52,8 @@ ACTIVITY_LINE_CHARS = 220
ACTIVITY_NAME_CHARS = 70
ACTIVITY_INTENT_CHARS = 100
SECTIONS_MAX_CHARS = 8000
+# Hidden reasoning tokens count against the cap too, so a cap near the answer size truncates answers.
+MAX_COMPLETION_TOKENS = 4000
# Masking and whitespace folding shorten a text, so the patterns read this many times its cap.
RAW_TEXT_FACTOR = 4
@@ -84,6 +91,33 @@ _LETTER = re.compile(r"[A-Za-z]")
_WHITESPACE = re.compile(r"\s+")
+def _limit(default: int, *, max_value: int) -> Any:
+ return dataclasses.field(default=default, metadata={"max_value": max_value})
+
+
+@frozen
+class RenderLimits:
+ """The limits that shape one summary. The `activity-summary` flag payload can override each one.
+
+ `max_value` clamps a payload value, so a typo cannot make one update slow or expensive."""
+
+ window_days: int = _limit(WINDOW_DAYS, max_value=90)
+ task_lines: int = _limit(TASK_LINES, max_value=50)
+ activity_lines: int = _limit(ACTIVITY_LINES, max_value=50)
+ max_units_per_day: int = _limit(MAX_UNITS_PER_DAY, max_value=20)
+ sections_max_chars: int = _limit(SECTIONS_MAX_CHARS, max_value=32000)
+ task_line_chars: int = _limit(TASK_LINE_CHARS, max_value=1000)
+ task_goal_chars: int = _limit(TASK_GOAL_CHARS, max_value=800)
+ activity_line_chars: int = _limit(ACTIVITY_LINE_CHARS, max_value=1000)
+ activity_intent_chars: int = _limit(ACTIVITY_INTENT_CHARS, max_value=500)
+ summary_max_words: int = _limit(SUMMARY_MAX_WORDS, max_value=300)
+ max_completion_tokens: int = _limit(MAX_COMPLETION_TOKENS, max_value=16000)
+
+
+DEFAULT_LIMITS = RenderLimits()
+MAX_LIMITS = RenderLimits(**{field.name: field.metadata["max_value"] for field in dataclasses.fields(RenderLimits)})
+
+
class Drop(StrEnum):
"""Why a unit did not reach the prompt."""
@@ -199,15 +233,15 @@ def _day(at: datetime.datetime) -> datetime.date:
return at.astimezone(datetime.UTC).date()
-def summary_goal(summary: str) -> str:
+def summary_goal(summary: str, max_chars: int = TASK_GOAL_CHARS) -> str:
"""The opening sentences of a task summary, which state what was asked, without the label."""
text = _SUMMARY_LABEL.sub("", clean_text(bounded_raw(summary, MAX_SUMMARY_CHARS), MAX_SUMMARY_CHARS))
goal = ""
for sentence in _SENTENCE_END.split(text):
- if goal and len(goal) + len(sentence) + 1 > TASK_GOAL_CHARS:
+ if goal and len(goal) + len(sentence) + 1 > max_chars:
break
goal = f"{goal} {sentence}".strip()
- return cap_line(goal, TASK_GOAL_CHARS)
+ return cap_line(goal, max_chars)
def _templated_label(title: str) -> str | None:
@@ -217,7 +251,7 @@ def _templated_label(title: str) -> str | None:
return None
-def task_units(tasks: Iterable[TaskItem], drops: Counter[str]) -> list[Unit]:
+def task_units(tasks: Iterable[TaskItem], drops: Counter[str], limits: RenderLimits = DEFAULT_LIMITS) -> list[Unit]:
"""One line per task, with templated titles collapsed to one line per template."""
units: list[Unit] = []
templated: dict[str, list[datetime.datetime]] = {}
@@ -233,22 +267,27 @@ def task_units(tasks: Iterable[TaskItem], drops: Counter[str]) -> list[Unit]:
if task.summary:
tags = [cap_line(clean_text(tag, MAX_TAG_CHARS * 2), MAX_TAG_CHARS) for tag in task.tags[:MAX_TAGS]]
tag_text = f"[{', '.join(tag for tag in tags if tag)}] " if any(tags) else ""
- line = f"- {day} {tag_text}{summary_goal(task.summary)}"
+ line = f"- {day} {tag_text}{summary_goal(task.summary, limits.task_goal_chars)}"
elif len(title.split()) >= MIN_BARE_TITLE_WORDS:
line = f"- {day} {cap_line(clean_text(title, TASK_TITLE_CHARS * 2), TASK_TITLE_CHARS)}"
else:
drops[Drop.BARE_TITLE] += 1
continue
- units.append(Unit(at=task.at, text=cap_line(line, TASK_LINE_CHARS)))
+ units.append(Unit(at=task.at, text=cap_line(line, limits.task_line_chars)))
for label, times in templated.items():
latest = max(times)
units.append(
- Unit(at=latest, text=cap_line(f"- {_day(latest).isoformat()} {label} (x{len(times)})", TASK_LINE_CHARS))
+ Unit(
+ at=latest,
+ text=cap_line(f"- {_day(latest).isoformat()} {label} (x{len(times)})", limits.task_line_chars),
+ )
)
return units
-def activity_units(items: Iterable[ActivityItem], drops: Counter[str]) -> list[Unit]:
+def activity_units(
+ items: Iterable[ActivityItem], drops: Counter[str], limits: RenderLimits = DEFAULT_LIMITS
+) -> list[Unit]:
"""Changes collapsed to one line per object, creates first, then the most changed, then the newest.
A line carries the newest agent intent of its rows. Lines are already collapsed per object, so
@@ -275,10 +314,10 @@ def activity_units(items: Iterable[ActivityItem], drops: Counter[str]) -> list[U
-max(row.at for row in entry[1]).timestamp(),
),
)
- if len(ranked) > ACTIVITY_LINES:
- drops[Drop.SAMPLE_CAP] += len(ranked) - ACTIVITY_LINES
+ if len(ranked) > limits.activity_lines:
+ drops[Drop.SAMPLE_CAP] += len(ranked) - limits.activity_lines
units: list[Unit] = []
- for (scope, activity, name), rows in ranked[:ACTIVITY_LINES]:
+ for (scope, activity, name), rows in ranked[: limits.activity_lines]:
latest = max(row.at for row in rows)
count = f" (x{len(rows)})" if len(rows) > 1 else ""
name_text = cap_line(clean_text(name, ACTIVITY_NAME_CHARS * 2), ACTIVITY_NAME_CHARS)
@@ -286,10 +325,10 @@ def activity_units(items: Iterable[ActivityItem], drops: Counter[str]) -> list[U
intents = [row for row in rows if row.intent and row.intent.strip()]
if intents:
intent = max(intents, key=lambda row: row.at).intent or ""
- intent_text = cap_line(clean_text(intent, ACTIVITY_INTENT_CHARS * 2), ACTIVITY_INTENT_CHARS)
+ intent_text = cap_line(clean_text(intent, limits.activity_intent_chars * 2), limits.activity_intent_chars)
if intent_text:
line += f" (agent: {intent_text})"
- units.append(Unit(at=latest, text=cap_line(line, ACTIVITY_LINE_CHARS)))
+ units.append(Unit(at=latest, text=cap_line(line, limits.activity_line_chars)))
return sorted(units, key=lambda unit: (unit.at, unit.text))
@@ -304,12 +343,12 @@ def dedupe_latest(units: Iterable[Unit], drops: Counter[str]) -> list[Unit]:
return sorted(latest.values(), key=lambda unit: (unit.at, unit.text))
-def burst_cap(units: Sequence[Unit], drops: Counter[str]) -> list[Unit]:
- """At most `MAX_UNITS_PER_DAY` units per UTC day, newest first, in chronological order."""
+def burst_cap(units: Sequence[Unit], drops: Counter[str], max_per_day: int = MAX_UNITS_PER_DAY) -> list[Unit]:
+ """At most `max_per_day` units per UTC day, newest first, in chronological order."""
kept: list[Unit] = []
per_day: Counter[datetime.date] = Counter()
for unit in sorted(units, key=lambda unit: (unit.at, unit.text), reverse=True):
- if per_day[_day(unit.at)] >= MAX_UNITS_PER_DAY:
+ if per_day[_day(unit.at)] >= max_per_day:
drops[Drop.BURST_CAP] += 1
continue
per_day[_day(unit.at)] += 1
@@ -352,9 +391,11 @@ def _sections_chars(sections: Sequence[tuple[str, list[Unit]]]) -> int:
return sum(len(_section(header, units)) + 2 for header, units in sections)
-def enforce_budget(sections: list[tuple[str, list[Unit]]], drops: Counter[str]) -> None:
- """Drop the oldest unit across sections until the rendered sections fit `SECTIONS_MAX_CHARS`."""
- while _sections_chars(sections) > SECTIONS_MAX_CHARS:
+def enforce_budget(
+ sections: list[tuple[str, list[Unit]]], drops: Counter[str], max_chars: int = SECTIONS_MAX_CHARS
+) -> None:
+ """Drop the oldest unit across sections until the rendered sections fit `max_chars`, or no unit is left."""
+ while _sections_chars(sections) > max_chars and any(units for _, units in sections):
oldest = min((units[0].at, index) for index, (_, units) in enumerate(sections) if units)
sections[oldest[1]][1].pop(0)
drops[Drop.SECTIONS_BUDGET] += 1
@@ -366,22 +407,23 @@ def build_input(
*,
as_of: datetime.datetime,
seed: int,
+ limits: RenderLimits = DEFAULT_LIMITS,
) -> RenderedInput | None:
"""The rendered input of one (project, person) pair, or None when nothing survives the filters.
Order: the named filters, then the burst cap, then the recency-weighted sample, then the hard
budget, so the sample is drawn from clean units only."""
drops: Counter[str] = Counter()
- task_pool = burst_cap(dedupe_latest(task_units(tasks, drops), drops), drops)
- activity_lines = activity_units(activity, drops)
+ task_pool = burst_cap(dedupe_latest(task_units(tasks, drops, limits), drops), drops, limits.max_units_per_day)
+ activity_lines = activity_units(activity, drops, limits)
# Only `activity_units` has counted a sample cap drop so far.
activity_pool = len(activity_lines) + drops[Drop.SAMPLE_CAP]
if not task_pool and not activity_lines:
return None
- chosen_tasks = sample_units(task_pool, limit=TASK_LINES, as_of=as_of, seed=seed)
+ chosen_tasks = sample_units(task_pool, limit=limits.task_lines, as_of=as_of, seed=seed)
drops[Drop.SAMPLE_CAP] += len(task_pool) - len(chosen_tasks)
sections = [(TASKS_HEADER, chosen_tasks), (ACTIVITY_HEADER, activity_lines)]
- enforce_budget(sections, drops)
+ enforce_budget(sections, drops, limits.sections_max_chars)
rendered = [_section(header, units) for header, units in sections]
text = "\n\n".join(part for part in rendered if part)
return RenderedInput(
diff --git a/products/signals/backend/activity_summary/summaries.py b/products/signals/backend/activity_summary/summaries.py
index 9c279d645e2b..5e47873eb3b8 100644
--- a/products/signals/backend/activity_summary/summaries.py
+++ b/products/signals/backend/activity_summary/summaries.py
@@ -16,11 +16,13 @@ from posthog.llm.gateway_client import build_openai_client, trace_id_from_seed
from products.signals.backend.activity_summary.prompt import (
RESPONSE_FORMAT,
- SYSTEM_PROMPT,
+ SUMMARY_MAX_WORDS,
MalformedSummaryError,
normalize_summary,
+ system_prompt,
user_message,
)
+from products.signals.backend.activity_summary.render import MAX_COMPLETION_TOKENS
AI_PRODUCT = "activity_summary"
# Not a person: the gateway attributes every call to the pipeline.
@@ -29,8 +31,6 @@ DEFAULT_MODEL = "gpt-6-luna"
# One retry on a malformed or truncated answer. The SDK already retries a failed request on its own.
MAX_ATTEMPTS = 2
REQUEST_TIMEOUT_SECONDS = 120.0
-# A full answer is about 300 tokens. The cap bounds the cost of a model that does not stop.
-MAX_COMPLETION_TOKENS = 600
TRUNCATED_FINISH_REASON = "length"
@@ -88,10 +88,12 @@ def summarize(
previous_summary: dict[str, Any] | None,
model: str,
as_of: datetime.date,
+ max_completion_tokens: int = MAX_COMPLETION_TOKENS,
+ summary_max_words: int = SUMMARY_MAX_WORDS,
) -> SummaryOutcome:
"""One summary. Tokens and latency add up over the attempts, because the retry costs too."""
messages: list[ChatCompletionMessageParam] = [
- {"role": "system", "content": SYSTEM_PROMPT},
+ {"role": "system", "content": system_prompt(summary_max_words)},
{"role": "user", "content": user_message(rendered_input, previous_summary, as_of)},
]
tokens_in: int | None = None
@@ -105,7 +107,7 @@ def summarize(
messages=messages,
user=PIPELINE_DISTINCT_ID,
response_format=RESPONSE_FORMAT,
- max_completion_tokens=MAX_COMPLETION_TOKENS,
+ max_completion_tokens=max_completion_tokens,
timeout=REQUEST_TIMEOUT_SECONDS,
)
except OpenAIError as api_error:
@@ -121,6 +123,7 @@ def summarize(
summary = normalize_summary(
choice.message.content if choice is not None else None,
had_previous_summary=previous_summary is not None,
+ max_words=summary_max_words,
)
except MalformedSummaryError as malformed:
error = f"malformed_output: {malformed}"
diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.py
index 26533a2466b6..b4cb7bd22a4c 100644
--- a/products/signals/backend/activity_summary/tick.py
+++ b/products/signals/backend/activity_summary/tick.py
@@ -40,7 +40,7 @@ from products.signals.backend.activity_summary.queue import (
max_update_cost_usd,
select_pairs,
)
-from products.signals.backend.activity_summary.render import RENDER_VERSION
+from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, RENDER_VERSION, RenderLimits
from products.signals.backend.activity_summary.summaries import DEFAULT_MODEL, PIPELINE_DISTINCT_ID
from products.signals.backend.models import UserActivitySummary
from products.signals.backend.report_embeddings import EMBEDDING_PRODUCT
@@ -86,6 +86,8 @@ class ActivitySummaryTickPlan:
emit_embeddings: bool
capture_events: bool
stop_reason: str | None
+ # A default, so a plan that a workflow history recorded before this field existed still decodes.
+ limits: RenderLimits = DEFAULT_LIMITS
@frozen
@@ -96,6 +98,8 @@ class ActivitySummaryPairInput:
min_update_interval_hours: float
emit_embeddings: bool
capture_events: bool
+ # A default, so a pair input that a tick queued before this field existed still decodes.
+ limits: RenderLimits = DEFAULT_LIMITS
@frozen
@@ -145,12 +149,13 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
candidates=len(due),
updates_left=updates_left,
cost_left_usd=cost_left,
- max_update_cost_usd=max_update_cost_usd(),
+ max_update_cost_usd=max_update_cost_usd(config.limits),
max_concurrency=config.max_concurrency,
min_units=config.min_units,
min_update_interval_hours=config.min_update_interval_hours,
emit_embeddings=config.emit_embeddings,
capture_events=config.capture_events,
+ limits=config.limits,
stop_reason=stop_reason,
)
@@ -213,6 +218,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A
input.user_id,
min_units=input.min_units,
min_update_interval=datetime.timedelta(hours=input.min_update_interval_hours),
+ limits=input.limits,
)
called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)
except Exception as error:
@@ -326,6 +332,7 @@ class ActivitySummaryTickWorkflow:
min_update_interval_hours=plan.min_update_interval_hours,
emit_embeddings=plan.emit_embeddings,
capture_events=plan.capture_events,
+ limits=plan.limits,
),
start_to_close_timeout=_PAIR_TIMEOUT,
heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,
diff --git a/products/signals/backend/management/commands/run_activity_summary.py b/products/signals/backend/management/commands/run_activity_summary.py
index 2589b8b140cc..6fffd03cb490 100644
--- a/products/signals/backend/management/commands/run_activity_summary.py
+++ b/products/signals/backend/management/commands/run_activity_summary.py
@@ -7,6 +7,7 @@ import json
from django.core.management.base import BaseCommand
+from products.signals.backend.activity_summary.config import parse_limits
from products.signals.backend.activity_summary.logic import update_activity_summary
@@ -18,10 +19,20 @@ class Command(BaseCommand):
parser.add_argument("--user-id", type=int, required=True)
parser.add_argument("--dry-run", action="store_true", help="Call the model, but write nothing.")
parser.add_argument("--force", action="store_true", help="Ignore next_eligible_at.")
+ parser.add_argument(
+ "--limits",
+ type=json.loads,
+ default=None,
+ help="Render limits as JSON, with the clamps of the flag payload, e.g. '{\"max_completion_tokens\": 2000}'.",
+ )
def handle(self, *args, **options):
result = update_activity_summary(
- options["team_id"], options["user_id"], dry_run=options["dry_run"], force=options["force"]
+ options["team_id"],
+ options["user_id"],
+ dry_run=options["dry_run"],
+ force=options["force"],
+ limits=parse_limits(options["limits"]),
)
self.stdout.write(f"status: {result.status}")
if result.rendered_input is not None:
diff --git a/products/signals/backend/test/test_activity_summary_render.py b/products/signals/backend/test/test_activity_summary_render.py
index 1a7824b02558..e7add5b02aa5 100644
--- a/products/signals/backend/test/test_activity_summary_render.py
+++ b/products/signals/backend/test/test_activity_summary_render.py
@@ -20,15 +20,14 @@ from products.signals.backend.activity_summary.prompt import (
)
from products.signals.backend.activity_summary.render import (
ACTIVITY_HEADER,
- ACTIVITY_LINE_CHARS,
- ACTIVITY_LINES,
+ DEFAULT_LIMITS,
+ MAX_COMPLETION_TOKENS,
MAX_UNITS_PER_DAY,
- SECTIONS_MAX_CHARS,
- TASK_LINE_CHARS,
TASK_LINES,
TASKS_HEADER,
ActivityItem,
Drop,
+ RenderLimits,
TaskItem,
Unit,
activity_units,
@@ -39,13 +38,7 @@ from products.signals.backend.activity_summary.render import (
sampling_seed,
task_units,
)
-from products.signals.backend.activity_summary.summaries import (
- MAX_COMPLETION_TOKENS,
- PIPELINE_DISTINCT_ID,
- SummaryPrices,
- cost_usd,
- summarize,
-)
+from products.signals.backend.activity_summary.summaries import PIPELINE_DISTINCT_ID, SummaryPrices, cost_usd, summarize
AS_OF = datetime.datetime(2026, 10, 19, tzinfo=datetime.UTC)
PATH_SECRET = "https://hooks.example.com/services/T000/B000/s3cr3tPathValue?sig=abc"
@@ -248,7 +241,23 @@ def test_build_input_renders_the_two_sections_in_order() -> None:
assert build_input([_task("root cause")], [], as_of=AS_OF, seed=1) is None
-def test_build_input_stays_inside_its_bounds_for_a_very_active_person() -> None:
+ [
+ ("default limits", DEFAULT_LIMITS),
+ (
+ "limits from the flag",
+ RenderLimits(
+ task_lines=5,
+ activity_lines=30,
+ max_units_per_day=1,
+ sections_max_chars=20000,
+ task_line_chars=240,
+ activity_line_chars=400,
+ ),
+ ),
+ ]
+)
+def test_build_input_stays_inside_its_bounds_for_a_very_active_person(_name: str, limits: RenderLimits) -> None:
runaway = "word " * 2000
tasks = [
_task(runaway, summary=f"{day} {n} {runaway}", tags=(runaway,) * 20, days_ago=day + 0.01 * n)
@@ -257,17 +266,30 @@ def test_build_input_stays_inside_its_bounds_for_a_very_active_person() -> None:
]
activity = [_change("Insight", "created", f"{runaway} {n}", days_ago=n % 28, intent=runaway) for n in range(500)]
- rendered = build_input(tasks, activity, as_of=AS_OF, seed=1)
+ rendered = build_input(tasks, activity, as_of=AS_OF, seed=1, limits=limits)
assert rendered is not None
- assert (rendered.n_tasks, rendered.n_activity_lines) == (TASK_LINES, ACTIVITY_LINES)
+ assert (rendered.n_tasks, rendered.n_activity_lines) == (limits.task_lines, limits.activity_lines)
tasks_text, activity_text = rendered.text.split("\n\n")
- assert max(len(line) for line in tasks_text.split("\n")[1:]) <= TASK_LINE_CHARS
- assert max(len(line) for line in activity_text.split("\n")[1:]) <= ACTIVITY_LINE_CHARS
- assert rendered.input_chars <= SECTIONS_MAX_CHARS
+ assert max(len(line) for line in tasks_text.split("\n")[1:]) <= limits.task_line_chars
+ assert max(len(line) for line in activity_text.split("\n")[1:]) <= limits.activity_line_chars
+ assert rendered.input_chars <= limits.sections_max_chars
assert rendered.drops[Drop.BURST_CAP] > 0 and rendered.drops[Drop.SAMPLE_CAP] > 0
+def test_build_input_drops_every_unit_for_a_budget_below_the_section_overhead() -> None:
+ rendered = build_input(
+ [_task("why", summary="Asked: why did signups drop?", days_ago=3)],
+ [_change("Dashboard", "created", "Growth overview", days_ago=1)],
+ as_of=AS_OF,
+ seed=1,
+ limits=RenderLimits(sections_max_chars=1),
+ )
+
+ assert rendered is not None
+ assert (rendered.text, rendered.units, rendered.drops[Drop.SECTIONS_BUDGET]) == ("", 0, 2)
+
+
PREVIOUS = {"summary": "Builds the signup funnel.", "focus_areas": [], "products": ["product_analytics"]}
diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.py
index 824622eb21d9..d1c7f282806f 100644
--- a/products/signals/backend/test/test_activity_summary_tick.py
+++ b/products/signals/backend/test/test_activity_summary_tick.py
@@ -2,6 +2,7 @@ import json
import uuid
import asyncio
import datetime
+import dataclasses
from typing import Any
import pytest
@@ -14,6 +15,7 @@ from django.test import override_settings
from parameterized import parameterized
from temporalio import activity
from temporalio.client import Client, ScheduleState
+from temporalio.converter import DataConverter
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
@@ -31,6 +33,7 @@ from products.signals.backend.activity_summary.queue import (
max_update_cost_usd,
select_pairs,
)
+from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, MAX_LIMITS, RenderLimits
from products.signals.backend.activity_summary.schedule import (
ACTIVITY_SUMMARY_TICK_SCHEDULE_ID,
create_activity_summary_tick_schedule,
@@ -103,6 +106,24 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
capture_events=False,
),
),
+ (
+ "limits fall back to the defaults when invalid and clamp when too large",
+ {
+ "enabled": True,
+ "limits": {
+ "task_lines": 20,
+ "activity_lines": 0,
+ "max_units_per_day": True,
+ "sections_max_chars": "9000",
+ "window_days": 10_000,
+ "max_completion_tokens": 10_000_000,
+ },
+ },
+ ActivitySummaryConfig(
+ enabled=True, limits=RenderLimits(task_lines=20, window_days=90, max_completion_tokens=16000)
+ ),
+ ),
+ ("limits that are not an object", {"enabled": True, "limits": [5]}, ActivitySummaryConfig(enabled=True)),
]
@@ -205,7 +226,7 @@ class TestActivitySummaryQueue(BaseTest):
usage = daily_usage(NOW)
assert usage.updates == 2
- assert usage.cost_usd == pytest.approx(0.25 + max_update_cost_usd())
+ assert usage.cost_usd == pytest.approx(0.25 + max_update_cost_usd(MAX_LIMITS))
with time_machine.travel(NOW + datetime.timedelta(days=1, minutes=1), tick=False):
assert daily_usage(NOW + datetime.timedelta(days=1, minutes=1)).updates == 0
@@ -371,6 +392,7 @@ def _plan(pairs: int, **overrides: Any) -> ActivitySummaryTickPlan:
"min_update_interval_hours": 24.0,
"emit_embeddings": True,
"capture_events": True,
+ "limits": DEFAULT_LIMITS,
"stop_reason": None,
}
return ActivitySummaryTickPlan(**{**fields, **overrides})
@@ -441,3 +463,32 @@ async def test_a_disabled_tick_runs_no_pair() -> None:
result, peak, captured = await _run_tick(_plan(0, enabled=False), 0.01)
assert (result, peak, captured) == (ActivitySummaryTickResult(), 0, [])
+
+
+ [
+ ("tick plan", ActivitySummaryTickPlan, lambda: _plan(1)),
+ (
+ "pair input",
+ ActivitySummaryPairInput,
+ lambda: ActivitySummaryPairInput(
+ team_id=1,
+ user_id=2,
+ min_units=3,
+ min_update_interval_hours=24.0,
+ emit_embeddings=True,
+ capture_events=True,
+ ),
+ ),
+ ]
+)
+def test_a_payload_recorded_before_limits_existed_decodes_with_the_default_limits(
+ _name: str, cls: type, build: Any
+) -> None:
+ converter = DataConverter.default.payload_converter
+ recorded = {key: value for key, value in dataclasses.asdict(build()).items() if key != "limits"}
+
+ [decoded] = converter.from_payloads(converter.to_payloads([recorded]), [cls])
+
+ assert decoded == build()
+ assert decoded.limits == DEFAULT_LIMITS
diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.py
index 82ec2cae6902..57ff9d937513 100644
--- a/products/signals/backend/test/test_activity_summary_update.py
+++ b/products/signals/backend/test/test_activity_summary_update.py
@@ -12,6 +12,7 @@ from posthog.models import ActivityLog
from products.signals.backend.activity_summary.inputs import user_activity
from products.signals.backend.activity_summary.logic import ActivitySummaryUpdate, UpdateStatus, update_activity_summary
+from products.signals.backend.activity_summary.render import RenderLimits
from products.signals.backend.models import UserActivitySummary
from products.signals.backend.test.test_activity_summary_render import FakeLLM
@@ -86,6 +87,21 @@ class TestUpdateActivitySummary(BaseTest):
assert "Signup funnel" not in next_sent.split("Recent activity")[1]
assert "Builds the signup funnel." in next_sent
+ def test_limits_reach_the_model_call_and_the_stored_inputs(self) -> None:
+ self._changes(3, days_ago=2)
+ self._change("Old funnel", days_ago=10)
+ llm = FakeLLM([ANSWER])
+
+ self._update(llm, limits=RenderLimits(window_days=7, summary_max_words=2, max_completion_tokens=1234))
+
+ stored = self._stored()
+ assert stored is not None and stored.summary is not None
+ assert llm.requests[0]["max_completion_tokens"] == 1234
+ assert "at most 2 words" in llm.requests[0]["messages"][0]["content"]
+ assert stored.summary["summary"] == "Builds the"
+ assert stored.inputs_used["activity_lines"] == 3
+ assert stored.inputs_used["limits"]["max_completion_tokens"] == 1234
+
def test_a_failed_update_keeps_the_stored_summary_and_watermark(self) -> None:
self._changes(3, days_ago=2)
self._update(FakeLLM([ANSWER]))