Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): run personal ranking tick pairs in a sliding window (#130770)

posthog[bot]7816bdd4bb8c
diff --git a/products/signals/backend/management/AGENTS.md b/products/signals/backend/management/AGENTS.md
index e2b2c0efce75..6822f9543ea2 100644
--- a/products/signals/backend/management/AGENTS.md
+++ b/products/signals/backend/management/AGENTS.md
@@ -352,8 +352,11 @@ The `tick` object of the `personal-inbox-ranking` payload holds its levers. With
- **Unchanged inputs:** the pair activity compares the input hash first. A match calls no model and sets `checked_at` on the latest row, so the pair is not due again until `target_interval_minutes` passes, and `for_me` serves that row as fresh. An enabled tick raises `freshness_minutes` to at least `target_interval_minutes`, so a plain read does not rebuild a list inside the tick's interval. A row whose model call failed, timed out or answered outside the schema, or that skipped the model without AI approval, is built again instead.
- **Limits:** a build that calls the model spends `max_tick_builds_per_day` (one Redis token bucket per region, closed when Redis is unreachable) and then the shared `max_builds_per_day`. `max_tick_builds_per_day` is limited to 80% of `max_builds_per_day`, so `refresh` and `focus` requests keep a reserve. To run the tick above that, raise the shared limit too. The tick stops at the first refusal, at `max_error_rate` after 10 model calls, and when 60% of its interval has passed.
- **Locks:** the pair takes the same per-person lock as `for_me`, so a tick skips a person whose build is running.
+- **Concurrency:** the tick keeps `max_concurrency` pairs running and starts the next pair as soon as any pair finishes. A stop starts no further pair, and the pairs already running finish. The pairs never started stay due for the next tick.
+- **Bounds:** the payload can set `max_pairs_per_tick` up to 2000, `max_concurrency` up to 50, and `max_tick_builds_per_day` and `max_builds_per_day` up to 50000. Larger values clamp to these.
Each enabled tick captures `personal inbox ranking tick` with the due pairs, the pairs it picked, a count per pair status (`built`, `failed`, `unchanged`, `running`, `not_enrolled`, `tick_limited`, `shared_limited`), the stop reason, `oldest_due_hours`, `fresh_share`, the enrolled organizations and teams, and the duration.
+It also carries `unchanged_pair_seconds_p50`, `unchanged_pair_seconds_p95`, `built_pair_seconds_p50` and `built_pair_seconds_p95`: the seconds from the start of a pair to its result, for sizing `max_concurrency` and `max_pairs_per_tick`.
`personal inbox ranking built` and the model call's trace carry `trigger` (`tick` or `on_demand`), so cost can be split by what started a build.
## Tips
diff --git a/products/signals/backend/personal_ranking/config.py b/products/signals/backend/personal_ranking/config.py
index b71acc6b21c6..2ced2c6d3c16 100644
--- a/products/signals/backend/personal_ranking/config.py
+++ b/products/signals/backend/personal_ranking/config.py
@@ -95,9 +95,9 @@ TICK_SHARE_OF_SHARED_BUILDS = 0.8
_TICK_MAXIMUMS: dict[str, int] = {
"target_interval_minutes": 1440,
"active_days": 90,
- "max_pairs_per_tick": 500,
- "max_concurrency": 20,
- "max_tick_builds_per_day": 10000,
+ "max_pairs_per_tick": 2000,
+ "max_concurrency": 50,
+ "max_tick_builds_per_day": 50000,
}
@@ -169,7 +169,7 @@ _MAXIMUMS: dict[str, int] = {
"build_burst": 20,
"builds_per_hour": 120,
"builds_per_day": 500,
- "max_builds_per_day": 10000,
+ "max_builds_per_day": 50000,
"max_rows_per_person": 100,
"reason_max_chars": 500,
}
diff --git a/products/signals/backend/personal_ranking/tick.py b/products/signals/backend/personal_ranking/tick.py
index 3b53a9a95049..a0cb90705eb1 100644
--- a/products/signals/backend/personal_ranking/tick.py
+++ b/products/signals/backend/personal_ranking/tick.py
@@ -8,6 +8,7 @@ compares the input hash first: a match records `checked_at` on the latest row an
calls the model spends the tick's daily limit and then the shared daily limit of the region.
"""
+import math
import uuid
import asyncio
import datetime
@@ -103,8 +104,11 @@ _PAIR_TIMEOUT = datetime.timedelta(minutes=4)
_PAIR_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)
# No retry: an attempt can have called the model, and the next tick picks the pair again.
_PAIR_RETRY_POLICY = RetryPolicy(maximum_attempts=1)
-# The tick starts no wave after this share of its interval, so the last wave ends before the next tick.
+# The tick starts no pair after this share of its interval, so the last pairs end before the next tick.
_TIME_BUDGET_SHARE = 0.6
+# Ticks recorded before this marker ran pairs in waves and replay the wave loop.
+_SLIDING_WINDOW_PATCH = "personal-ranking-tick-sliding-window-2026-11"
+_TIMED_STATUSES: tuple[PairStatus, ...] = ("unchanged", "built")
@frozen
@@ -180,6 +184,43 @@ class PersonalRankingTickResult:
enrolled_organizations: int = 0
enrolled_teams: int = 0
duration_seconds: float = 0.0
+ # Seconds from the start of a pair to its result, by pair status. Empty for a tick that ran in waves.
+ pair_seconds_p50: dict[str, float] = dataclasses.field(default_factory=dict)
+ pair_seconds_p95: dict[str, float] = dataclasses.field(default_factory=dict)
+
+
+def _percentile(values: list[float], share: float) -> float:
+ ordered = sorted(values)
+ return ordered[max(0, math.ceil(share * len(ordered)) - 1)]
+
+
[email protected](frozen=False)
+class _TickTally:
+ statuses: dict[str, int] = dataclasses.field(default_factory=dict)
+ changed_parts: dict[str, int] = dataclasses.field(default_factory=dict)
+ pair_seconds: dict[str, list[float]] = dataclasses.field(default_factory=dict)
+ errored: int = 0
+ # Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.
+ calls: int = 0
+ failures: int = 0
+
+ def add(self, outcome: PersonalRankingPairResult | BaseException) -> str | None:
+ """Counts one pair result, and returns the stop reason of a limit that refused it."""
+ if isinstance(outcome, BaseException):
+ # The attempt can have called the model, so it counts as a failed call.
+ self.errored += 1
+ self.calls += 1
+ self.failures += 1
+ return None
+ self.statuses[outcome.status] = self.statuses.get(outcome.status, 0) + 1
+ for part in outcome.changed_parts:
+ self.changed_parts[part] = self.changed_parts.get(part, 0) + 1
+ self.calls += outcome.model_called
+ self.failures += outcome.status == "failed"
+ return _LIMIT_STOPS.get(outcome.status)
+
+ def error_rate_reached(self, max_error_rate: float) -> bool:
+ return self.calls >= _MIN_ERROR_RATE_SAMPLE and self.failures >= max_error_rate * self.calls
def _in_rollout(percentage: float, organization_id: uuid.UUID) -> bool:
@@ -408,6 +449,8 @@ def capture_tick(result: PersonalRankingTickResult) -> None:
"enrolled_organizations": result.enrolled_organizations,
"enrolled_teams": result.enrolled_teams,
"duration_seconds": result.duration_seconds,
+ **{f"{status}_pair_seconds_p50": result.pair_seconds_p50.get(status) for status in _TIMED_STATUSES},
+ **{f"{status}_pair_seconds_p95": result.pair_seconds_p95.get(status) for status in _TIMED_STATUSES},
}
logger.info("personal_inbox_ranking_tick_finished", **properties)
ph_background_capture()(distinct_id=PIPELINE_DISTINCT_ID, event=TICK_EVENT, properties=properties)
@@ -450,19 +493,94 @@ class PersonalRankingTickWorkflow:
return PersonalRankingTickResult(stop_reason=plan.stop_reason)
deadline = started + datetime.timedelta(seconds=plan.time_budget_seconds)
- statuses: dict[str, int] = {}
- changed_parts: dict[str, int] = {}
- errored = 0
- # Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.
- calls = 0
- failures = 0
+ tally = _TickTally()
+ if workflow.patched(_SLIDING_WINDOW_PATCH):
+ picked, stop_reason = await self._run_sliding_window(plan, deadline, tally)
+ else:
+ picked, stop_reason = await self._run_waves(plan, deadline, tally)
+
+ result = PersonalRankingTickResult(
+ enabled=True,
+ due_pairs=plan.due_pairs,
+ due_without_list=plan.due_without_list,
+ picked=picked,
+ statuses=tally.statuses,
+ changed_parts=tally.changed_parts,
+ errored=tally.errored,
+ stop_reason=stop_reason,
+ oldest_due_hours=plan.oldest_due_hours,
+ fresh_share=plan.fresh_share,
+ qualifying_organizations=plan.qualifying_organizations,
+ enrolled_organizations=plan.enrolled_organizations,
+ enrolled_teams=plan.enrolled_teams,
+ duration_seconds=(workflow.now() - started).total_seconds(),
+ pair_seconds_p50={status: _percentile(values, 0.5) for status, values in tally.pair_seconds.items()},
+ pair_seconds_p95={status: _percentile(values, 0.95) for status, values in tally.pair_seconds.items()},
+ )
+ await workflow.execute_activity(
+ capture_personal_ranking_tick_activity,
+ result,
+ start_to_close_timeout=datetime.timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=2),
+ )
+ return result
+
+ @staticmethod
+ def _start_pair(pair: TickPair) -> workflow.ActivityHandle[PersonalRankingPairResult]:
+ return workflow.start_activity(
+ build_personal_ranking_pair_activity,
+ pair,
+ start_to_close_timeout=_PAIR_TIMEOUT,
+ heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,
+ retry_policy=_PAIR_RETRY_POLICY,
+ )
+
+ async def _run_sliding_window(
+ self, plan: PersonalRankingTickPlan, deadline: datetime.datetime, tally: _TickTally
+ ) -> tuple[int, str | None]:
+ """Starts the next pair as soon as any running pair finishes, so `max_concurrency` pairs run at once.
+
+ A stop starts no further pair, and the pairs already running finish. The pairs never started stay due,
+ so the next tick picks them.
+ """
+ concurrency = max(1, plan.max_concurrency)
+ running: dict[workflow.ActivityHandle[PersonalRankingPairResult], datetime.datetime] = {}
+ stop_reason: str | None = None
+ index = 0
+ while True:
+ while stop_reason is None and index < len(plan.pairs) and len(running) < concurrency:
+ if tally.error_rate_reached(plan.max_error_rate):
+ stop_reason = STOP_ERROR_RATE
+ elif workflow.now() >= deadline:
+ stop_reason = STOP_TIME_BUDGET
+ else:
+ running[self._start_pair(plan.pairs[index])] = workflow.now()
+ index += 1
+ if not running:
+ break
+ done, _ = await workflow.wait(running, return_when=asyncio.FIRST_COMPLETED)
+ for handle in done:
+ started_at = running.pop(handle)
+ outcome = handle.exception() or handle.result()
+ if not isinstance(outcome, BaseException) and outcome.status in _TIMED_STATUSES:
+ tally.pair_seconds.setdefault(outcome.status, []).append(
+ (workflow.now() - started_at).total_seconds()
+ )
+ limit_stop = tally.add(outcome)
+ stop_reason = stop_reason or limit_stop
+ return index, stop_reason or plan.stop_reason
+
+ async def _run_waves(
+ self, plan: PersonalRankingTickPlan, deadline: datetime.datetime, tally: _TickTally
+ ) -> tuple[int, str | None]:
+ """The loop of ticks recorded before `_SLIDING_WINDOW_PATCH`, kept so they replay."""
picked = 0
stop_reason = plan.stop_reason
concurrency = max(1, plan.max_concurrency)
index = 0
while index < len(plan.pairs):
# The pairs not attempted yet stay due, so the next tick picks them.
- if calls >= _MIN_ERROR_RATE_SAMPLE and failures >= plan.max_error_rate * calls:
+ if tally.error_rate_reached(plan.max_error_rate):
stop_reason = STOP_ERROR_RATE
break
if workflow.now() >= deadline:
@@ -486,42 +604,9 @@ class PersonalRankingTickWorkflow:
)
limit_stop: str | None = None
for outcome in results:
- if isinstance(outcome, BaseException):
- # The attempt can have called the model, so it counts as a failed call.
- errored += 1
- calls += 1
- failures += 1
- continue
- statuses[outcome.status] = statuses.get(outcome.status, 0) + 1
- for part in outcome.changed_parts:
- changed_parts[part] = changed_parts.get(part, 0) + 1
- calls += outcome.model_called
- failures += outcome.status == "failed"
- limit_stop = limit_stop or _LIMIT_STOPS.get(outcome.status)
+ refused = tally.add(outcome)
+ limit_stop = limit_stop or refused
if limit_stop is not None:
stop_reason = limit_stop
break
-
- result = PersonalRankingTickResult(
- enabled=True,
- due_pairs=plan.due_pairs,
- due_without_list=plan.due_without_list,
- picked=picked,
- statuses=statuses,
- changed_parts=changed_parts,
- errored=errored,
- stop_reason=stop_reason,
- oldest_due_hours=plan.oldest_due_hours,
- fresh_share=plan.fresh_share,
- qualifying_organizations=plan.qualifying_organizations,
- enrolled_organizations=plan.enrolled_organizations,
- enrolled_teams=plan.enrolled_teams,
- duration_seconds=(workflow.now() - started).total_seconds(),
- )
- await workflow.execute_activity(
- capture_personal_ranking_tick_activity,
- result,
- start_to_close_timeout=datetime.timedelta(minutes=1),
- retry_policy=RetryPolicy(maximum_attempts=2),
- )
- return result
+ return picked, stop_reason
diff --git a/products/signals/backend/test/test_personal_ranking.py b/products/signals/backend/test/test_personal_ranking.py
index 5050e1beed0d..08ce6046f8f7 100644
--- a/products/signals/backend/test/test_personal_ranking.py
+++ b/products/signals/backend/test/test_personal_ranking.py
@@ -549,14 +549,21 @@ class TestConfig(SimpleTestCase):
"skip_team_ids": [1, True, "2"],
"self_driving_organization_percentage": 101,
"max_pairs_per_tick": 10_000,
+ "max_concurrency": 100,
"max_tick_builds_per_day": -1,
"max_org_share": 2,
- }
+ },
+ "max_builds_per_day": 100_000,
},
PersonalRankingConfig(
+ max_builds_per_day=50_000,
tick=PersonalRankingTickConfig(
- enabled=True, organization_ids=("org",), skip_team_ids=(1,), max_pairs_per_tick=500
- )
+ enabled=True,
+ organization_ids=("org",),
+ skip_team_ids=(1,),
+ max_pairs_per_tick=2000,
+ max_concurrency=50,
+ ),
),
),
(
diff --git a/products/signals/backend/test/test_personal_ranking_tick.py b/products/signals/backend/test/test_personal_ranking_tick.py
index 36a400327104..6c6c98729dcf 100644
--- a/products/signals/backend/test/test_personal_ranking_tick.py
+++ b/products/signals/backend/test/test_personal_ranking_tick.py
@@ -2,6 +2,8 @@ import json
import uuid
import asyncio
import datetime
+import contextlib
+from collections.abc import Awaitable, Callable
from types import SimpleNamespace
from typing import Any
@@ -14,9 +16,9 @@ from django.core.cache import cache
from django.utils import timezone
from parameterized import parameterized
-from temporalio import activity
+from temporalio import activity, workflow
from temporalio.testing import WorkflowEnvironment
-from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from posthog.models import Organization, Team, User
from posthog.redis import TEST_clear_clients
@@ -319,7 +321,10 @@ def _plan(pairs: int, **overrides: Any) -> PersonalRankingTickPlan:
async def _run_tick(
- plan: PersonalRankingTickPlan, pair_status: str
+ plan: PersonalRankingTickPlan,
+ pair_status: str | Callable[[TickPair], Awaitable[str]],
+ *,
+ sliding_window: bool = True,
) -> tuple[PersonalRankingTickResult, int, list[PersonalRankingTickResult]]:
running = peak = 0
captured: list[PersonalRankingTickResult] = []
@@ -333,18 +338,27 @@ async def _run_tick(
nonlocal running, peak
running += 1
peak = max(peak, running)
- await asyncio.sleep(0.01)
+ if isinstance(pair_status, str):
+ status = pair_status
+ await asyncio.sleep(0.01)
+ else:
+ status = await pair_status(pair)
running -= 1
return PersonalRankingPairResult(
- status=pair_status, # type: ignore[arg-type]
- model_called=pair_status in ("built", "failed"),
- changed_parts=["scores", "needs_you"] if pair_status == "built" else [],
+ status=status, # type: ignore[arg-type]
+ model_called=status in ("built", "failed"),
+ changed_parts=["scores", "needs_you"] if status == "built" else [],
)
@activity.defn(name="capture_personal_ranking_tick_activity")
async def fake_capture(result: PersonalRankingTickResult) -> None:
captured.append(result)
+ original_patched = workflow.patched
+
+ def patched(patch_id: str) -> bool:
+ return sliding_window and original_patched(patch_id)
+
async with await WorkflowEnvironment.start_time_skipping() as env:
async with Worker(
env.client,
@@ -353,33 +367,41 @@ async def _run_tick(
activities=[fake_plan, fake_pair, fake_capture],
workflow_runner=UnsandboxedWorkflowRunner(),
):
- result = await env.client.execute_workflow(
- PersonalRankingTickWorkflow.run,
- PersonalRankingTickInput(),
- id=f"personal-ranking-tick-{uuid.uuid4()}",
- task_queue="personal-ranking-tick-test",
- )
+ with patch.object(workflow, "patched", patched):
+ handle = await env.client.start_workflow(
+ PersonalRankingTickWorkflow.run,
+ PersonalRankingTickInput(),
+ id=f"personal-ranking-tick-{uuid.uuid4()}",
+ task_queue="personal-ranking-tick-test",
+ )
+ result = await handle.result()
+ history = await handle.fetch_history()
+ # A tick that runs during a deploy replays its history on the new code.
+ await Replayer(
+ workflows=[PersonalRankingTickWorkflow], workflow_runner=UnsandboxedWorkflowRunner()
+ ).replay_workflow(history)
return result, peak, captured
@pytest.mark.asyncio
[email protected]("sliding_window", [True, False])
@pytest.mark.parametrize(
"plan,pair_status,picked,stop_reason",
[
(_plan(5), "built", 5, None),
(_plan(5), "unchanged", 5, None),
- # The wave that met the limit finishes, and no later wave starts.
+ # The pairs running when the limit refused a build finish, and no later pair starts.
(_plan(5), "tick_limited", 2, STOP_MAX_TICK_BUILDS_PER_DAY),
(_plan(5), "shared_limited", 2, STOP_MAX_BUILDS_PER_DAY),
- # The breaker waits for a sample of 10 model calls, which is five waves.
- (_plan(20), "failed", 10, STOP_ERROR_RATE),
+ # The breaker waits for a sample of 10 model calls.
+ (_plan(20, max_concurrency=1), "failed", 10, STOP_ERROR_RATE),
(_plan(5, time_budget_seconds=0.0), "built", 0, STOP_TIME_BUDGET),
],
)
async def test_the_tick_bounds_concurrency_and_stops_at_each_limit(
- plan: PersonalRankingTickPlan, pair_status: str, picked: int, stop_reason: str | None
+ plan: PersonalRankingTickPlan, pair_status: str, picked: int, stop_reason: str | None, sliding_window: bool
) -> None:
- result, peak, captured = await _run_tick(plan, pair_status)
+ result, peak, captured = await _run_tick(plan, pair_status, sliding_window=sliding_window)
assert peak <= plan.max_concurrency
assert (result.picked, result.statuses, result.stop_reason) == (
@@ -391,9 +413,43 @@ async def test_the_tick_bounds_concurrency_and_stops_at_each_limit(
assert result.changed_parts == (
{"scores": picked, "needs_you": picked} if pair_status == "built" and picked else {}
)
+ timed = sliding_window and picked and pair_status in ("built", "unchanged")
+ assert (set(result.pair_seconds_p50), set(result.pair_seconds_p95)) == (
+ {pair_status} if timed else set(),
+ {pair_status} if timed else set(),
+ )
assert captured == [result]
+async def test_fast_pairs_start_while_a_slow_pair_runs() -> None:
+ fast_done = asyncio.Event()
+ finished: list[int] = []
+
+ async def pair_status(pair: TickPair) -> str:
+ if pair.user_id == 0:
+ # A wave loop never starts the fast pairs behind this one, so the wait times out.
+ with contextlib.suppress(TimeoutError):
+ await asyncio.wait_for(fast_done.wait(), timeout=5)
+ finished.append(pair.user_id)
+ return "built"
+ await asyncio.sleep(0.01)
+ finished.append(pair.user_id)
+ if len(finished) == 4:
+ fast_done.set()
+ return "unchanged"
+
+ result, peak, _ = await _run_tick(_plan(5), pair_status)
+
+ assert finished[-1] == 0
+ assert (result.picked, result.statuses, result.stop_reason, peak) == (
+ 5,
+ {"built": 1, "unchanged": 4},
+ None,
+ 2,
+ )
+
+
@pytest.mark.asyncio
async def test_a_disabled_tick_runs_no_pair() -> None:
result, peak, captured = await _run_tick(_plan(0, enabled=False), "built")