:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): build personal inbox rankings on a scheduled tick (#127617)
posthog[bot]2f15a3f27b03
diff --git a/posthog/settings/object_storage.py b/posthog/settings/object_storage.pyindex adb2a5b0d661..986a465d53ba 100644--- a/posthog/settings/object_storage.py+++ b/posthog/settings/object_storage.py@@ -145,6 +145,12 @@ INBOX_RANKING_SCORING_MAX_REPORTS_PER_TICK = get_from_env(# Activity summary tick (products/signals/backend/activity_summary/tick.py). The `activity-summary`# flag payload gates the work and holds the caps, so the schedule ticks in every environment.ACTIVITY_SUMMARY_TICK_INTERVAL_MINUTES = get_from_env("ACTIVITY_SUMMARY_TICK_INTERVAL_MINUTES", 60, type_cast=int)+# Personal inbox ranking tick (products/signals/backend/personal_ranking/tick.py). The `tick` object of the+# `personal-inbox-ranking` flag payload gates the work. It ticks more often than the list's target interval,+# so a list comes due soon after the interval passes.+PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES = get_from_env(+ "PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES", 15, type_cast=int+)# Identity matching scratch storage (products/growth `identity_matching_job`). The job writes# per-run Parquet objects via ClickHouse `INSERT INTO FUNCTION s3(...)` and the read API globsdiff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.pyindex f561fc3c8b23..808184f0feef 100644--- a/posthog/temporal/schedule.py+++ b/posthog/temporal/schedule.py@@ -145,6 +145,7 @@ from products.signals.backend.facade.temporal import (create_anomaly_scoring_tick_schedule,create_conversations_signals_coordinator_schedule,create_inbox_ranking_scoring_schedule,+ create_personal_inbox_ranking_tick_schedule,create_scout_suggestions_coordinator_schedule,create_signals_scout_coordinator_schedule,)@@ -1021,6 +1022,7 @@ schedules = [create_signals_scout_coordinator_schedule,create_inbox_ranking_scoring_schedule,create_activity_summary_tick_schedule,+ create_personal_inbox_ranking_tick_schedule,create_anomaly_scoring_tick_schedule,create_anomaly_scoring_due_list_refresh_schedule,create_scout_suggestions_coordinator_schedule,diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.pyindex c99b647b4786..565e6e65d0f7 100644--- a/products/signals/backend/activity_summary/queue.py+++ b/products/signals/backend/activity_summary/queue.py@@ -15,7 +15,6 @@ day per day.import mathimport uuid-import hashlibimport datetimeimport dataclassesfrom collections.abc import Callable, Iterable, Sequence@@ -45,6 +44,7 @@ from django.db.models.functions import Cast, Coalesce, Extract, Greatest, Ln, Rofrom posthog.dataclasses import frozenfrom posthog.models import ActivityLog, FileSystemViewLog, Team+from products.signals.backend import org_rolloutfrom products.signals.backend.activity_summary.config import ActivitySummaryConfig, SourceWeightsfrom products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTSfrom products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX@@ -202,17 +202,15 @@ def daily_usage(now: datetime.datetime) -> DailyUsage:# A new salt reorders every organization, so change the version only to start a new rollout order._ROLLOUT_SALT = "activity-summary-self-driving-v1"-_ROLLOUT_BUCKETS = 10_000def organization_rollout_bucket(organization_id: uuid.UUID) -> int:- """The organization's fixed place in the percentage rollout, from 0 to `_ROLLOUT_BUCKETS` - 1."""- digest = hashlib.sha256(f"{_ROLLOUT_SALT}:{organization_id}".encode()).digest()- return int.from_bytes(digest[:8], "big") % _ROLLOUT_BUCKETS+ """The organization's fixed place in the percentage rollout, from 0 to 9999."""+ return org_rollout.organization_rollout_bucket(_ROLLOUT_SALT, organization_id)def _in_rollout(percentage: float, organization_id: uuid.UUID) -> bool:- return organization_rollout_bucket(organization_id) < percentage * _ROLLOUT_BUCKETS / 100+ return org_rollout.in_rollout(_ROLLOUT_SALT, percentage, organization_id)@frozen@@ -312,15 +310,17 @@ def is_team_enrolled_for_read(config: ActivitySummaryConfig, team: Team) -> boolreturn config.enroll_all_approved or team.organization_id in _parse_organization_ids(config.organization_ids)-def _parse_organization_ids(organization_ids: tuple[str, ...]) -> set[uuid.UUID]:- # A UUID parse, so an id in another UUID format still matches, and an invalid id matches nothing.- parsed: set[uuid.UUID] = set()- for organization_id in organization_ids:- try:- parsed.add(uuid.UUID(organization_id))- except ValueError:- continue- return parsed+def read_enrolled_teams(config: ActivitySummaryConfig, teams: QuerySet[Team]) -> QuerySet[Team]:+ """The teams of `teams` that `is_team_enrolled_for_read` enrolls, in one query."""+ if not config.enabled:+ return teams.none()+ teams = teams.filter(organization__is_ai_data_processing_approved=True).exclude(id__in=config.skip_team_ids)+ if config.enroll_all_approved:+ return teams+ return teams.filter(organization_id__in=_parse_organization_ids(config.organization_ids))+++_parse_organization_ids = org_rollout.parse_organization_idsdef _add(pending: _Pending, pair: ActivitySummaryPair, units: int, at: datetime.datetime, score: float = 0.0) -> None:diff --git a/products/signals/backend/facade/temporal.py b/products/signals/backend/facade/temporal.pyindex 162622d3749d..42713e07cfe8 100644--- a/products/signals/backend/facade/temporal.py+++ b/products/signals/backend/facade/temporal.py@@ -7,6 +7,7 @@ from products.signals.backend.anomaly_scores.schedule import ()from products.signals.backend.emission.conversations_schedule import create_conversations_signals_coordinator_schedulefrom products.signals.backend.emission.temporal_settings import EMIT_SIGNALS_ACTIVITIES, EMIT_SIGNALS_WORKFLOWS+from products.signals.backend.personal_ranking.schedule import create_personal_inbox_ranking_tick_schedulefrom products.signals.backend.ranking.schedule import create_inbox_ranking_scoring_schedulefrom products.signals.backend.temporal import ACTIVITIES, SELF_DRIVING_ACTIVITIES, SELF_DRIVING_WORKFLOWS, WORKFLOWSfrom products.signals.backend.temporal.agentic.schedule import (@@ -33,6 +34,7 @@ __all__ = ["create_anomaly_scoring_tick_schedule","create_conversations_signals_coordinator_schedule","create_inbox_ranking_scoring_schedule",+ "create_personal_inbox_ranking_tick_schedule","create_scout_suggestions_coordinator_schedule","create_signals_scout_coordinator_schedule","emit_eval_signal_activity",diff --git a/products/signals/backend/management/AGENTS.md b/products/signals/backend/management/AGENTS.mdindex e962750edcdc..34faf23bf665 100644--- a/products/signals/backend/management/AGENTS.md+++ b/products/signals/backend/management/AGENTS.md@@ -305,13 +305,48 @@ The `personal inbox ranking built` event records `model_called`, so builds withThe command spends no build limit, so use it only for single runs.`GET /api/projects/:id/signals/reports/for_me/` (`../personal_ranking/serve.py`) serves the same list to the user it describes, when the flag is on for that user.-It reuses the latest row for `freshness_minutes` while the input hash matches.+It reuses the latest row for `freshness_minutes` after the row was built or last checked by the tick, while the input hash matches.Otherwise it queues one `build_personal_inbox_ranking` Celery task per person behind a cache lock, and serves the latest row as stale, or the candidates in inbox order with `fallback_reason: building`.It limits builds per person with `build_burst`, `builds_per_hour` and `builds_per_day` from the same payload.`max_builds_per_day` (default 500, at most 10000) limits the builds that call the model across every person, in one Redis token bucket per region (`../personal_ranking/build_budget.py`).A build with no ready activity summary, or for an organization that has not approved AI data processing, does not call the model and does not spend that limit.When a limit refuses a build, the read serves the latest row as stale, or the candidates in inbox order with `fallback_reason: rate_limited`.-A scheduled caller must spend from the same bucket with `spend_shared_build` before it builds.+The scheduled tick spends from the same bucket.++### Scheduled personal ranking tick++`personal-inbox-ranking-tick` (`../personal_ranking/tick.py`) is a Temporal schedule on `SELF_DRIVING_TASK_QUEUE` with overlap `SKIP`.+It runs every `PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES` (default 15) in every environment, and builds lists ahead of the first `for_me` read.+The `tick` object of the `personal-inbox-ranking` payload holds its levers. Without `"enabled": true` in that object the tick does nothing, and `"enabled": false` at the top level stops it too.++```json+{+ "tick": {+ "enabled": true,+ "self_driving_organization_percentage": 5,+ "organization_ids": ["<organization uuid>"],+ "deny_organization_ids": [],+ "skip_team_ids": [],+ "target_interval_minutes": 60,+ "active_days": 7,+ "max_pairs_per_tick": 20,+ "max_concurrency": 4,+ "max_tick_builds_per_day": 100,+ "max_org_share": 0.25,+ "max_error_rate": 0.5+ }+}+```++- **Enrollment:** every project of the listed organizations, plus the salted `self_driving_organization_percentage` of the organizations with a project that has an enabled signal source. The percentage enrolls only those projects, and a higher percentage keeps the organizations of a lower one. The organization must approve AI data processing, and the activity summary must be readable for the project. The deny list and `skip_team_ids` win. Enrollment decides who gets scheduled builds. It does not change who can read `for_me`.+- **Due pairs:** a member with a ready activity summary, activity in the last `active_days`, and a latest list that was built or checked longer ago than `target_interval_minutes`, or no list.+- **Order:** paying organizations first, then the most recent activity, then the oldest list. One organization takes at most `max_org_share` of `max_pairs_per_tick`.+- **Unchanged inputs:** the pair activity compares the input hash first. A match calls no model and sets `checked_at` on the latest row, so the pair is not due again until `target_interval_minutes` passes, and `for_me` serves that row as fresh. An enabled tick raises `freshness_minutes` to at least `target_interval_minutes`, so a plain read does not rebuild a list inside the tick's interval. A row whose model call failed, timed out or answered outside the schema, or that skipped the model without AI approval, is built again instead.+- **Limits:** a build that calls the model spends `max_tick_builds_per_day` (one Redis token bucket per region, closed when Redis is unreachable) and then the shared `max_builds_per_day`. `max_tick_builds_per_day` is limited to 80% of `max_builds_per_day`, so `refresh` and `focus` requests keep a reserve. To run the tick above that, raise the shared limit too. The tick stops at the first refusal, at `max_error_rate` after 10 model calls, and when 60% of its interval has passed.+- **Locks:** the pair takes the same per-person lock as `for_me`, so a tick skips a person whose build is running.++Each enabled tick captures `personal inbox ranking tick` with the due pairs, the pairs it picked, a count per pair status (`built`, `failed`, `unchanged`, `running`, `not_enrolled`, `tick_limited`, `shared_limited`), the stop reason, `oldest_due_hours`, `fresh_share`, the enrolled organizations and teams, and the duration.+`personal inbox ranking built` and the model call's trace carry `trigger` (`tick` or `on_demand`), so cost can be split by what started a build.## Tipsdiff --git a/products/signals/backend/migrations/0227_personalinboxranking_checked_at.py b/products/signals/backend/migrations/0227_personalinboxranking_checked_at.pynew file mode 100644index 000000000000..9c2c43995253--- /dev/null+++ b/products/signals/backend/migrations/0227_personalinboxranking_checked_at.py@@ -0,0 +1,17 @@+# Generated by Django 6.1.2 on 2026-11-02 16:48++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("signals", "0226_signalreport_trgm_fastupdate_off"),+ ]++ operations = [+ migrations.AddField(+ model_name="personalinboxranking",+ name="checked_at",+ field=models.DateTimeField(blank=True, null=True),+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex 8ab28bae09ce..fc3442974dc3 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0226_signalreport_trgm_fastupdate_off+0227_personalinboxranking_checked_atdiff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex e42c6ff9294d..92ebaff3dbe6 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -4345,6 +4345,8 @@ class PersonalInboxRanking(TeamScopedRootMixin, UUIDModel):status = models.CharField(max_length=20, choices=Status)# One of the fixed strings of `personal_ranking/logic.py`, never model output.fallback_reason = models.CharField(max_length=100, null=True, blank=True)+ # The last time the scheduled tick found the inputs unchanged, so the list is current as of then.+ checked_at = models.DateTimeField(null=True, blank=True)class Meta:default_manager_name = "all_teams"diff --git a/products/signals/backend/org_rollout.py b/products/signals/backend/org_rollout.pynew file mode 100644index 000000000000..07ecf82c46da--- /dev/null+++ b/products/signals/backend/org_rollout.py@@ -0,0 +1,27 @@+"""A salted percentage rollout over organizations, and the organization id lists of a flag payload."""++import uuid+import hashlib++_ROLLOUT_BUCKETS = 10_000+++def organization_rollout_bucket(salt: str, organization_id: uuid.UUID) -> int:+ """The organization's fixed place in the rollout of `salt`, from 0 to `_ROLLOUT_BUCKETS` - 1."""+ digest = hashlib.sha256(f"{salt}:{organization_id}".encode()).digest()+ return int.from_bytes(digest[:8], "big") % _ROLLOUT_BUCKETS+++def in_rollout(salt: str, percentage: float, organization_id: uuid.UUID) -> bool:+ return organization_rollout_bucket(salt, organization_id) < percentage * _ROLLOUT_BUCKETS / 100+++def parse_organization_ids(organization_ids: tuple[str, ...]) -> set[uuid.UUID]:+ # A UUID parse, so an id in another UUID format still matches, and an invalid id matches nothing.+ parsed: set[uuid.UUID] = set()+ for organization_id in organization_ids:+ try:+ parsed.add(uuid.UUID(organization_id))+ except ValueError:+ continue+ return parseddiff --git a/products/signals/backend/personal_ranking/build_budget.py b/products/signals/backend/personal_ranking/build_budget.pyindex d9b1315cc241..081f33922e35 100644--- a/products/signals/backend/personal_ranking/build_budget.py+++ b/products/signals/backend/personal_ranking/build_budget.py@@ -1,31 +1,37 @@"""The daily limit on personal ranking builds that call the model, shared by every person in one region.-The `for_me` read spends one build here before it queues a build that may call the model. A scheduled caller-spends from the same bucket. Each region has its own Redis, so each region counts on its own.+The `for_me` read spends one build here before it queues a build that may call the model. The scheduled tick+spends from the same bucket, after it spends from its own bucket of `max_tick_builds_per_day`. Each region has its+own Redis, so each region counts on its own."""from django.conf import settingsimport structlog-from posthog.token_bucket import BucketUnavailable, Budget, consume+from posthog.token_bucket import BucketUnavailable, Budget, consume, refundfrom products.signals.backend.personal_ranking.config import PersonalRankingConfiglogger = structlog.get_logger(__name__)_BUCKET_PREFIX = "personal_ranking_build_rate:shared"+_TICK_BUCKET_PREFIX = "personal_ranking_build_rate:tick"# A build costs one token per hour of the day, and the bucket refills `max_builds_per_day` tokens an hour,# so it refills exactly `max_builds_per_day` builds a day. A whole-token rate per build would round it up.DAILY_BUILD_COST = 24-def _key() -> str:- return f"{_BUCKET_PREFIX}:{settings.CLOUD_DEPLOYMENT or 'local'}"+def _key(prefix: str = _BUCKET_PREFIX) -> str:+ return f"{prefix}:{settings.CLOUD_DEPLOYMENT or 'local'}"+++def _daily_budget(builds_per_day: int) -> Budget:+ return Budget(burst=builds_per_day * DAILY_BUILD_COST, per_hour=builds_per_day)def _budget(config: PersonalRankingConfig) -> Budget:- return Budget(burst=config.max_builds_per_day * DAILY_BUILD_COST, per_hour=config.max_builds_per_day)+ return _daily_budget(config.max_builds_per_day)def spend_shared_build(config: PersonalRankingConfig) -> int | None:@@ -36,3 +42,19 @@ def spend_shared_build(config: PersonalRankingConfig) -> int | None:logger.warning("signals.personal_ranking.shared_build_bucket_unavailable")return Nonereturn None if decision.allowed else decision.retry_after+++def spend_tick_build(config: PersonalRankingConfig) -> bool:+ """Spend one build of the tick's daily limit. Returns False when the limit or an unreachable Redis refuses it."""+ decision = consume(+ _key(_TICK_BUCKET_PREFIX), _daily_budget(config.tick.max_tick_builds_per_day), cost=DAILY_BUILD_COST+ )+ if isinstance(decision, BucketUnavailable):+ # Fail closed: nobody waits on a tick build, so a skipped build costs only a stale list.+ logger.warning("signals.personal_ranking.tick_build_bucket_unavailable")+ return False+ return decision.allowed+++def refund_tick_build(config: PersonalRankingConfig) -> None:+ refund(_key(_TICK_BUCKET_PREFIX), _daily_budget(config.tick.max_tick_builds_per_day), cost=DAILY_BUILD_COST)diff --git a/products/signals/backend/personal_ranking/config.py b/products/signals/backend/personal_ranking/config.pyindex 8871638bc8ad..9c8f3aa3885c 100644--- a/products/signals/backend/personal_ranking/config.py+++ b/products/signals/backend/personal_ranking/config.py@@ -7,9 +7,15 @@ person in one region.The optional `pools` list replaces the default candidate pools, in order. An invalid entry is droppedand logged. When no entry is valid, the build uses the default pools.++The optional `tick` object holds the levers of the scheduled tick (`tick.py`). Its parse fails closed:+a missing or malformed object, or one without `"enabled": true`, turns the tick off. An enabled tick raises+`freshness_minutes` to its `target_interval_minutes`, and limits `max_tick_builds_per_day` to a share of+`max_builds_per_day`."""import re+from dataclasses import replacefrom typing import Any, Literal, cast, get_argsimport structlog@@ -80,6 +86,44 @@ class CandidatePool:window_days: int | None = None+@frozen+class PersonalRankingTickConfig:+ enabled: bool = False+ # Builds for every project of these organizations.+ organization_ids: tuple[str, ...] = ()+ # Builds for this percentage of the organizations with a project that has an enabled signal source.+ # Only those projects get builds. A higher percentage keeps the organizations of a lower one.+ self_driving_organization_percentage: float = 0.0+ # The tick never builds for these organizations, even when another field enrolls them.+ deny_organization_ids: tuple[str, ...] = ()+ skip_team_ids: tuple[int, ...] = ()+ # A pair is due when its latest list was built or checked longer ago than this.+ target_interval_minutes: int = 60+ # Only pairs with activity in this many days are due.+ active_days: int = 7+ max_pairs_per_tick: int = 20+ max_concurrency: int = 4+ # Tick builds that call the model, in one Redis token bucket per region. They also spend `max_builds_per_day`.+ max_tick_builds_per_day: int = 100+ # The largest share of one tick that the pairs of one organization can take.+ max_org_share: float = 0.25+ # The tick stops when this share of its model calls failed or raised.+ max_error_rate: float = 0.5+++TICK_DISABLED = PersonalRankingTickConfig()+# The rest of the shared limit stays free for `refresh` and `focus` requests, which a person is waiting on.+TICK_SHARE_OF_SHARED_BUILDS = 0.8++_TICK_MAXIMUMS: dict[str, int] = {+ "target_interval_minutes": 1440,+ "active_days": 90,+ "max_pairs_per_tick": 500,+ "max_concurrency": 20,+ "max_tick_builds_per_day": 10000,+}++@frozenclass PersonalRankingConfig:enabled: bool = True@@ -123,6 +167,7 @@ class PersonalRankingConfig:max_builds_per_day: int = 500# Each build keeps this many of the person's newest rows and deletes the older ones.max_rows_per_person: int = 5+ tick: PersonalRankingTickConfig = TICK_DISABLEDDEFAULTS = PersonalRankingConfig()@@ -191,6 +236,51 @@ def _reason_limits(value: Any) -> ReasonLimits:)+def _fraction(value: Any, default: float) -> float:+ if isinstance(value, bool) or not isinstance(value, int | float) or not 0 < value <= 1:+ return default+ return float(value)+++def _percentage(value: Any) -> float:+ if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value <= 100:+ return TICK_DISABLED.self_driving_organization_percentage+ return float(value)+++def _organization_ids(value: Any) -> tuple[str, ...]:+ return tuple(item for item in value if isinstance(item, str) and item) if isinstance(value, list) else ()+++def _team_ids(value: Any) -> tuple[int, ...]:+ if not isinstance(value, list):+ return ()+ return tuple(item for item in value if isinstance(item, int) and not isinstance(item, bool))+++def parse_tick_config(payload: Any) -> PersonalRankingTickConfig:+ if not isinstance(payload, dict) or payload.get("enabled") is not True:+ return TICK_DISABLED+ return PersonalRankingTickConfig(+ enabled=True,+ organization_ids=_organization_ids(payload.get("organization_ids")),+ self_driving_organization_percentage=_percentage(payload.get("self_driving_organization_percentage")),+ deny_organization_ids=_organization_ids(payload.get("deny_organization_ids")),+ skip_team_ids=_team_ids(payload.get("skip_team_ids")),+ max_org_share=_fraction(payload.get("max_org_share"), TICK_DISABLED.max_org_share),+ max_error_rate=_fraction(payload.get("max_error_rate"), TICK_DISABLED.max_error_rate),+ **{+ name: _bounded_int(payload.get(name), getattr(TICK_DISABLED, name), maximum)+ for name, maximum in _TICK_MAXIMUMS.items()+ },+ )+++def _fit_tick_to_shared_limit(tick: PersonalRankingTickConfig, max_builds_per_day: int) -> PersonalRankingTickConfig:+ largest = max(int(max_builds_per_day * TICK_SHARE_OF_SHARED_BUILDS), 1)+ return replace(tick, max_tick_builds_per_day=min(tick.max_tick_builds_per_day, largest))++def default_pools(config: PersonalRankingConfig) -> tuple[CandidatePool, ...]:return (CandidatePool(name="for_you_recommended", source="for_you_recommended", size=config.for_you_recommended_pool),@@ -297,12 +387,18 @@ def parse_config(payload: Any) -> PersonalRankingConfig:if name not in {"timeout_seconds", "reason_max_chars", "pool_size"}}max_limit = ints.pop("max_limit")+ tick = parse_tick_config(payload.get("tick"))+ if tick.enabled:+ tick = _fit_tick_to_shared_limit(tick, ints["max_builds_per_day"])+ # A shorter freshness makes a plain read rebuild a list that the tick checked inside its interval.+ ints["freshness_minutes"] = max(ints["freshness_minutes"], tick.target_interval_minutes)return PersonalRankingConfig(enabled=payload.get("enabled") is not False,model=_model(payload.get("model")),reasoning_effort=_reasoning_effort(payload.get("reasoning_effort")),reason_limits=_reason_limits(payload.get("reason_max_chars")),pools=_pools(payload.get("pools")),+ tick=tick,timeout_seconds=_bounded_number(payload.get("timeout_seconds"), DEFAULTS.timeout_seconds, _MAXIMUMS["timeout_seconds"]),diff --git a/products/signals/backend/personal_ranking/logic.py b/products/signals/backend/personal_ranking/logic.pyindex e78083241c34..9ea94dcc8fc7 100644--- a/products/signals/backend/personal_ranking/logic.py+++ b/products/signals/backend/personal_ranking/logic.py@@ -66,8 +66,14 @@ FALLBACK_TRUNCATED = "truncated_output"FALLBACK_NO_CANDIDATES = "no_candidates"FALLBACK_AI_NOT_APPROVED = "ai_not_approved"FALLBACK_NO_ACTIVITY_SUMMARY = "no_activity_summary"+# A later build with the same input hash can rank the list. AI approval is not part of the hash.+RETRYABLE_FALLBACK_REASONS = frozenset(+ {FALLBACK_LLM_ERROR, FALLBACK_TIMEOUT, FALLBACK_MALFORMED, FALLBACK_TRUNCATED, FALLBACK_AI_NOT_APPROVED}+)BuildStatus = Literal["ok", "fallback", "empty", "disabled"]+# What started a build, on the build event and the trace, so cost can be split by it.+BuildTrigger = Literal["on_demand", "tick"]_MODEL_ORDER_REASON = "Kept in inbox order."_NEEDS_YOU_REASONS = {@@ -181,12 +187,19 @@ def input_hash(candidates: list[Candidate], *, activity_summary_version: int | Nreturn hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()-def _client(team: Team, user: User, now: datetime.datetime, config: PersonalRankingConfig) -> OpenAI:+def _client(+ team: Team, user: User, now: datetime.datetime, config: PersonalRankingConfig, trigger: BuildTrigger+) -> OpenAI:client = build_openai_client("signals",ai_product=AI_PRODUCT,trace_id=trace_id_from_seed(f"{AI_PRODUCT}:{team.id}:{user.id}:{now.isoformat()}"),- properties={"team_id": str(team.id), "user_id": str(user.id), "prompt_version": str(PROMPT_VERSION)},+ properties={+ "team_id": str(team.id),+ "user_id": str(user.id),+ "prompt_version": str(PROMPT_VERSION),+ "trigger": trigger,+ },distinct_id=PIPELINE_DISTINCT_ID,)# One attempt: a retry would run past the timeout, and the fallback list is always there.@@ -229,6 +242,7 @@ def _call_model(now: datetime.datetime,config: PersonalRankingConfig,messages: list[ChatCompletionMessageParam],+ trigger: BuildTrigger,) -> _ModelCall:started = time.monotonic()@@ -236,7 +250,7 @@ def _call_model(return round((time.monotonic() - started) * 1000)try:- response = _create_completion(_client(team, user, now, config), config, messages)+ response = _create_completion(_client(team, user, now, config, trigger), config, messages)except APITimeoutError:return _ModelCall(content=None, tokens_in=None, tokens_out=None, latency_ms=elapsed_ms(), fallback_reason=FALLBACK_TIMEOUT@@ -259,7 +273,13 @@ def _call_model(def _capture(- team: Team, user: User, build: PersonalRankingBuild, config: PersonalRankingConfig, *, has_focus: bool+ team: Team,+ user: User,+ build: PersonalRankingBuild,+ config: PersonalRankingConfig,+ *,+ has_focus: bool,+ trigger: BuildTrigger,) -> None:# Ids, counts and accounting only. Report text, reasons and the summary are user content.try:@@ -284,6 +304,7 @@ def _capture("prompt_version": PROMPT_VERSION,"activity_summary_version": build.activity_summary_version,"has_focus": has_focus,+ "trigger": trigger,"limit": build.limit,"rank_top": config.rank_top,"reasoning_effort": config.reasoning_effort,@@ -340,6 +361,7 @@ def build_personal_ranking(dry_run: bool = False,limit: int | None = None,config: PersonalRankingConfig | None = None,+ trigger: BuildTrigger = "on_demand",) -> PersonalRankingBuild:"""Rank the person's candidate reports. `dry_run` calls the model but stores no row and records no event."""config = config or read_config()@@ -393,7 +415,7 @@ def build_personal_ranking(),},]- call = _call_model(team, user, now, config, messages)+ call = _call_model(team, user, now, config, messages, trigger)fallback_reason = call.fallback_reasonif fallback_reason is None:try:@@ -441,5 +463,5 @@ def build_personal_ranking(_prune(team, user, config.max_rows_per_person)except Exception:logger.exception("signals.personal_ranking.prune_failed", team_id=team.id, user_id=user.id)- _capture(team, user, build, config, has_focus=focus is not None)+ _capture(team, user, build, config, has_focus=focus is not None, trigger=trigger)return builddiff --git a/products/signals/backend/personal_ranking/schedule.py b/products/signals/backend/personal_ranking/schedule.pynew file mode 100644index 000000000000..59e898d78193--- /dev/null+++ b/products/signals/backend/personal_ranking/schedule.py@@ -0,0 +1,57 @@+"""Schedule registration for the personal inbox ranking tick."""++from dataclasses import asdict+from datetime import timedelta++from django.conf import settings++from temporalio.client import (+ Client,+ Schedule,+ ScheduleActionStartWorkflow,+ ScheduleIntervalSpec,+ ScheduleOverlapPolicy,+ SchedulePolicy,+ ScheduleSpec,+)++from posthog.scheduling.jitter import deterministic_offset+from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule++from products.signals.backend.personal_ranking.tick import TICK_WORKFLOW_NAME, PersonalRankingTickInput++PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID = "personal-inbox-ranking-tick-schedule"+++async def create_personal_inbox_ranking_tick_schedule(client: Client) -> None:+ """Create or update the tick schedule on the self-driving task queue.++ The schedule exists in every environment. The `tick` object of the `personal-inbox-ranking` flag+ payload gates the work, so turning the tick on or off needs no schedule change. SKIP on overlap: a slow+ tick leaves the rest of the due pairs to the next one.+ """+ interval = timedelta(minutes=settings.PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES)+ schedule = Schedule(+ action=ScheduleActionStartWorkflow(+ TICK_WORKFLOW_NAME,+ asdict(PersonalRankingTickInput()),+ id=PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID,+ task_queue=settings.SELF_DRIVING_TASK_QUEUE,+ execution_timeout=interval,+ ),+ spec=ScheduleSpec(+ intervals=[+ ScheduleIntervalSpec(+ every=interval, offset=deterministic_offset(PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID, interval)+ )+ ]+ ),+ policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),+ )+ if await a_schedule_exists(client, PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID):+ # Keep the live state, so a deploy does not resume a schedule that an operator paused.+ description = await client.get_schedule_handle(PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID).describe()+ schedule.state = description.schedule.state+ await a_update_schedule(client, PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID, schedule)+ else:+ await a_create_schedule(client, PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID, schedule, trigger_immediately=False)diff --git a/products/signals/backend/personal_ranking/serve.py b/products/signals/backend/personal_ranking/serve.pyindex df34cab98852..cb608fa5b7ed 100644--- a/products/signals/backend/personal_ranking/serve.py+++ b/products/signals/backend/personal_ranking/serve.py@@ -1,13 +1,13 @@"""Serve one person's ranked inbox list to the `for_me` endpoint. A request never waits on the model.-A stored list is reused while it is younger than `freshness_minutes` and its input hash matches the current-inputs. Otherwise, and for every `refresh` or `focus` request, the read queues one background build per person-and serves the latest stored list as stale, or the candidates in inbox order when there is none. The focus is-part of the hash, so an unfocused request never reuses a focused list. A build that stores a list for the current-inputs while the read runs answers the read, also a `refresh` read, so the read queues no second build. A lock lets one build per person run at a-time, and each queued build spends two per-person token buckets. A build that may call the model also spends-the shared daily limit of the region (`build_budget.py`). `wait_seconds` waits a bounded time for the queued build-to store its list.+A stored list is reused while it was built or checked by the tick less than `freshness_minutes` ago and its input+hash matches the current inputs. Otherwise, and for every `refresh` or `focus` request, the read queues one+background build per person and serves the latest stored list as stale, or the candidates in inbox order when+there is none. The focus is part of the hash, so an unfocused request never reuses a focused list. A build that+stores a list for the current inputs while the read runs answers the read, also a `refresh` read, so the read+queues no second build. A lock lets one build per person run at a time, and each queued build spends two+per-person token buckets. A build that may call the model also spends the shared daily limit of the region+(`build_budget.py`). `wait_seconds` waits a bounded time for the queued build to store its list."""import time@@ -198,14 +198,14 @@ def _start_build(return "failed"-def _latest_row(team: Team, user: User) -> PersonalInboxRanking | None:+def latest_row(team: Team, user: User) -> PersonalInboxRanking | None:return PersonalInboxRanking.objects.for_team(team.id).filter(user_id=user.id).order_by("-created_at", "-id").first()def _row_stored_since(team: Team, user: User, seen: PersonalInboxRanking | None, current_hash: str) -> PersonalInboxRanking | None:- row = _latest_row(team, user)+ row = latest_row(team, user)if row is None or (seen is not None and row.id == seen.id) or row.input_hash != current_hash:return Nonereturn row@@ -216,7 +216,7 @@ def _wait_for_new_row() -> PersonalInboxRanking | None:deadline = time.monotonic() + wait_secondswhile True:- row = _latest_row(team, user)+ row = latest_row(team, user)if row is not None and (previous is None or row.id != previous.id):return rowif time.monotonic() >= deadline:@@ -224,7 +224,7 @@ def _wait_for_new_row(time.sleep(_WAIT_POLL_SECONDS)-def _ready_summary_version(team: Team, user: User) -> int | None:+def ready_summary_version(team: Team, user: User) -> int | None:summary = get_summary(team, user.id)return summary.version if summary is not None and summary.status == ActivitySummaryStatus.READY else None@@ -353,15 +353,15 @@ def serve_personal_ranking(wait_seconds = min(max(wait_seconds, 0), MAX_WAIT_SECONDS)now = timezone.now()- started_with = _latest_row(team, user)+ started_with = latest_row(team, user)candidates = gather_candidates(team, user, config, now=now, with_sources=False)- summary_version = _ready_summary_version(team, user)+ summary_version = ready_summary_version(team, user)current_hash = input_hash(candidates, activity_summary_version=summary_version, focus=focus)# Read again, because a build can store a row while the candidates are gathered.- latest = _latest_row(team, user)+ latest = latest_row(team, user)current = (latest is not None- and latest.created_at >= now - datetime.timedelta(minutes=config.freshness_minutes)+ and (latest.checked_at or latest.created_at) >= now - datetime.timedelta(minutes=config.freshness_minutes)and latest.input_hash == current_hash)# A row stored after this read started also answers a refresh.diff --git a/products/signals/backend/personal_ranking/tick.py b/products/signals/backend/personal_ranking/tick.pynew file mode 100644index 000000000000..e5872660a8b9--- /dev/null+++ b/products/signals/backend/personal_ranking/tick.py@@ -0,0 +1,522 @@+"""The scheduled personal ranking tick: a Temporal workflow that builds the lists of due (team, user) pairs.++The `tick` object of the `personal-inbox-ranking` flag payload holds every lever, and a missing or malformed+object turns the tick off. A pair is due when it has a ready activity summary, recent activity, and a latest+list that was built or checked longer ago than `target_interval_minutes`, or no list. The pair activity takes+the same per-person lock as the `for_me` read, so a tick and a read never build the same list together. It+compares the input hash first: a match records `checked_at` on the latest row and calls no model. A build that+calls the model spends the tick's daily limit and then the shared daily limit of the region.+"""++import uuid+import asyncio+import datetime+import dataclasses+from typing import Any, Literal++from django.conf import settings+from django.core.cache import cache+from django.db.models import Count, DateTimeField, Exists, F, Min, OuterRef, Q, QuerySet, Subquery, Value, Window+from django.db.models.functions import Coalesce, RowNumber+from django.utils import timezone++import structlog+from temporalio import activity, workflow+from temporalio.common import RetryPolicy++from posthog.dataclasses import frozen+from posthog.llm.gateway_client import openai_gateway_configured+from posthog.models import OrganizationMembership, Team, User+from posthog.ph_client import ph_background_capture+from posthog.sync import database_sync_to_async+from posthog.temporal.common.heartbeat import Heartbeater+from posthog.temporal.common.scoped import scoped_temporal+from posthog.temporal.common.utils import close_db_connections++from products.signals.backend.activity_summary.config import read_config as read_activity_summary_config+from products.signals.backend.activity_summary.queue import read_enrolled_teams+from products.signals.backend.facade.api import has_enabled_source, team_ids_with_enabled_source+from products.signals.backend.models import PersonalInboxRanking, UserActivitySummary, UserActivitySummaryQueueState+from products.signals.backend.org_rollout import in_rollout, parse_organization_ids+from products.signals.backend.personal_ranking.build_budget import (+ refund_tick_build,+ spend_shared_build,+ spend_tick_build,+)+from products.signals.backend.personal_ranking.candidates import gather_candidates+from products.signals.backend.personal_ranking.config import (+ PersonalRankingConfig,+ PersonalRankingTickConfig,+ read_config,+)+from products.signals.backend.personal_ranking.logic import (+ PIPELINE_DISTINCT_ID,+ RETRYABLE_FALLBACK_REASONS,+ build_personal_ranking,+ input_hash,+ may_call_model,+)+from products.signals.backend.personal_ranking.serve import latest_row, ready_summary_version+from products.signals.backend.personal_ranking_tasks import BUILD_LOCK_TTL_SECONDS, build_lock_key++logger = structlog.get_logger(__name__)++TICK_WORKFLOW_NAME = "personal-inbox-ranking-tick"+TICK_EVENT = "personal inbox ranking tick"++# Stop reasons of a tick. Null when the tick handled every due pair.+STOP_MAX_PAIRS_PER_TICK = "max_pairs_per_tick"+# Due pairs are left, but each organization with due pairs already has its share of the tick.+STOP_MAX_ORG_SHARE = "max_org_share"+STOP_MAX_TICK_BUILDS_PER_DAY = "max_tick_builds_per_day"+STOP_MAX_BUILDS_PER_DAY = "max_builds_per_day"+STOP_ERROR_RATE = "error_rate"+STOP_GATEWAY_NOT_CONFIGURED = "gateway_not_configured"+# The tick stopped before the next tick starts, so the schedule does not skip it.+STOP_TIME_BUDGET = "time_budget"++PairStatus = Literal[+ "built", "failed", "unchanged", "running", "not_enrolled", "tick_limited", "shared_limited", "disabled"+]+PAIR_STATUSES: tuple[PairStatus, ...] = (+ "built",+ "failed",+ "unchanged",+ "running",+ "not_enrolled",+ "tick_limited",+ "shared_limited",+ "disabled",+)+_LIMIT_STOPS: dict[PairStatus, str] = {+ "tick_limited": STOP_MAX_TICK_BUILDS_PER_DAY,+ "shared_limited": STOP_MAX_BUILDS_PER_DAY,+}++# A new salt reorders every organization, so change the version only to start a new rollout order.+_ROLLOUT_SALT = "personal-inbox-ranking-tick-v1"+# The circuit breaker waits for this many model calls, so a few early failures do not stop the tick.+_MIN_ERROR_RATE_SAMPLE = 10++_PLAN_TIMEOUT = datetime.timedelta(minutes=5)+# One build has a model call of at most 120 seconds, and the candidate reads around it.+_PAIR_TIMEOUT = datetime.timedelta(minutes=4)+_PAIR_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)+# No retry: an attempt can have called the model, and the next tick picks the pair again.+_PAIR_RETRY_POLICY = RetryPolicy(maximum_attempts=1)+# The tick starts no wave after this share of its interval, so the last wave ends before the next tick.+_TIME_BUDGET_SHARE = 0.6+++@frozen+class PersonalRankingTickInput:+ # A manual run can override the per-tick cap of the payload.+ limit: int | None = None+++@frozen+class TickPair:+ team_id: int+ user_id: int+++@frozen+class TickEnrollment:+ team_ids: list[int]+ organization_ids: list[uuid.UUID]+ # The organizations that a 100 percent rollout enrolls, with or without manual enrollment.+ qualifying_organization_ids: list[uuid.UUID]+++@frozen+class TickSelection:+ pairs: list[TickPair]+ due_pairs: int = 0+ due_without_list: int = 0+ # Hours since the oldest list of a due pair was built or checked. Null when no due pair has a list.+ oldest_due_hours: float | None = None+ # The share of eligible pairs whose list was built or checked inside `target_interval_minutes`.+ fresh_share: float | None = None+++@frozen+class PersonalRankingTickPlan:+ enabled: bool+ pairs: list[TickPair]+ max_concurrency: int+ max_error_rate: float+ time_budget_seconds: float+ stop_reason: str | None+ due_pairs: int = 0+ due_without_list: int = 0+ oldest_due_hours: float | None = None+ fresh_share: float | None = None+ qualifying_organizations: int = 0+ enrolled_organizations: int = 0+ enrolled_teams: int = 0+++@frozen+class PersonalRankingPairResult:+ status: PairStatus+ model_called: bool = False+++@frozen+class PersonalRankingTickResult:+ enabled: bool = False+ due_pairs: int = 0+ due_without_list: int = 0+ picked: int = 0+ statuses: dict[str, int] = dataclasses.field(default_factory=dict)+ errored: int = 0+ stop_reason: str | None = None+ oldest_due_hours: float | None = None+ fresh_share: float | None = None+ qualifying_organizations: int = 0+ enrolled_organizations: int = 0+ enrolled_teams: int = 0+ duration_seconds: float = 0.0+++def _in_rollout(percentage: float, organization_id: uuid.UUID) -> bool:+ return in_rollout(_ROLLOUT_SALT, percentage, organization_id)+++def _eligible_teams(tick: PersonalRankingTickConfig) -> QuerySet[Team]:+ """Teams with AI data processing approval and a readable activity summary, minus the skip and deny lists."""+ teams = read_enrolled_teams(read_activity_summary_config(), Team.objects.all())+ return teams.exclude(id__in=tick.skip_team_ids).exclude(+ organization_id__in=parse_organization_ids(tick.deny_organization_ids)+ )+++def enrollment(config: PersonalRankingConfig) -> TickEnrollment:+ """The teams that `is_team_enrolled` enrolls, in a fixed number of queries."""+ tick = config.tick+ if not config.enabled or not tick.enabled:+ return TickEnrollment(team_ids=[], organization_ids=[], qualifying_organization_ids=[])+ teams = _eligible_teams(tick)+ listed = parse_organization_ids(tick.organization_ids)+ with_source: dict[int, uuid.UUID] = dict(+ teams.filter(id__in=team_ids_with_enabled_source()).values_list("id", "organization_id")+ )+ enrolled = dict(teams.filter(organization_id__in=listed).values_list("id", "organization_id")) if listed else {}+ enrolled |= {+ team_id: organization_id+ for team_id, organization_id in with_source.items()+ if _in_rollout(tick.self_driving_organization_percentage, organization_id)+ }+ return TickEnrollment(+ team_ids=sorted(enrolled),+ organization_ids=sorted(set(enrolled.values())),+ qualifying_organization_ids=sorted(set(with_source.values())),+ )+++def is_team_enrolled(config: PersonalRankingConfig, team: Team) -> bool:+ """The tick may build lists for the team. The single-team form of `enrollment`.++ Enrollment decides who gets scheduled builds. It does not change who can read `for_me`.+ """+ tick = config.tick+ if not config.enabled or not tick.enabled:+ return False+ if not _eligible_teams(tick).filter(id=team.id).exists():+ return False+ if team.organization_id in parse_organization_ids(tick.organization_ids):+ return True+ return _in_rollout(tick.self_driving_organization_percentage, team.organization_id) and has_enabled_source(team.id)+++def _list_at() -> Subquery:+ """When the pair's latest list was built, or checked unchanged by a later tick."""+ return Subquery(+ PersonalInboxRanking.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id"))+ .order_by("-created_at", "-id")+ .annotate(at=Coalesce("checked_at", "created_at"))+ .values("at")[:1],+ output_field=DateTimeField(),+ )+++def _eligible_pairs(tick: PersonalRankingTickConfig, now: datetime.datetime, team_ids: list[int]) -> QuerySet:+ """The pairs of these teams with a ready summary, an active member and activity in the last `active_days`."""+ last_activity_at = Subquery(+ UserActivitySummaryQueueState.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id")).values(+ "last_activity_at"+ )[:1],+ output_field=DateTimeField(),+ )+ is_member = OrganizationMembership.objects.filter(+ user_id=OuterRef("user_id"), organization_id=OuterRef("team__organization_id")+ )+ return (+ # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)+ UserActivitySummary.all_teams.filter(+ team_id__in=team_ids, summary__isnull=False, generated_at__isnull=False, user__is_active=True+ )+ .filter(Exists(is_member))+ .annotate(+ organization_id=F("team__organization_id"),+ paying=Coalesce(F("team__organization__has_active_subscription"), Value(False)),+ last_activity_at=last_activity_at,+ list_at=_list_at(),+ )+ .filter(last_activity_at__gte=now - datetime.timedelta(days=tick.active_days))+ )+++def select_pairs(+ config: PersonalRankingConfig, now: datetime.datetime, limit: int, team_ids: list[int]+) -> TickSelection:+ """The due pairs of the enrolled teams, cut to `limit`.++ Paying organizations come first, then the most recent activity, then the oldest list. No organization+ takes more than `max_org_share` of `limit`.+ """+ if not team_ids:+ return TickSelection(pairs=[])+ tick = config.tick+ stale_before = now - datetime.timedelta(minutes=tick.target_interval_minutes)+ eligible = _eligible_pairs(tick, now, team_ids)+ due = eligible.filter(Q(list_at__isnull=True) | Q(list_at__lt=stale_before))+ order = (+ F("paying").desc(),+ F("last_activity_at").desc(nulls_last=True),+ F("list_at").asc(nulls_first=True),+ "team_id",+ "user_id",+ )+ rows = list(+ due.annotate(org_rank=Window(RowNumber(), partition_by=[F("organization_id")], order_by=order))+ .filter(org_rank__lte=max(1, int(tick.max_org_share * limit)))+ .order_by(*order)+ .values("team_id", "user_id")[:limit]+ )+ due_stats = due.aggregate(+ pairs=Count("id"), without_list=Count("id", filter=Q(list_at__isnull=True)), oldest=Min("list_at")+ )+ eligible_stats = eligible.aggregate(pairs=Count("id"), fresh=Count("id", filter=Q(list_at__gte=stale_before)))+ oldest: datetime.datetime | None = due_stats["oldest"]+ return TickSelection(+ pairs=[TickPair(team_id=row["team_id"], user_id=row["user_id"]) for row in rows],+ due_pairs=due_stats["pairs"],+ due_without_list=due_stats["without_list"],+ oldest_due_hours=(now - oldest).total_seconds() / 3600 if oldest is not None else None,+ fresh_share=eligible_stats["fresh"] / eligible_stats["pairs"] if eligible_stats["pairs"] else None,+ )+++def plan_tick(limit: int | None, now: datetime.datetime) -> PersonalRankingTickPlan:+ config = read_config()+ tick = config.tick+ enabled = config.enabled and tick.enabled+ interval_seconds = settings.PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES * 60+ time_budget_seconds = interval_seconds * _TIME_BUDGET_SHARE+ if enabled and not openai_gateway_configured():+ # Every build would fail on the same missing gateway, so the tick runs none.+ logger.warning("personal_inbox_ranking_tick_gateway_not_configured")+ return PersonalRankingTickPlan(+ enabled=False,+ pairs=[],+ max_concurrency=tick.max_concurrency,+ max_error_rate=tick.max_error_rate,+ time_budget_seconds=time_budget_seconds,+ stop_reason=STOP_GATEWAY_NOT_CONFIGURED,+ )+ enrolled = enrollment(config)+ tick_limit = limit or tick.max_pairs_per_tick+ selection = select_pairs(config, now, tick_limit, enrolled.team_ids) if enabled else TickSelection(pairs=[])+ stop_reason: str | None = None+ if selection.due_pairs > len(selection.pairs):+ stop_reason = STOP_MAX_PAIRS_PER_TICK if len(selection.pairs) >= tick_limit else STOP_MAX_ORG_SHARE+ return PersonalRankingTickPlan(+ enabled=enabled,+ pairs=selection.pairs,+ max_concurrency=tick.max_concurrency,+ max_error_rate=tick.max_error_rate,+ time_budget_seconds=time_budget_seconds,+ stop_reason=stop_reason,+ due_pairs=selection.due_pairs,+ due_without_list=selection.due_without_list,+ oldest_due_hours=selection.oldest_due_hours,+ fresh_share=selection.fresh_share,+ qualifying_organizations=len(enrolled.qualifying_organization_ids),+ enrolled_organizations=len(enrolled.organization_ids),+ enrolled_teams=len(enrolled.team_ids),+ )+++def _check_and_build(team: Team, user: User, config: PersonalRankingConfig) -> PersonalRankingPairResult:+ now = timezone.now()+ latest = latest_row(team, user)+ candidates = gather_candidates(team, user, config, now=now, with_sources=False)+ summary_version = ready_summary_version(team, user)+ current_hash = input_hash(candidates, activity_summary_version=summary_version, focus=None)+ if (+ latest is not None+ and latest.input_hash == current_hash+ and latest.fallback_reason not in RETRYABLE_FALLBACK_REASONS+ ):+ # The list is still current, so the pair leaves the due set until `target_interval_minutes` passes again.+ PersonalInboxRanking.objects.for_team(team.id).filter(id=latest.id).update(checked_at=now)+ return PersonalRankingPairResult(status="unchanged")+ calls_model = bool(candidates) and may_call_model(team, summary_ready=summary_version is not None)+ if calls_model:+ if not spend_tick_build(config):+ return PersonalRankingPairResult(status="tick_limited")+ if spend_shared_build(config) is not None:+ refund_tick_build(config)+ return PersonalRankingPairResult(status="shared_limited")+ build = build_personal_ranking(team, user, config=config, trigger="tick")+ if build.status == "disabled":+ return PersonalRankingPairResult(status="disabled")+ failed = build.model_called and build.status == "fallback"+ return PersonalRankingPairResult(status="failed" if failed else "built", model_called=build.model_called)+++def run_pair(pair: TickPair) -> PersonalRankingPairResult:+ # The payload can change between the plan and the pair, so the pair reads it again.+ config = read_config()+ team = Team.objects.select_related("organization").filter(id=pair.team_id).first()+ user = User.objects.filter(id=pair.user_id, is_active=True).first()+ if team is None or user is None or not is_team_enrolled(config, team):+ return PersonalRankingPairResult(status="not_enrolled")+ lock_key = build_lock_key(team.id, user.id)+ if not cache.add(lock_key, "1", timeout=BUILD_LOCK_TTL_SECONDS):+ return PersonalRankingPairResult(status="running")+ try:+ return _check_and_build(team, user, config)+ finally:+ cache.delete(lock_key)+++def capture_tick(result: PersonalRankingTickResult) -> None:+ properties: dict[str, Any] = {+ "deployment": settings.CLOUD_DEPLOYMENT,+ "due_pairs": result.due_pairs,+ "due_without_list": result.due_without_list,+ "picked": result.picked,+ **{status: result.statuses.get(status, 0) for status in PAIR_STATUSES},+ "errored": result.errored,+ "stop_reason": result.stop_reason,+ "oldest_due_hours": result.oldest_due_hours,+ "fresh_share": result.fresh_share,+ "qualifying_organizations": result.qualifying_organizations,+ "enrolled_organizations": result.enrolled_organizations,+ "enrolled_teams": result.enrolled_teams,+ "duration_seconds": result.duration_seconds,+ }+ logger.info("personal_inbox_ranking_tick_finished", **properties)+ ph_background_capture()(distinct_id=PIPELINE_DISTINCT_ID, event=TICK_EVENT, properties=properties)+++@scoped_temporal()+@close_db_connections+async def plan_personal_ranking_tick_activity(input: PersonalRankingTickInput) -> PersonalRankingTickPlan:+ return await database_sync_to_async(plan_tick, thread_sensitive=False)(input.limit, timezone.now())+++@scoped_temporal()+@close_db_connections+async def build_personal_ranking_pair_activity(pair: TickPair) -> PersonalRankingPairResult:+ async with Heartbeater():+ return await database_sync_to_async(run_pair, thread_sensitive=False)(pair)+++@scoped_temporal()+@close_db_connections+async def capture_personal_ranking_tick_activity(result: PersonalRankingTickResult) -> None:+ await database_sync_to_async(capture_tick, thread_sensitive=False)(result)++[email protected](name=TICK_WORKFLOW_NAME)+class PersonalRankingTickWorkflow:+ @workflow.run+ async def run(self, inputs: PersonalRankingTickInput) -> PersonalRankingTickResult:+ started = workflow.now()+ plan = await workflow.execute_activity(+ plan_personal_ranking_tick_activity,+ inputs,+ start_to_close_timeout=_PLAN_TIMEOUT,+ retry_policy=RetryPolicy(maximum_attempts=2),+ )+ if not plan.enabled:+ return PersonalRankingTickResult(stop_reason=plan.stop_reason)++ deadline = started + datetime.timedelta(seconds=plan.time_budget_seconds)+ statuses: dict[str, int] = {}+ errored = 0+ # Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.+ calls = 0+ failures = 0+ picked = 0+ stop_reason = plan.stop_reason+ concurrency = max(1, plan.max_concurrency)+ index = 0+ while index < len(plan.pairs):+ # The pairs not attempted yet stay due, so the next tick picks them.+ if calls >= _MIN_ERROR_RATE_SAMPLE and failures >= plan.max_error_rate * calls:+ stop_reason = STOP_ERROR_RATE+ break+ if workflow.now() >= deadline:+ stop_reason = STOP_TIME_BUDGET+ break+ wave = plan.pairs[index : index + concurrency]+ index += len(wave)+ picked += len(wave)+ results = await asyncio.gather(+ *(+ workflow.execute_activity(+ build_personal_ranking_pair_activity,+ pair,+ start_to_close_timeout=_PAIR_TIMEOUT,+ heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,+ retry_policy=_PAIR_RETRY_POLICY,+ )+ for pair in wave+ ),+ return_exceptions=True,+ )+ limit_stop: str | None = None+ for outcome in results:+ if isinstance(outcome, BaseException):+ # The attempt can have called the model, so it counts as a failed call.+ errored += 1+ calls += 1+ failures += 1+ continue+ statuses[outcome.status] = statuses.get(outcome.status, 0) + 1+ calls += outcome.model_called+ failures += outcome.status == "failed"+ limit_stop = limit_stop or _LIMIT_STOPS.get(outcome.status)+ if limit_stop is not None:+ stop_reason = limit_stop+ break++ result = PersonalRankingTickResult(+ enabled=True,+ due_pairs=plan.due_pairs,+ due_without_list=plan.due_without_list,+ picked=picked,+ statuses=statuses,+ errored=errored,+ stop_reason=stop_reason,+ oldest_due_hours=plan.oldest_due_hours,+ fresh_share=plan.fresh_share,+ qualifying_organizations=plan.qualifying_organizations,+ enrolled_organizations=plan.enrolled_organizations,+ enrolled_teams=plan.enrolled_teams,+ duration_seconds=(workflow.now() - started).total_seconds(),+ )+ await workflow.execute_activity(+ capture_personal_ranking_tick_activity,+ result,+ start_to_close_timeout=datetime.timedelta(minutes=1),+ retry_policy=RetryPolicy(maximum_attempts=2),+ )+ return resultdiff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.pyindex 3bdaae305790..314135814091 100644--- a/products/signals/backend/temporal/__init__.py+++ b/products/signals/backend/temporal/__init__.py@@ -14,6 +14,12 @@ from products.signals.backend.anomaly_scores.tick import (plan_anomaly_scoring_tick_activity,score_anomaly_insight_activity,)+from products.signals.backend.personal_ranking.tick import (+ PersonalRankingTickWorkflow,+ build_personal_ranking_pair_activity,+ capture_personal_ranking_tick_activity,+ plan_personal_ranking_tick_activity,+)from products.signals.backend.ranking.sweep import InboxRankingScoringWorkflow, score_inbox_reports_activityfrom products.signals.backend.temporal.agentic.report import run_agentic_report_activityfrom products.signals.backend.temporal.agentic.scout_coordinator import (@@ -251,6 +257,7 @@ ACTIVITIES = [SELF_DRIVING_WORKFLOWS = [InboxRankingScoringWorkflow,ActivitySummaryTickWorkflow,+ PersonalRankingTickWorkflow,AnomalyScoringTickWorkflow,AnomalyScoringDueListRefreshWorkflow,]@@ -259,6 +266,9 @@ SELF_DRIVING_ACTIVITIES = [plan_activity_summary_tick_activity,update_activity_summary_pair_activity,capture_activity_summary_tick_activity,+ plan_personal_ranking_tick_activity,+ build_personal_ranking_pair_activity,+ capture_personal_ranking_tick_activity,plan_anomaly_scoring_tick_activity,score_anomaly_insight_activity,capture_anomaly_scoring_tick_activity,diff --git a/products/signals/backend/test/test_personal_ranking.py b/products/signals/backend/test/test_personal_ranking.pyindex c3ac114d0f08..d367991ca051 100644--- a/products/signals/backend/test/test_personal_ranking.py+++ b/products/signals/backend/test/test_personal_ranking.py@@ -39,6 +39,7 @@ from products.signals.backend.personal_ranking.candidates import Candidatefrom products.signals.backend.personal_ranking.config import (CandidatePool,PersonalRankingConfig,+ PersonalRankingTickConfig,ReasonLimits,parse_config,)@@ -364,6 +365,45 @@ class TestConfig(SimpleTestCase):),("reasoning_effort", {"reasoning_effort": "high"}, PersonalRankingConfig(reasoning_effort="high")),("invalid_reasoning_effort", {"reasoning_effort": "max"}, PersonalRankingConfig()),+ ("tick_without_enabled_stays_off", {"tick": {"organization_ids": ["org"]}}, PersonalRankingConfig()),+ (+ "tick_drops_invalid_values_and_caps_large_ones",+ {+ "tick": {+ "enabled": True,+ "organization_ids": ["org", 3, ""],+ "skip_team_ids": [1, True, "2"],+ "self_driving_organization_percentage": 101,+ "max_pairs_per_tick": 10_000,+ "max_tick_builds_per_day": -1,+ "max_org_share": 2,+ }+ },+ PersonalRankingConfig(+ tick=PersonalRankingTickConfig(+ enabled=True, organization_ids=("org",), skip_team_ids=(1,), max_pairs_per_tick=500+ )+ ),+ ),+ (+ "enabled_tick_raises_freshness_and_leaves_a_shared_reserve",+ {+ "max_builds_per_day": 100,+ "tick": {"enabled": True, "target_interval_minutes": 180, "max_tick_builds_per_day": 100},+ },+ PersonalRankingConfig(+ max_builds_per_day=100,+ freshness_minutes=180,+ tick=PersonalRankingTickConfig(+ enabled=True, target_interval_minutes=180, max_tick_builds_per_day=80+ ),+ ),+ ),+ (+ "disabled_tick_leaves_freshness_alone",+ {"tick": {"target_interval_minutes": 180}},+ PersonalRankingConfig(),+ ),])def test_parse_config(self, _name: str, payload: Any, expected: PersonalRankingConfig) -> None:@@ -889,6 +929,23 @@ class TestForMeEndpoint(APIBaseTest):assert [args[1] for args in self.queued] == [self.user.id, third.id]+ def test_a_list_the_tick_checked_stays_fresh_past_its_build_time(self) -> None:+ self._claimed_report("First")+ with time_machine.travel(timezone.now(), tick=False) as traveller:+ self._get()+ self._run_builds()+ traveller.shift(datetime.timedelta(hours=2))+ assert self._get()["stale"] is True+ self._run_builds()++ traveller.shift(datetime.timedelta(hours=2))+ PersonalInboxRanking.objects.for_team(self.team.id).filter(user_id=self.user.id).update(+ checked_at=timezone.now()+ )+ body = self._get()++ assert (self._state(body), self.queued) == ((True, False, False), [])+def test_another_user_never_reads_your_list(self) -> None:self._get()self._run_builds()diff --git a/products/signals/backend/test/test_personal_ranking_tick.py b/products/signals/backend/test/test_personal_ranking_tick.pynew file mode 100644index 000000000000..489cf4360cbd--- /dev/null+++ b/products/signals/backend/test/test_personal_ranking_tick.py@@ -0,0 +1,377 @@+import json+import uuid+import asyncio+import datetime+from types import SimpleNamespace+from typing import Any++import pytest+import time_machine+from posthog.test.base import BaseTest+from unittest.mock import MagicMock, patch++from django.core.cache import cache+from django.utils import timezone++from parameterized import parameterized+from temporalio import activity+from temporalio.testing import WorkflowEnvironment+from temporalio.worker import UnsandboxedWorkflowRunner, Worker++from posthog.models import Organization, Team, User+from posthog.redis import TEST_clear_clients+from posthog.token_bucket import TEST_reset_scripts++from products.signals.backend.activity_summary.config import ActivitySummaryConfig+from products.signals.backend.activity_summary.read import ActivitySummaryStatus, ActivitySummaryView+from products.signals.backend.models import (+ PersonalInboxRanking,+ SignalReport,+ SignalReportAssignment,+ UserActivitySummary,+ UserActivitySummaryQueueState,+)+from products.signals.backend.personal_ranking.build_budget import spend_shared_build, spend_tick_build+from products.signals.backend.personal_ranking.config import parse_config+from products.signals.backend.personal_ranking.tick import (+ STOP_ERROR_RATE,+ STOP_MAX_BUILDS_PER_DAY,+ STOP_MAX_TICK_BUILDS_PER_DAY,+ STOP_TIME_BUDGET,+ PersonalRankingPairResult,+ PersonalRankingTickInput,+ PersonalRankingTickPlan,+ PersonalRankingTickResult,+ PersonalRankingTickWorkflow,+ TickPair,+ enrollment,+ is_team_enrolled,+ run_pair,+ select_pairs,+)+from products.signals.backend.personal_ranking_tasks import build_lock_key++NOW = datetime.datetime(2026, 10, 20, 12, 0, tzinfo=datetime.UTC)+_TICK = "products.signals.backend.personal_ranking.tick"+++class _TickTest(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.tick: dict[str, Any] = {"enabled": True, "organization_ids": [str(self.organization.id)]}+ self.payload: dict[str, Any] = {}+ self.source_team_ids: set[int] = set()+ for target, value in (+ (+ "products.signals.backend.personal_ranking.config.read_flag_payload",+ MagicMock(side_effect=lambda *_: {**self.payload, "tick": self.tick}),+ ),+ (+ f"{_TICK}.read_activity_summary_config",+ MagicMock(return_value=ActivitySummaryConfig(enabled=True, enroll_all_approved=True)),+ ),+ (f"{_TICK}.team_ids_with_enabled_source", MagicMock(side_effect=lambda: self.source_team_ids)),+ (f"{_TICK}.has_enabled_source", MagicMock(side_effect=lambda team_id: team_id in self.source_team_ids)),+ ):+ patcher = patch(target, value)+ patcher.start()+ self.addCleanup(patcher.stop)++ def _config(self):+ return parse_config({**self.payload, "tick": self.tick})++ def _pair(+ self,+ team: Team,+ email: str,+ *,+ active_hours_ago: float,+ list_hours_ago: float | None = None,+ checked_hours_ago: float | None = None,+ ) -> User:+ now = timezone.now()+ user = User.objects.create_and_join(team.organization, email, None)+ UserActivitySummary.objects.for_team(team.id).create(+ team_id=team.id, user_id=user.id, status=UserActivitySummary.Status.OK, summary={}, generated_at=now+ )+ UserActivitySummaryQueueState.objects.for_team(team.id).create(+ team_id=team.id, user_id=user.id, last_activity_at=now - datetime.timedelta(hours=active_hours_ago)+ )+ if list_hours_ago is not None:+ row = PersonalInboxRanking.objects.for_team(team.id).create(+ team_id=team.id, user_id=user.id, input_hash="old", model="m", prompt_version=1, status="ok"+ )+ PersonalInboxRanking.objects.for_team(team.id).filter(id=row.id).update(+ created_at=now - datetime.timedelta(hours=list_hours_ago),+ checked_at=None if checked_hours_ago is None else now - datetime.timedelta(hours=checked_hours_ago),+ )+ return user+++class TestTickEnrollment(_TickTest):+ @parameterized.expand(+ [+ ("nothing_at_zero_without_a_list", {"organization_ids": []}, True, True, False),+ ("a_listed_organization", {}, False, True, True),+ ("a_listed_organization_without_approval", {}, False, False, False),+ ("a_denied_organization", {"deny": True}, False, True, False),+ ("a_skipped_team", {"skip": True}, False, True, False),+ ("every_team_with_a_source_at_100", {"organization_ids": [], "percentage": 100}, True, True, True),+ ("no_team_without_a_source_at_100", {"organization_ids": [], "percentage": 100}, False, True, False),+ ("the_tick_off", {"enabled": False}, True, True, False),+ ]+ )+ def test_enrollment(+ self, _name: str, overrides: dict[str, Any], has_source: bool, approved: bool, expected: bool+ ) -> None:+ self.organization.is_ai_data_processing_approved = approved+ self.organization.save()+ if has_source:+ self.source_team_ids = {self.team.id}+ self.tick = {+ **self.tick,+ "enabled": overrides.get("enabled", True),+ "organization_ids": overrides.get("organization_ids", self.tick["organization_ids"]),+ "self_driving_organization_percentage": overrides.get("percentage", 0),+ "deny_organization_ids": [str(self.organization.id)] if overrides.get("deny") else [],+ "skip_team_ids": [self.team.id] if overrides.get("skip") else [],+ }+ config = self._config()+ self.team.refresh_from_db()++ assert (enrollment(config).team_ids, is_team_enrolled(config, self.team)) == (+ [self.team.id] if expected else [],+ expected,+ )+++class TestTickSelection(_TickTest):+ def test_selects_due_pairs_paying_first_then_by_activity_and_oldest_list_within_the_org_share(self) -> None:+ paying = Team.objects.create(+ organization=Organization.objects.create(name="Paying", has_active_subscription=True), name="Paying"+ )+ free = Team.objects.create(organization=Organization.objects.create(name="Free"), name="Free")+ with time_machine.travel(NOW, tick=False):+ users = {+ "paying": self._pair(paying, "[email protected]", active_hours_ago=30),+ "recent_no_list": self._pair(free, "[email protected]", active_hours_ago=1),+ "recent_old_list": self._pair(free, "[email protected]", active_hours_ago=1, list_hours_ago=3),+ # Past this organization's share of the tick.+ "older": self._pair(free, "[email protected]", active_hours_ago=2),+ "oldest_activity": self._pair(self.team, "[email protected]", active_hours_ago=3, list_hours_ago=5),+ "fresh_list": self._pair(self.team, "[email protected]", active_hours_ago=1, list_hours_ago=0.2),+ "checked_recently": self._pair(+ self.team, "[email protected]", active_hours_ago=1, list_hours_ago=3, checked_hours_ago=0.2+ ),+ "inactive": self._pair(self.team, "[email protected]", active_hours_ago=24 * 10),+ }+ selection = select_pairs(+ parse_config({"tick": {"enabled": True, "max_org_share": 0.5}}),+ NOW,+ 4,+ [paying.id, free.id, self.team.id],+ )++ assert [pair.user_id for pair in selection.pairs] == [+ users[name].id for name in ("paying", "recent_no_list", "recent_old_list", "oldest_activity")+ ]+ assert (selection.due_pairs, selection.due_without_list, selection.oldest_due_hours) == (5, 3, 5.0)+ assert selection.fresh_share == 2 / 7+++class TestTickPair(_TickTest):+ def setUp(self) -> None:+ super().setUp()+ TEST_clear_clients()+ TEST_reset_scripts()+ cache.delete(build_lock_key(self.team.id, self.user.id))+ self.malformed_answers = 0+ self.llm = MagicMock()+ self.create = self.llm.with_options.return_value.chat.completions.create+ self.create.side_effect = lambda **_: SimpleNamespace(+ choices=[SimpleNamespace(finish_reason="stop", message=SimpleNamespace(content=self._answer()))],+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5),+ )+ scoped_capture = MagicMock()+ self.capture = scoped_capture.return_value.__enter__.return_value+ summary = ActivitySummaryView(status=ActivitySummaryStatus.READY, version=1)+ for target, value in (+ ("products.signals.backend.personal_ranking.logic.build_openai_client", MagicMock(return_value=self.llm)),+ ("products.signals.backend.personal_ranking.logic.get_summary", MagicMock(return_value=summary)),+ ("products.signals.backend.personal_ranking.serve.get_summary", MagicMock(return_value=summary)),+ ("products.signals.backend.personal_ranking.logic.ph_scoped_capture", scoped_capture),+ (+ "products.signals.backend.personal_ranking.candidates.fetch_source_products_for_reports",+ MagicMock(return_value={}),+ ),+ ):+ patcher = patch(target, value)+ patcher.start()+ self.addCleanup(patcher.stop)+ report = SignalReport.objects.create(+ team=self.team, title="Claimed", summary="Claimed", signal_count=1, total_weight=1.0, status="ready"+ )+ SignalReportAssignment.all_teams.create(team=self.team, report=report, actor_kind="user", actor_user=self.user)+ UserActivitySummary.objects.for_team(self.team.id).create(+ team_id=self.team.id,+ user_id=self.user.id,+ status=UserActivitySummary.Status.OK,+ summary={},+ generated_at=timezone.now(),+ )+ UserActivitySummaryQueueState.objects.for_team(self.team.id).create(+ team_id=self.team.id, user_id=self.user.id, last_activity_at=timezone.now()+ )++ def _answer(self) -> str:+ if self.malformed_answers:+ self.malformed_answers -= 1+ return "not json"+ return json.dumps(+ {+ "ranked": [{"ref": "c1", "rank": 1, "tier": "top", "reason": "r", "matched": None, "next_step": None}],+ "folded": [],+ }+ )++ def _run(self) -> PersonalRankingPairResult:+ return run_pair(TickPair(team_id=self.team.id, user_id=self.user.id))++ def _rows(self):+ return PersonalInboxRanking.objects.for_team(self.team.id).filter(user_id=self.user.id)++ @parameterized.expand(+ [+ ("a_ranked_list_is_kept", 0, "built", "unchanged", 1),+ ("a_failed_model_call_is_retried", 1, "failed", "built", 2),+ ]+ )+ def test_an_unchanged_pair_calls_no_model_unless_its_last_build_failed(+ self, _name: str, malformed_answers: int, first: str, second: str, model_calls: int+ ) -> None:+ self.malformed_answers = malformed_answers+ assert self._run() == PersonalRankingPairResult(status=first, model_called=True) # type: ignore[arg-type]+ assert self.capture.call_args.kwargs["properties"]["trigger"] == "tick"+ self._rows().update(created_at=timezone.now() - datetime.timedelta(hours=2))+ assert len(select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs) == 1++ assert self._run() == PersonalRankingPairResult(+ status=second, # type: ignore[arg-type]+ model_called=second == "built",+ )++ assert self.create.call_count == model_calls+ assert select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs == []++ @parameterized.expand([("running",), ("tick_limited",), ("shared_limited",), ("not_enrolled",)])+ def test_a_pair_that_may_not_build_calls_no_model(self, status: str) -> None:+ self.payload = {"max_builds_per_day": 1}+ self.tick = {**self.tick, "max_tick_builds_per_day": 1}+ if status == "running":+ cache.add(build_lock_key(self.team.id, self.user.id), "1")+ elif status == "tick_limited":+ assert spend_tick_build(self._config())+ elif status == "shared_limited":+ assert spend_shared_build(self._config()) is None+ else:+ self.tick = {**self.tick, "organization_ids": []}++ assert self._run() == PersonalRankingPairResult(status=status) # type: ignore[arg-type]++ assert (self.create.call_count, self._rows().count()) == (0, 0)+ if status == "shared_limited":+ # The tick's own limit gets its build back.+ assert spend_tick_build(self._config())+++def _plan(pairs: int, **overrides: Any) -> PersonalRankingTickPlan:+ fields: dict[str, Any] = {+ "enabled": True,+ "pairs": [TickPair(team_id=1, user_id=user_id) for user_id in range(pairs)],+ "max_concurrency": 2,+ "max_error_rate": 0.5,+ "time_budget_seconds": 600.0,+ "stop_reason": None,+ "due_pairs": pairs,+ "enrolled_teams": 3,+ }+ return PersonalRankingTickPlan(**{**fields, **overrides})+++async def _run_tick(+ plan: PersonalRankingTickPlan, pair_status: str+) -> tuple[PersonalRankingTickResult, int, list[PersonalRankingTickResult]]:+ running = peak = 0+ captured: list[PersonalRankingTickResult] = []++ @activity.defn(name="plan_personal_ranking_tick_activity")+ async def fake_plan(input: PersonalRankingTickInput) -> PersonalRankingTickPlan:+ return plan++ @activity.defn(name="build_personal_ranking_pair_activity")+ async def fake_pair(pair: TickPair) -> PersonalRankingPairResult:+ nonlocal running, peak+ running += 1+ peak = max(peak, running)+ await asyncio.sleep(0.01)+ running -= 1+ return PersonalRankingPairResult(+ status=pair_status, # type: ignore[arg-type]+ model_called=pair_status in ("built", "failed"),+ )++ @activity.defn(name="capture_personal_ranking_tick_activity")+ async def fake_capture(result: PersonalRankingTickResult) -> None:+ captured.append(result)++ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue="personal-ranking-tick-test",+ workflows=[PersonalRankingTickWorkflow],+ activities=[fake_plan, fake_pair, fake_capture],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ result = await env.client.execute_workflow(+ PersonalRankingTickWorkflow.run,+ PersonalRankingTickInput(),+ id=f"personal-ranking-tick-{uuid.uuid4()}",+ task_queue="personal-ranking-tick-test",+ )+ return result, peak, captured+++ "plan,pair_status,picked,stop_reason",+ [+ (_plan(5), "built", 5, None),+ (_plan(5), "unchanged", 5, None),+ # The wave that met the limit finishes, and no later wave starts.+ (_plan(5), "tick_limited", 2, STOP_MAX_TICK_BUILDS_PER_DAY),+ (_plan(5), "shared_limited", 2, STOP_MAX_BUILDS_PER_DAY),+ # The breaker waits for a sample of 10 model calls, which is five waves.+ (_plan(20), "failed", 10, STOP_ERROR_RATE),+ (_plan(5, time_budget_seconds=0.0), "built", 0, STOP_TIME_BUDGET),+ ],+)+async def test_the_tick_bounds_concurrency_and_stops_at_each_limit(+ plan: PersonalRankingTickPlan, pair_status: str, picked: int, stop_reason: str | None+) -> None:+ result, peak, captured = await _run_tick(plan, pair_status)++ assert peak <= plan.max_concurrency+ assert (result.picked, result.statuses, result.stop_reason) == (+ picked,+ {pair_status: picked} if picked else {},+ stop_reason,+ )+ assert (result.due_pairs, result.enrolled_teams) == (plan.due_pairs, plan.enrolled_teams)+ assert captured == [result]+++async def test_a_disabled_tick_runs_no_pair() -> None:+ result, peak, captured = await _run_tick(_plan(0, enabled=False), "built")++ assert (result, peak, captured) == (PersonalRankingTickResult(), 0, [])