Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): build personal inbox rankings on a scheduled tick (#127617)

posthog[bot]2f15a3f27b03
diff --git a/posthog/settings/object_storage.py b/posthog/settings/object_storage.py
index adb2a5b0d661..986a465d53ba 100644
--- a/posthog/settings/object_storage.py
+++ b/posthog/settings/object_storage.py
@@ -145,6 +145,12 @@ INBOX_RANKING_SCORING_MAX_REPORTS_PER_TICK = get_from_env(
# Activity summary tick (products/signals/backend/activity_summary/tick.py). The `activity-summary`
# flag payload gates the work and holds the caps, so the schedule ticks in every environment.
ACTIVITY_SUMMARY_TICK_INTERVAL_MINUTES = get_from_env("ACTIVITY_SUMMARY_TICK_INTERVAL_MINUTES", 60, type_cast=int)
+# Personal inbox ranking tick (products/signals/backend/personal_ranking/tick.py). The `tick` object of the
+# `personal-inbox-ranking` flag payload gates the work. It ticks more often than the list's target interval,
+# so a list comes due soon after the interval passes.
+PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES = get_from_env(
+ "PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES", 15, type_cast=int
+)
# Identity matching scratch storage (products/growth `identity_matching_job`). The job writes
# per-run Parquet objects via ClickHouse `INSERT INTO FUNCTION s3(...)` and the read API globs
diff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.py
index f561fc3c8b23..808184f0feef 100644
--- a/posthog/temporal/schedule.py
+++ b/posthog/temporal/schedule.py
@@ -145,6 +145,7 @@ from products.signals.backend.facade.temporal import (
create_anomaly_scoring_tick_schedule,
create_conversations_signals_coordinator_schedule,
create_inbox_ranking_scoring_schedule,
+ create_personal_inbox_ranking_tick_schedule,
create_scout_suggestions_coordinator_schedule,
create_signals_scout_coordinator_schedule,
)
@@ -1021,6 +1022,7 @@ schedules = [
create_signals_scout_coordinator_schedule,
create_inbox_ranking_scoring_schedule,
create_activity_summary_tick_schedule,
+ create_personal_inbox_ranking_tick_schedule,
create_anomaly_scoring_tick_schedule,
create_anomaly_scoring_due_list_refresh_schedule,
create_scout_suggestions_coordinator_schedule,
diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.py
index c99b647b4786..565e6e65d0f7 100644
--- a/products/signals/backend/activity_summary/queue.py
+++ b/products/signals/backend/activity_summary/queue.py
@@ -15,7 +15,6 @@ day per day.
import math
import uuid
-import hashlib
import datetime
import dataclasses
from collections.abc import Callable, Iterable, Sequence
@@ -45,6 +44,7 @@ from django.db.models.functions import Cast, Coalesce, Extract, Greatest, Ln, Ro
from posthog.dataclasses import frozen
from posthog.models import ActivityLog, FileSystemViewLog, Team
+from products.signals.backend import org_rollout
from products.signals.backend.activity_summary.config import ActivitySummaryConfig, SourceWeights
from products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS
from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX
@@ -202,17 +202,15 @@ def daily_usage(now: datetime.datetime) -> DailyUsage:
# A new salt reorders every organization, so change the version only to start a new rollout order.
_ROLLOUT_SALT = "activity-summary-self-driving-v1"
-_ROLLOUT_BUCKETS = 10_000
def organization_rollout_bucket(organization_id: uuid.UUID) -> int:
- """The organization's fixed place in the percentage rollout, from 0 to `_ROLLOUT_BUCKETS` - 1."""
- digest = hashlib.sha256(f"{_ROLLOUT_SALT}:{organization_id}".encode()).digest()
- return int.from_bytes(digest[:8], "big") % _ROLLOUT_BUCKETS
+ """The organization's fixed place in the percentage rollout, from 0 to 9999."""
+ return org_rollout.organization_rollout_bucket(_ROLLOUT_SALT, organization_id)
def _in_rollout(percentage: float, organization_id: uuid.UUID) -> bool:
- return organization_rollout_bucket(organization_id) < percentage * _ROLLOUT_BUCKETS / 100
+ return org_rollout.in_rollout(_ROLLOUT_SALT, percentage, organization_id)
@frozen
@@ -312,15 +310,17 @@ def is_team_enrolled_for_read(config: ActivitySummaryConfig, team: Team) -> bool
return config.enroll_all_approved or team.organization_id in _parse_organization_ids(config.organization_ids)
-def _parse_organization_ids(organization_ids: tuple[str, ...]) -> set[uuid.UUID]:
- # A UUID parse, so an id in another UUID format still matches, and an invalid id matches nothing.
- parsed: set[uuid.UUID] = set()
- for organization_id in organization_ids:
- try:
- parsed.add(uuid.UUID(organization_id))
- except ValueError:
- continue
- return parsed
+def read_enrolled_teams(config: ActivitySummaryConfig, teams: QuerySet[Team]) -> QuerySet[Team]:
+ """The teams of `teams` that `is_team_enrolled_for_read` enrolls, in one query."""
+ if not config.enabled:
+ return teams.none()
+ teams = teams.filter(organization__is_ai_data_processing_approved=True).exclude(id__in=config.skip_team_ids)
+ if config.enroll_all_approved:
+ return teams
+ return teams.filter(organization_id__in=_parse_organization_ids(config.organization_ids))
+
+
+_parse_organization_ids = org_rollout.parse_organization_ids
def _add(pending: _Pending, pair: ActivitySummaryPair, units: int, at: datetime.datetime, score: float = 0.0) -> None:
diff --git a/products/signals/backend/facade/temporal.py b/products/signals/backend/facade/temporal.py
index 162622d3749d..42713e07cfe8 100644
--- a/products/signals/backend/facade/temporal.py
+++ b/products/signals/backend/facade/temporal.py
@@ -7,6 +7,7 @@ from products.signals.backend.anomaly_scores.schedule import (
)
from products.signals.backend.emission.conversations_schedule import create_conversations_signals_coordinator_schedule
from products.signals.backend.emission.temporal_settings import EMIT_SIGNALS_ACTIVITIES, EMIT_SIGNALS_WORKFLOWS
+from products.signals.backend.personal_ranking.schedule import create_personal_inbox_ranking_tick_schedule
from products.signals.backend.ranking.schedule import create_inbox_ranking_scoring_schedule
from products.signals.backend.temporal import ACTIVITIES, SELF_DRIVING_ACTIVITIES, SELF_DRIVING_WORKFLOWS, WORKFLOWS
from products.signals.backend.temporal.agentic.schedule import (
@@ -33,6 +34,7 @@ __all__ = [
"create_anomaly_scoring_tick_schedule",
"create_conversations_signals_coordinator_schedule",
"create_inbox_ranking_scoring_schedule",
+ "create_personal_inbox_ranking_tick_schedule",
"create_scout_suggestions_coordinator_schedule",
"create_signals_scout_coordinator_schedule",
"emit_eval_signal_activity",
diff --git a/products/signals/backend/management/AGENTS.md b/products/signals/backend/management/AGENTS.md
index e962750edcdc..34faf23bf665 100644
--- a/products/signals/backend/management/AGENTS.md
+++ b/products/signals/backend/management/AGENTS.md
@@ -305,13 +305,48 @@ The `personal inbox ranking built` event records `model_called`, so builds with
The command spends no build limit, so use it only for single runs.
`GET /api/projects/:id/signals/reports/for_me/` (`../personal_ranking/serve.py`) serves the same list to the user it describes, when the flag is on for that user.
-It reuses the latest row for `freshness_minutes` while the input hash matches.
+It reuses the latest row for `freshness_minutes` after the row was built or last checked by the tick, while the input hash matches.
Otherwise it queues one `build_personal_inbox_ranking` Celery task per person behind a cache lock, and serves the latest row as stale, or the candidates in inbox order with `fallback_reason: building`.
It limits builds per person with `build_burst`, `builds_per_hour` and `builds_per_day` from the same payload.
`max_builds_per_day` (default 500, at most 10000) limits the builds that call the model across every person, in one Redis token bucket per region (`../personal_ranking/build_budget.py`).
A build with no ready activity summary, or for an organization that has not approved AI data processing, does not call the model and does not spend that limit.
When a limit refuses a build, the read serves the latest row as stale, or the candidates in inbox order with `fallback_reason: rate_limited`.
-A scheduled caller must spend from the same bucket with `spend_shared_build` before it builds.
+The scheduled tick spends from the same bucket.
+
+### Scheduled personal ranking tick
+
+`personal-inbox-ranking-tick` (`../personal_ranking/tick.py`) is a Temporal schedule on `SELF_DRIVING_TASK_QUEUE` with overlap `SKIP`.
+It runs every `PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES` (default 15) in every environment, and builds lists ahead of the first `for_me` read.
+The `tick` object of the `personal-inbox-ranking` payload holds its levers. Without `"enabled": true` in that object the tick does nothing, and `"enabled": false` at the top level stops it too.
+
+```json
+{
+ "tick": {
+ "enabled": true,
+ "self_driving_organization_percentage": 5,
+ "organization_ids": ["<organization uuid>"],
+ "deny_organization_ids": [],
+ "skip_team_ids": [],
+ "target_interval_minutes": 60,
+ "active_days": 7,
+ "max_pairs_per_tick": 20,
+ "max_concurrency": 4,
+ "max_tick_builds_per_day": 100,
+ "max_org_share": 0.25,
+ "max_error_rate": 0.5
+ }
+}
+```
+
+- **Enrollment:** every project of the listed organizations, plus the salted `self_driving_organization_percentage` of the organizations with a project that has an enabled signal source. The percentage enrolls only those projects, and a higher percentage keeps the organizations of a lower one. The organization must approve AI data processing, and the activity summary must be readable for the project. The deny list and `skip_team_ids` win. Enrollment decides who gets scheduled builds. It does not change who can read `for_me`.
+- **Due pairs:** a member with a ready activity summary, activity in the last `active_days`, and a latest list that was built or checked longer ago than `target_interval_minutes`, or no list.
+- **Order:** paying organizations first, then the most recent activity, then the oldest list. One organization takes at most `max_org_share` of `max_pairs_per_tick`.
+- **Unchanged inputs:** the pair activity compares the input hash first. A match calls no model and sets `checked_at` on the latest row, so the pair is not due again until `target_interval_minutes` passes, and `for_me` serves that row as fresh. An enabled tick raises `freshness_minutes` to at least `target_interval_minutes`, so a plain read does not rebuild a list inside the tick's interval. A row whose model call failed, timed out or answered outside the schema, or that skipped the model without AI approval, is built again instead.
+- **Limits:** a build that calls the model spends `max_tick_builds_per_day` (one Redis token bucket per region, closed when Redis is unreachable) and then the shared `max_builds_per_day`. `max_tick_builds_per_day` is limited to 80% of `max_builds_per_day`, so `refresh` and `focus` requests keep a reserve. To run the tick above that, raise the shared limit too. The tick stops at the first refusal, at `max_error_rate` after 10 model calls, and when 60% of its interval has passed.
+- **Locks:** the pair takes the same per-person lock as `for_me`, so a tick skips a person whose build is running.
+
+Each enabled tick captures `personal inbox ranking tick` with the due pairs, the pairs it picked, a count per pair status (`built`, `failed`, `unchanged`, `running`, `not_enrolled`, `tick_limited`, `shared_limited`), the stop reason, `oldest_due_hours`, `fresh_share`, the enrolled organizations and teams, and the duration.
+`personal inbox ranking built` and the model call's trace carry `trigger` (`tick` or `on_demand`), so cost can be split by what started a build.
## Tips
diff --git a/products/signals/backend/migrations/0227_personalinboxranking_checked_at.py b/products/signals/backend/migrations/0227_personalinboxranking_checked_at.py
new file mode 100644
index 000000000000..9c2c43995253
--- /dev/null
+++ b/products/signals/backend/migrations/0227_personalinboxranking_checked_at.py
@@ -0,0 +1,17 @@
+# Generated by Django 6.1.2 on 2026-11-02 16:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0226_signalreport_trgm_fastupdate_off"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="personalinboxranking",
+ name="checked_at",
+ field=models.DateTimeField(blank=True, null=True),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 8ab28bae09ce..fc3442974dc3 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0226_signalreport_trgm_fastupdate_off
+0227_personalinboxranking_checked_at
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index e42c6ff9294d..92ebaff3dbe6 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -4345,6 +4345,8 @@ class PersonalInboxRanking(TeamScopedRootMixin, UUIDModel):
status = models.CharField(max_length=20, choices=Status)
# One of the fixed strings of `personal_ranking/logic.py`, never model output.
fallback_reason = models.CharField(max_length=100, null=True, blank=True)
+ # The last time the scheduled tick found the inputs unchanged, so the list is current as of then.
+ checked_at = models.DateTimeField(null=True, blank=True)
class Meta:
default_manager_name = "all_teams"
diff --git a/products/signals/backend/org_rollout.py b/products/signals/backend/org_rollout.py
new file mode 100644
index 000000000000..07ecf82c46da
--- /dev/null
+++ b/products/signals/backend/org_rollout.py
@@ -0,0 +1,27 @@
+"""A salted percentage rollout over organizations, and the organization id lists of a flag payload."""
+
+import uuid
+import hashlib
+
+_ROLLOUT_BUCKETS = 10_000
+
+
+def organization_rollout_bucket(salt: str, organization_id: uuid.UUID) -> int:
+ """The organization's fixed place in the rollout of `salt`, from 0 to `_ROLLOUT_BUCKETS` - 1."""
+ digest = hashlib.sha256(f"{salt}:{organization_id}".encode()).digest()
+ return int.from_bytes(digest[:8], "big") % _ROLLOUT_BUCKETS
+
+
+def in_rollout(salt: str, percentage: float, organization_id: uuid.UUID) -> bool:
+ return organization_rollout_bucket(salt, organization_id) < percentage * _ROLLOUT_BUCKETS / 100
+
+
+def parse_organization_ids(organization_ids: tuple[str, ...]) -> set[uuid.UUID]:
+ # A UUID parse, so an id in another UUID format still matches, and an invalid id matches nothing.
+ parsed: set[uuid.UUID] = set()
+ for organization_id in organization_ids:
+ try:
+ parsed.add(uuid.UUID(organization_id))
+ except ValueError:
+ continue
+ return parsed
diff --git a/products/signals/backend/personal_ranking/build_budget.py b/products/signals/backend/personal_ranking/build_budget.py
index d9b1315cc241..081f33922e35 100644
--- a/products/signals/backend/personal_ranking/build_budget.py
+++ b/products/signals/backend/personal_ranking/build_budget.py
@@ -1,31 +1,37 @@
"""The daily limit on personal ranking builds that call the model, shared by every person in one region.
-The `for_me` read spends one build here before it queues a build that may call the model. A scheduled caller
-spends from the same bucket. Each region has its own Redis, so each region counts on its own.
+The `for_me` read spends one build here before it queues a build that may call the model. The scheduled tick
+spends from the same bucket, after it spends from its own bucket of `max_tick_builds_per_day`. Each region has its
+own Redis, so each region counts on its own.
"""
from django.conf import settings
import structlog
-from posthog.token_bucket import BucketUnavailable, Budget, consume
+from posthog.token_bucket import BucketUnavailable, Budget, consume, refund
from products.signals.backend.personal_ranking.config import PersonalRankingConfig
logger = structlog.get_logger(__name__)
_BUCKET_PREFIX = "personal_ranking_build_rate:shared"
+_TICK_BUCKET_PREFIX = "personal_ranking_build_rate:tick"
# A build costs one token per hour of the day, and the bucket refills `max_builds_per_day` tokens an hour,
# so it refills exactly `max_builds_per_day` builds a day. A whole-token rate per build would round it up.
DAILY_BUILD_COST = 24
-def _key() -> str:
- return f"{_BUCKET_PREFIX}:{settings.CLOUD_DEPLOYMENT or 'local'}"
+def _key(prefix: str = _BUCKET_PREFIX) -> str:
+ return f"{prefix}:{settings.CLOUD_DEPLOYMENT or 'local'}"
+
+
+def _daily_budget(builds_per_day: int) -> Budget:
+ return Budget(burst=builds_per_day * DAILY_BUILD_COST, per_hour=builds_per_day)
def _budget(config: PersonalRankingConfig) -> Budget:
- return Budget(burst=config.max_builds_per_day * DAILY_BUILD_COST, per_hour=config.max_builds_per_day)
+ return _daily_budget(config.max_builds_per_day)
def spend_shared_build(config: PersonalRankingConfig) -> int | None:
@@ -36,3 +42,19 @@ def spend_shared_build(config: PersonalRankingConfig) -> int | None:
logger.warning("signals.personal_ranking.shared_build_bucket_unavailable")
return None
return None if decision.allowed else decision.retry_after
+
+
+def spend_tick_build(config: PersonalRankingConfig) -> bool:
+ """Spend one build of the tick's daily limit. Returns False when the limit or an unreachable Redis refuses it."""
+ decision = consume(
+ _key(_TICK_BUCKET_PREFIX), _daily_budget(config.tick.max_tick_builds_per_day), cost=DAILY_BUILD_COST
+ )
+ if isinstance(decision, BucketUnavailable):
+ # Fail closed: nobody waits on a tick build, so a skipped build costs only a stale list.
+ logger.warning("signals.personal_ranking.tick_build_bucket_unavailable")
+ return False
+ return decision.allowed
+
+
+def refund_tick_build(config: PersonalRankingConfig) -> None:
+ refund(_key(_TICK_BUCKET_PREFIX), _daily_budget(config.tick.max_tick_builds_per_day), cost=DAILY_BUILD_COST)
diff --git a/products/signals/backend/personal_ranking/config.py b/products/signals/backend/personal_ranking/config.py
index 8871638bc8ad..9c8f3aa3885c 100644
--- a/products/signals/backend/personal_ranking/config.py
+++ b/products/signals/backend/personal_ranking/config.py
@@ -7,9 +7,15 @@ person in one region.
The optional `pools` list replaces the default candidate pools, in order. An invalid entry is dropped
and logged. When no entry is valid, the build uses the default pools.
+
+The optional `tick` object holds the levers of the scheduled tick (`tick.py`). Its parse fails closed:
+a missing or malformed object, or one without `"enabled": true`, turns the tick off. An enabled tick raises
+`freshness_minutes` to its `target_interval_minutes`, and limits `max_tick_builds_per_day` to a share of
+`max_builds_per_day`.
"""
import re
+from dataclasses import replace
from typing import Any, Literal, cast, get_args
import structlog
@@ -80,6 +86,44 @@ class CandidatePool:
window_days: int | None = None
+@frozen
+class PersonalRankingTickConfig:
+ enabled: bool = False
+ # Builds for every project of these organizations.
+ organization_ids: tuple[str, ...] = ()
+ # Builds for this percentage of the organizations with a project that has an enabled signal source.
+ # Only those projects get builds. A higher percentage keeps the organizations of a lower one.
+ self_driving_organization_percentage: float = 0.0
+ # The tick never builds for these organizations, even when another field enrolls them.
+ deny_organization_ids: tuple[str, ...] = ()
+ skip_team_ids: tuple[int, ...] = ()
+ # A pair is due when its latest list was built or checked longer ago than this.
+ target_interval_minutes: int = 60
+ # Only pairs with activity in this many days are due.
+ active_days: int = 7
+ max_pairs_per_tick: int = 20
+ max_concurrency: int = 4
+ # Tick builds that call the model, in one Redis token bucket per region. They also spend `max_builds_per_day`.
+ max_tick_builds_per_day: int = 100
+ # The largest share of one tick that the pairs of one organization can take.
+ max_org_share: float = 0.25
+ # The tick stops when this share of its model calls failed or raised.
+ max_error_rate: float = 0.5
+
+
+TICK_DISABLED = PersonalRankingTickConfig()
+# The rest of the shared limit stays free for `refresh` and `focus` requests, which a person is waiting on.
+TICK_SHARE_OF_SHARED_BUILDS = 0.8
+
+_TICK_MAXIMUMS: dict[str, int] = {
+ "target_interval_minutes": 1440,
+ "active_days": 90,
+ "max_pairs_per_tick": 500,
+ "max_concurrency": 20,
+ "max_tick_builds_per_day": 10000,
+}
+
+
@frozen
class PersonalRankingConfig:
enabled: bool = True
@@ -123,6 +167,7 @@ class PersonalRankingConfig:
max_builds_per_day: int = 500
# Each build keeps this many of the person's newest rows and deletes the older ones.
max_rows_per_person: int = 5
+ tick: PersonalRankingTickConfig = TICK_DISABLED
DEFAULTS = PersonalRankingConfig()
@@ -191,6 +236,51 @@ def _reason_limits(value: Any) -> ReasonLimits:
)
+def _fraction(value: Any, default: float) -> float:
+ if isinstance(value, bool) or not isinstance(value, int | float) or not 0 < value <= 1:
+ return default
+ return float(value)
+
+
+def _percentage(value: Any) -> float:
+ if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value <= 100:
+ return TICK_DISABLED.self_driving_organization_percentage
+ return float(value)
+
+
+def _organization_ids(value: Any) -> tuple[str, ...]:
+ return tuple(item for item in value if isinstance(item, str) and item) if isinstance(value, list) else ()
+
+
+def _team_ids(value: Any) -> tuple[int, ...]:
+ if not isinstance(value, list):
+ return ()
+ return tuple(item for item in value if isinstance(item, int) and not isinstance(item, bool))
+
+
+def parse_tick_config(payload: Any) -> PersonalRankingTickConfig:
+ if not isinstance(payload, dict) or payload.get("enabled") is not True:
+ return TICK_DISABLED
+ return PersonalRankingTickConfig(
+ enabled=True,
+ organization_ids=_organization_ids(payload.get("organization_ids")),
+ self_driving_organization_percentage=_percentage(payload.get("self_driving_organization_percentage")),
+ deny_organization_ids=_organization_ids(payload.get("deny_organization_ids")),
+ skip_team_ids=_team_ids(payload.get("skip_team_ids")),
+ max_org_share=_fraction(payload.get("max_org_share"), TICK_DISABLED.max_org_share),
+ max_error_rate=_fraction(payload.get("max_error_rate"), TICK_DISABLED.max_error_rate),
+ **{
+ name: _bounded_int(payload.get(name), getattr(TICK_DISABLED, name), maximum)
+ for name, maximum in _TICK_MAXIMUMS.items()
+ },
+ )
+
+
+def _fit_tick_to_shared_limit(tick: PersonalRankingTickConfig, max_builds_per_day: int) -> PersonalRankingTickConfig:
+ largest = max(int(max_builds_per_day * TICK_SHARE_OF_SHARED_BUILDS), 1)
+ return replace(tick, max_tick_builds_per_day=min(tick.max_tick_builds_per_day, largest))
+
+
def default_pools(config: PersonalRankingConfig) -> tuple[CandidatePool, ...]:
return (
CandidatePool(name="for_you_recommended", source="for_you_recommended", size=config.for_you_recommended_pool),
@@ -297,12 +387,18 @@ def parse_config(payload: Any) -> PersonalRankingConfig:
if name not in {"timeout_seconds", "reason_max_chars", "pool_size"}
}
max_limit = ints.pop("max_limit")
+ tick = parse_tick_config(payload.get("tick"))
+ if tick.enabled:
+ tick = _fit_tick_to_shared_limit(tick, ints["max_builds_per_day"])
+ # A shorter freshness makes a plain read rebuild a list that the tick checked inside its interval.
+ ints["freshness_minutes"] = max(ints["freshness_minutes"], tick.target_interval_minutes)
return PersonalRankingConfig(
enabled=payload.get("enabled") is not False,
model=_model(payload.get("model")),
reasoning_effort=_reasoning_effort(payload.get("reasoning_effort")),
reason_limits=_reason_limits(payload.get("reason_max_chars")),
pools=_pools(payload.get("pools")),
+ tick=tick,
timeout_seconds=_bounded_number(
payload.get("timeout_seconds"), DEFAULTS.timeout_seconds, _MAXIMUMS["timeout_seconds"]
),
diff --git a/products/signals/backend/personal_ranking/logic.py b/products/signals/backend/personal_ranking/logic.py
index e78083241c34..9ea94dcc8fc7 100644
--- a/products/signals/backend/personal_ranking/logic.py
+++ b/products/signals/backend/personal_ranking/logic.py
@@ -66,8 +66,14 @@ FALLBACK_TRUNCATED = "truncated_output"
FALLBACK_NO_CANDIDATES = "no_candidates"
FALLBACK_AI_NOT_APPROVED = "ai_not_approved"
FALLBACK_NO_ACTIVITY_SUMMARY = "no_activity_summary"
+# A later build with the same input hash can rank the list. AI approval is not part of the hash.
+RETRYABLE_FALLBACK_REASONS = frozenset(
+ {FALLBACK_LLM_ERROR, FALLBACK_TIMEOUT, FALLBACK_MALFORMED, FALLBACK_TRUNCATED, FALLBACK_AI_NOT_APPROVED}
+)
BuildStatus = Literal["ok", "fallback", "empty", "disabled"]
+# What started a build, on the build event and the trace, so cost can be split by it.
+BuildTrigger = Literal["on_demand", "tick"]
_MODEL_ORDER_REASON = "Kept in inbox order."
_NEEDS_YOU_REASONS = {
@@ -181,12 +187,19 @@ def input_hash(candidates: list[Candidate], *, activity_summary_version: int | N
return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()
-def _client(team: Team, user: User, now: datetime.datetime, config: PersonalRankingConfig) -> OpenAI:
+def _client(
+ team: Team, user: User, now: datetime.datetime, config: PersonalRankingConfig, trigger: BuildTrigger
+) -> OpenAI:
client = build_openai_client(
"signals",
ai_product=AI_PRODUCT,
trace_id=trace_id_from_seed(f"{AI_PRODUCT}:{team.id}:{user.id}:{now.isoformat()}"),
- properties={"team_id": str(team.id), "user_id": str(user.id), "prompt_version": str(PROMPT_VERSION)},
+ properties={
+ "team_id": str(team.id),
+ "user_id": str(user.id),
+ "prompt_version": str(PROMPT_VERSION),
+ "trigger": trigger,
+ },
distinct_id=PIPELINE_DISTINCT_ID,
)
# One attempt: a retry would run past the timeout, and the fallback list is always there.
@@ -229,6 +242,7 @@ def _call_model(
now: datetime.datetime,
config: PersonalRankingConfig,
messages: list[ChatCompletionMessageParam],
+ trigger: BuildTrigger,
) -> _ModelCall:
started = time.monotonic()
@@ -236,7 +250,7 @@ def _call_model(
return round((time.monotonic() - started) * 1000)
try:
- response = _create_completion(_client(team, user, now, config), config, messages)
+ response = _create_completion(_client(team, user, now, config, trigger), config, messages)
except APITimeoutError:
return _ModelCall(
content=None, tokens_in=None, tokens_out=None, latency_ms=elapsed_ms(), fallback_reason=FALLBACK_TIMEOUT
@@ -259,7 +273,13 @@ def _call_model(
def _capture(
- team: Team, user: User, build: PersonalRankingBuild, config: PersonalRankingConfig, *, has_focus: bool
+ team: Team,
+ user: User,
+ build: PersonalRankingBuild,
+ config: PersonalRankingConfig,
+ *,
+ has_focus: bool,
+ trigger: BuildTrigger,
) -> None:
# Ids, counts and accounting only. Report text, reasons and the summary are user content.
try:
@@ -284,6 +304,7 @@ def _capture(
"prompt_version": PROMPT_VERSION,
"activity_summary_version": build.activity_summary_version,
"has_focus": has_focus,
+ "trigger": trigger,
"limit": build.limit,
"rank_top": config.rank_top,
"reasoning_effort": config.reasoning_effort,
@@ -340,6 +361,7 @@ def build_personal_ranking(
dry_run: bool = False,
limit: int | None = None,
config: PersonalRankingConfig | None = None,
+ trigger: BuildTrigger = "on_demand",
) -> PersonalRankingBuild:
"""Rank the person's candidate reports. `dry_run` calls the model but stores no row and records no event."""
config = config or read_config()
@@ -393,7 +415,7 @@ def build_personal_ranking(
),
},
]
- call = _call_model(team, user, now, config, messages)
+ call = _call_model(team, user, now, config, messages, trigger)
fallback_reason = call.fallback_reason
if fallback_reason is None:
try:
@@ -441,5 +463,5 @@ def build_personal_ranking(
_prune(team, user, config.max_rows_per_person)
except Exception:
logger.exception("signals.personal_ranking.prune_failed", team_id=team.id, user_id=user.id)
- _capture(team, user, build, config, has_focus=focus is not None)
+ _capture(team, user, build, config, has_focus=focus is not None, trigger=trigger)
return build
diff --git a/products/signals/backend/personal_ranking/schedule.py b/products/signals/backend/personal_ranking/schedule.py
new file mode 100644
index 000000000000..59e898d78193
--- /dev/null
+++ b/products/signals/backend/personal_ranking/schedule.py
@@ -0,0 +1,57 @@
+"""Schedule registration for the personal inbox ranking tick."""
+
+from dataclasses import asdict
+from datetime import timedelta
+
+from django.conf import settings
+
+from temporalio.client import (
+ Client,
+ Schedule,
+ ScheduleActionStartWorkflow,
+ ScheduleIntervalSpec,
+ ScheduleOverlapPolicy,
+ SchedulePolicy,
+ ScheduleSpec,
+)
+
+from posthog.scheduling.jitter import deterministic_offset
+from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule
+
+from products.signals.backend.personal_ranking.tick import TICK_WORKFLOW_NAME, PersonalRankingTickInput
+
+PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID = "personal-inbox-ranking-tick-schedule"
+
+
+async def create_personal_inbox_ranking_tick_schedule(client: Client) -> None:
+ """Create or update the tick schedule on the self-driving task queue.
+
+ The schedule exists in every environment. The `tick` object of the `personal-inbox-ranking` flag
+ payload gates the work, so turning the tick on or off needs no schedule change. SKIP on overlap: a slow
+ tick leaves the rest of the due pairs to the next one.
+ """
+ interval = timedelta(minutes=settings.PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES)
+ schedule = Schedule(
+ action=ScheduleActionStartWorkflow(
+ TICK_WORKFLOW_NAME,
+ asdict(PersonalRankingTickInput()),
+ id=PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID,
+ task_queue=settings.SELF_DRIVING_TASK_QUEUE,
+ execution_timeout=interval,
+ ),
+ spec=ScheduleSpec(
+ intervals=[
+ ScheduleIntervalSpec(
+ every=interval, offset=deterministic_offset(PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID, interval)
+ )
+ ]
+ ),
+ policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),
+ )
+ if await a_schedule_exists(client, PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID):
+ # Keep the live state, so a deploy does not resume a schedule that an operator paused.
+ description = await client.get_schedule_handle(PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID).describe()
+ schedule.state = description.schedule.state
+ await a_update_schedule(client, PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID, schedule)
+ else:
+ await a_create_schedule(client, PERSONAL_INBOX_RANKING_TICK_SCHEDULE_ID, schedule, trigger_immediately=False)
diff --git a/products/signals/backend/personal_ranking/serve.py b/products/signals/backend/personal_ranking/serve.py
index df34cab98852..cb608fa5b7ed 100644
--- a/products/signals/backend/personal_ranking/serve.py
+++ b/products/signals/backend/personal_ranking/serve.py
@@ -1,13 +1,13 @@
"""Serve one person's ranked inbox list to the `for_me` endpoint. A request never waits on the model.
-A stored list is reused while it is younger than `freshness_minutes` and its input hash matches the current
-inputs. Otherwise, and for every `refresh` or `focus` request, the read queues one background build per person
-and serves the latest stored list as stale, or the candidates in inbox order when there is none. The focus is
-part of the hash, so an unfocused request never reuses a focused list. A build that stores a list for the current
-inputs while the read runs answers the read, also a `refresh` read, so the read queues no second build. A lock lets one build per person run at a
-time, and each queued build spends two per-person token buckets. A build that may call the model also spends
-the shared daily limit of the region (`build_budget.py`). `wait_seconds` waits a bounded time for the queued build
-to store its list.
+A stored list is reused while it was built or checked by the tick less than `freshness_minutes` ago and its input
+hash matches the current inputs. Otherwise, and for every `refresh` or `focus` request, the read queues one
+background build per person and serves the latest stored list as stale, or the candidates in inbox order when
+there is none. The focus is part of the hash, so an unfocused request never reuses a focused list. A build that
+stores a list for the current inputs while the read runs answers the read, also a `refresh` read, so the read
+queues no second build. A lock lets one build per person run at a time, and each queued build spends two
+per-person token buckets. A build that may call the model also spends the shared daily limit of the region
+(`build_budget.py`). `wait_seconds` waits a bounded time for the queued build to store its list.
"""
import time
@@ -198,14 +198,14 @@ def _start_build(
return "failed"
-def _latest_row(team: Team, user: User) -> PersonalInboxRanking | None:
+def latest_row(team: Team, user: User) -> PersonalInboxRanking | None:
return PersonalInboxRanking.objects.for_team(team.id).filter(user_id=user.id).order_by("-created_at", "-id").first()
def _row_stored_since(
team: Team, user: User, seen: PersonalInboxRanking | None, current_hash: str
) -> PersonalInboxRanking | None:
- row = _latest_row(team, user)
+ row = latest_row(team, user)
if row is None or (seen is not None and row.id == seen.id) or row.input_hash != current_hash:
return None
return row
@@ -216,7 +216,7 @@ def _wait_for_new_row(
) -> PersonalInboxRanking | None:
deadline = time.monotonic() + wait_seconds
while True:
- row = _latest_row(team, user)
+ row = latest_row(team, user)
if row is not None and (previous is None or row.id != previous.id):
return row
if time.monotonic() >= deadline:
@@ -224,7 +224,7 @@ def _wait_for_new_row(
time.sleep(_WAIT_POLL_SECONDS)
-def _ready_summary_version(team: Team, user: User) -> int | None:
+def ready_summary_version(team: Team, user: User) -> int | None:
summary = get_summary(team, user.id)
return summary.version if summary is not None and summary.status == ActivitySummaryStatus.READY else None
@@ -353,15 +353,15 @@ def serve_personal_ranking(
wait_seconds = min(max(wait_seconds, 0), MAX_WAIT_SECONDS)
now = timezone.now()
- started_with = _latest_row(team, user)
+ started_with = latest_row(team, user)
candidates = gather_candidates(team, user, config, now=now, with_sources=False)
- summary_version = _ready_summary_version(team, user)
+ summary_version = ready_summary_version(team, user)
current_hash = input_hash(candidates, activity_summary_version=summary_version, focus=focus)
# Read again, because a build can store a row while the candidates are gathered.
- latest = _latest_row(team, user)
+ latest = latest_row(team, user)
current = (
latest is not None
- and latest.created_at >= now - datetime.timedelta(minutes=config.freshness_minutes)
+ and (latest.checked_at or latest.created_at) >= now - datetime.timedelta(minutes=config.freshness_minutes)
and latest.input_hash == current_hash
)
# A row stored after this read started also answers a refresh.
diff --git a/products/signals/backend/personal_ranking/tick.py b/products/signals/backend/personal_ranking/tick.py
new file mode 100644
index 000000000000..e5872660a8b9
--- /dev/null
+++ b/products/signals/backend/personal_ranking/tick.py
@@ -0,0 +1,522 @@
+"""The scheduled personal ranking tick: a Temporal workflow that builds the lists of due (team, user) pairs.
+
+The `tick` object of the `personal-inbox-ranking` flag payload holds every lever, and a missing or malformed
+object turns the tick off. A pair is due when it has a ready activity summary, recent activity, and a latest
+list that was built or checked longer ago than `target_interval_minutes`, or no list. The pair activity takes
+the same per-person lock as the `for_me` read, so a tick and a read never build the same list together. It
+compares the input hash first: a match records `checked_at` on the latest row and calls no model. A build that
+calls the model spends the tick's daily limit and then the shared daily limit of the region.
+"""
+
+import uuid
+import asyncio
+import datetime
+import dataclasses
+from typing import Any, Literal
+
+from django.conf import settings
+from django.core.cache import cache
+from django.db.models import Count, DateTimeField, Exists, F, Min, OuterRef, Q, QuerySet, Subquery, Value, Window
+from django.db.models.functions import Coalesce, RowNumber
+from django.utils import timezone
+
+import structlog
+from temporalio import activity, workflow
+from temporalio.common import RetryPolicy
+
+from posthog.dataclasses import frozen
+from posthog.llm.gateway_client import openai_gateway_configured
+from posthog.models import OrganizationMembership, Team, User
+from posthog.ph_client import ph_background_capture
+from posthog.sync import database_sync_to_async
+from posthog.temporal.common.heartbeat import Heartbeater
+from posthog.temporal.common.scoped import scoped_temporal
+from posthog.temporal.common.utils import close_db_connections
+
+from products.signals.backend.activity_summary.config import read_config as read_activity_summary_config
+from products.signals.backend.activity_summary.queue import read_enrolled_teams
+from products.signals.backend.facade.api import has_enabled_source, team_ids_with_enabled_source
+from products.signals.backend.models import PersonalInboxRanking, UserActivitySummary, UserActivitySummaryQueueState
+from products.signals.backend.org_rollout import in_rollout, parse_organization_ids
+from products.signals.backend.personal_ranking.build_budget import (
+ refund_tick_build,
+ spend_shared_build,
+ spend_tick_build,
+)
+from products.signals.backend.personal_ranking.candidates import gather_candidates
+from products.signals.backend.personal_ranking.config import (
+ PersonalRankingConfig,
+ PersonalRankingTickConfig,
+ read_config,
+)
+from products.signals.backend.personal_ranking.logic import (
+ PIPELINE_DISTINCT_ID,
+ RETRYABLE_FALLBACK_REASONS,
+ build_personal_ranking,
+ input_hash,
+ may_call_model,
+)
+from products.signals.backend.personal_ranking.serve import latest_row, ready_summary_version
+from products.signals.backend.personal_ranking_tasks import BUILD_LOCK_TTL_SECONDS, build_lock_key
+
+logger = structlog.get_logger(__name__)
+
+TICK_WORKFLOW_NAME = "personal-inbox-ranking-tick"
+TICK_EVENT = "personal inbox ranking tick"
+
+# Stop reasons of a tick. Null when the tick handled every due pair.
+STOP_MAX_PAIRS_PER_TICK = "max_pairs_per_tick"
+# Due pairs are left, but each organization with due pairs already has its share of the tick.
+STOP_MAX_ORG_SHARE = "max_org_share"
+STOP_MAX_TICK_BUILDS_PER_DAY = "max_tick_builds_per_day"
+STOP_MAX_BUILDS_PER_DAY = "max_builds_per_day"
+STOP_ERROR_RATE = "error_rate"
+STOP_GATEWAY_NOT_CONFIGURED = "gateway_not_configured"
+# The tick stopped before the next tick starts, so the schedule does not skip it.
+STOP_TIME_BUDGET = "time_budget"
+
+PairStatus = Literal[
+ "built", "failed", "unchanged", "running", "not_enrolled", "tick_limited", "shared_limited", "disabled"
+]
+PAIR_STATUSES: tuple[PairStatus, ...] = (
+ "built",
+ "failed",
+ "unchanged",
+ "running",
+ "not_enrolled",
+ "tick_limited",
+ "shared_limited",
+ "disabled",
+)
+_LIMIT_STOPS: dict[PairStatus, str] = {
+ "tick_limited": STOP_MAX_TICK_BUILDS_PER_DAY,
+ "shared_limited": STOP_MAX_BUILDS_PER_DAY,
+}
+
+# A new salt reorders every organization, so change the version only to start a new rollout order.
+_ROLLOUT_SALT = "personal-inbox-ranking-tick-v1"
+# The circuit breaker waits for this many model calls, so a few early failures do not stop the tick.
+_MIN_ERROR_RATE_SAMPLE = 10
+
+_PLAN_TIMEOUT = datetime.timedelta(minutes=5)
+# One build has a model call of at most 120 seconds, and the candidate reads around it.
+_PAIR_TIMEOUT = datetime.timedelta(minutes=4)
+_PAIR_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)
+# No retry: an attempt can have called the model, and the next tick picks the pair again.
+_PAIR_RETRY_POLICY = RetryPolicy(maximum_attempts=1)
+# The tick starts no wave after this share of its interval, so the last wave ends before the next tick.
+_TIME_BUDGET_SHARE = 0.6
+
+
+@frozen
+class PersonalRankingTickInput:
+ # A manual run can override the per-tick cap of the payload.
+ limit: int | None = None
+
+
+@frozen
+class TickPair:
+ team_id: int
+ user_id: int
+
+
+@frozen
+class TickEnrollment:
+ team_ids: list[int]
+ organization_ids: list[uuid.UUID]
+ # The organizations that a 100 percent rollout enrolls, with or without manual enrollment.
+ qualifying_organization_ids: list[uuid.UUID]
+
+
+@frozen
+class TickSelection:
+ pairs: list[TickPair]
+ due_pairs: int = 0
+ due_without_list: int = 0
+ # Hours since the oldest list of a due pair was built or checked. Null when no due pair has a list.
+ oldest_due_hours: float | None = None
+ # The share of eligible pairs whose list was built or checked inside `target_interval_minutes`.
+ fresh_share: float | None = None
+
+
+@frozen
+class PersonalRankingTickPlan:
+ enabled: bool
+ pairs: list[TickPair]
+ max_concurrency: int
+ max_error_rate: float
+ time_budget_seconds: float
+ stop_reason: str | None
+ due_pairs: int = 0
+ due_without_list: int = 0
+ oldest_due_hours: float | None = None
+ fresh_share: float | None = None
+ qualifying_organizations: int = 0
+ enrolled_organizations: int = 0
+ enrolled_teams: int = 0
+
+
+@frozen
+class PersonalRankingPairResult:
+ status: PairStatus
+ model_called: bool = False
+
+
+@frozen
+class PersonalRankingTickResult:
+ enabled: bool = False
+ due_pairs: int = 0
+ due_without_list: int = 0
+ picked: int = 0
+ statuses: dict[str, int] = dataclasses.field(default_factory=dict)
+ errored: int = 0
+ stop_reason: str | None = None
+ oldest_due_hours: float | None = None
+ fresh_share: float | None = None
+ qualifying_organizations: int = 0
+ enrolled_organizations: int = 0
+ enrolled_teams: int = 0
+ duration_seconds: float = 0.0
+
+
+def _in_rollout(percentage: float, organization_id: uuid.UUID) -> bool:
+ return in_rollout(_ROLLOUT_SALT, percentage, organization_id)
+
+
+def _eligible_teams(tick: PersonalRankingTickConfig) -> QuerySet[Team]:
+ """Teams with AI data processing approval and a readable activity summary, minus the skip and deny lists."""
+ teams = read_enrolled_teams(read_activity_summary_config(), Team.objects.all())
+ return teams.exclude(id__in=tick.skip_team_ids).exclude(
+ organization_id__in=parse_organization_ids(tick.deny_organization_ids)
+ )
+
+
+def enrollment(config: PersonalRankingConfig) -> TickEnrollment:
+ """The teams that `is_team_enrolled` enrolls, in a fixed number of queries."""
+ tick = config.tick
+ if not config.enabled or not tick.enabled:
+ return TickEnrollment(team_ids=[], organization_ids=[], qualifying_organization_ids=[])
+ teams = _eligible_teams(tick)
+ listed = parse_organization_ids(tick.organization_ids)
+ with_source: dict[int, uuid.UUID] = dict(
+ teams.filter(id__in=team_ids_with_enabled_source()).values_list("id", "organization_id")
+ )
+ enrolled = dict(teams.filter(organization_id__in=listed).values_list("id", "organization_id")) if listed else {}
+ enrolled |= {
+ team_id: organization_id
+ for team_id, organization_id in with_source.items()
+ if _in_rollout(tick.self_driving_organization_percentage, organization_id)
+ }
+ return TickEnrollment(
+ team_ids=sorted(enrolled),
+ organization_ids=sorted(set(enrolled.values())),
+ qualifying_organization_ids=sorted(set(with_source.values())),
+ )
+
+
+def is_team_enrolled(config: PersonalRankingConfig, team: Team) -> bool:
+ """The tick may build lists for the team. The single-team form of `enrollment`.
+
+ Enrollment decides who gets scheduled builds. It does not change who can read `for_me`.
+ """
+ tick = config.tick
+ if not config.enabled or not tick.enabled:
+ return False
+ if not _eligible_teams(tick).filter(id=team.id).exists():
+ return False
+ if team.organization_id in parse_organization_ids(tick.organization_ids):
+ return True
+ return _in_rollout(tick.self_driving_organization_percentage, team.organization_id) and has_enabled_source(team.id)
+
+
+def _list_at() -> Subquery:
+ """When the pair's latest list was built, or checked unchanged by a later tick."""
+ return Subquery(
+ PersonalInboxRanking.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id"))
+ .order_by("-created_at", "-id")
+ .annotate(at=Coalesce("checked_at", "created_at"))
+ .values("at")[:1],
+ output_field=DateTimeField(),
+ )
+
+
+def _eligible_pairs(tick: PersonalRankingTickConfig, now: datetime.datetime, team_ids: list[int]) -> QuerySet:
+ """The pairs of these teams with a ready summary, an active member and activity in the last `active_days`."""
+ last_activity_at = Subquery(
+ UserActivitySummaryQueueState.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id")).values(
+ "last_activity_at"
+ )[:1],
+ output_field=DateTimeField(),
+ )
+ is_member = OrganizationMembership.objects.filter(
+ user_id=OuterRef("user_id"), organization_id=OuterRef("team__organization_id")
+ )
+ return (
+ # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)
+ UserActivitySummary.all_teams.filter(
+ team_id__in=team_ids, summary__isnull=False, generated_at__isnull=False, user__is_active=True
+ )
+ .filter(Exists(is_member))
+ .annotate(
+ organization_id=F("team__organization_id"),
+ paying=Coalesce(F("team__organization__has_active_subscription"), Value(False)),
+ last_activity_at=last_activity_at,
+ list_at=_list_at(),
+ )
+ .filter(last_activity_at__gte=now - datetime.timedelta(days=tick.active_days))
+ )
+
+
+def select_pairs(
+ config: PersonalRankingConfig, now: datetime.datetime, limit: int, team_ids: list[int]
+) -> TickSelection:
+ """The due pairs of the enrolled teams, cut to `limit`.
+
+ Paying organizations come first, then the most recent activity, then the oldest list. No organization
+ takes more than `max_org_share` of `limit`.
+ """
+ if not team_ids:
+ return TickSelection(pairs=[])
+ tick = config.tick
+ stale_before = now - datetime.timedelta(minutes=tick.target_interval_minutes)
+ eligible = _eligible_pairs(tick, now, team_ids)
+ due = eligible.filter(Q(list_at__isnull=True) | Q(list_at__lt=stale_before))
+ order = (
+ F("paying").desc(),
+ F("last_activity_at").desc(nulls_last=True),
+ F("list_at").asc(nulls_first=True),
+ "team_id",
+ "user_id",
+ )
+ rows = list(
+ due.annotate(org_rank=Window(RowNumber(), partition_by=[F("organization_id")], order_by=order))
+ .filter(org_rank__lte=max(1, int(tick.max_org_share * limit)))
+ .order_by(*order)
+ .values("team_id", "user_id")[:limit]
+ )
+ due_stats = due.aggregate(
+ pairs=Count("id"), without_list=Count("id", filter=Q(list_at__isnull=True)), oldest=Min("list_at")
+ )
+ eligible_stats = eligible.aggregate(pairs=Count("id"), fresh=Count("id", filter=Q(list_at__gte=stale_before)))
+ oldest: datetime.datetime | None = due_stats["oldest"]
+ return TickSelection(
+ pairs=[TickPair(team_id=row["team_id"], user_id=row["user_id"]) for row in rows],
+ due_pairs=due_stats["pairs"],
+ due_without_list=due_stats["without_list"],
+ oldest_due_hours=(now - oldest).total_seconds() / 3600 if oldest is not None else None,
+ fresh_share=eligible_stats["fresh"] / eligible_stats["pairs"] if eligible_stats["pairs"] else None,
+ )
+
+
+def plan_tick(limit: int | None, now: datetime.datetime) -> PersonalRankingTickPlan:
+ config = read_config()
+ tick = config.tick
+ enabled = config.enabled and tick.enabled
+ interval_seconds = settings.PERSONAL_INBOX_RANKING_TICK_INTERVAL_MINUTES * 60
+ time_budget_seconds = interval_seconds * _TIME_BUDGET_SHARE
+ if enabled and not openai_gateway_configured():
+ # Every build would fail on the same missing gateway, so the tick runs none.
+ logger.warning("personal_inbox_ranking_tick_gateway_not_configured")
+ return PersonalRankingTickPlan(
+ enabled=False,
+ pairs=[],
+ max_concurrency=tick.max_concurrency,
+ max_error_rate=tick.max_error_rate,
+ time_budget_seconds=time_budget_seconds,
+ stop_reason=STOP_GATEWAY_NOT_CONFIGURED,
+ )
+ enrolled = enrollment(config)
+ tick_limit = limit or tick.max_pairs_per_tick
+ selection = select_pairs(config, now, tick_limit, enrolled.team_ids) if enabled else TickSelection(pairs=[])
+ stop_reason: str | None = None
+ if selection.due_pairs > len(selection.pairs):
+ stop_reason = STOP_MAX_PAIRS_PER_TICK if len(selection.pairs) >= tick_limit else STOP_MAX_ORG_SHARE
+ return PersonalRankingTickPlan(
+ enabled=enabled,
+ pairs=selection.pairs,
+ max_concurrency=tick.max_concurrency,
+ max_error_rate=tick.max_error_rate,
+ time_budget_seconds=time_budget_seconds,
+ stop_reason=stop_reason,
+ due_pairs=selection.due_pairs,
+ due_without_list=selection.due_without_list,
+ oldest_due_hours=selection.oldest_due_hours,
+ fresh_share=selection.fresh_share,
+ qualifying_organizations=len(enrolled.qualifying_organization_ids),
+ enrolled_organizations=len(enrolled.organization_ids),
+ enrolled_teams=len(enrolled.team_ids),
+ )
+
+
+def _check_and_build(team: Team, user: User, config: PersonalRankingConfig) -> PersonalRankingPairResult:
+ now = timezone.now()
+ latest = latest_row(team, user)
+ candidates = gather_candidates(team, user, config, now=now, with_sources=False)
+ summary_version = ready_summary_version(team, user)
+ current_hash = input_hash(candidates, activity_summary_version=summary_version, focus=None)
+ if (
+ latest is not None
+ and latest.input_hash == current_hash
+ and latest.fallback_reason not in RETRYABLE_FALLBACK_REASONS
+ ):
+ # The list is still current, so the pair leaves the due set until `target_interval_minutes` passes again.
+ PersonalInboxRanking.objects.for_team(team.id).filter(id=latest.id).update(checked_at=now)
+ return PersonalRankingPairResult(status="unchanged")
+ calls_model = bool(candidates) and may_call_model(team, summary_ready=summary_version is not None)
+ if calls_model:
+ if not spend_tick_build(config):
+ return PersonalRankingPairResult(status="tick_limited")
+ if spend_shared_build(config) is not None:
+ refund_tick_build(config)
+ return PersonalRankingPairResult(status="shared_limited")
+ build = build_personal_ranking(team, user, config=config, trigger="tick")
+ if build.status == "disabled":
+ return PersonalRankingPairResult(status="disabled")
+ failed = build.model_called and build.status == "fallback"
+ return PersonalRankingPairResult(status="failed" if failed else "built", model_called=build.model_called)
+
+
+def run_pair(pair: TickPair) -> PersonalRankingPairResult:
+ # The payload can change between the plan and the pair, so the pair reads it again.
+ config = read_config()
+ team = Team.objects.select_related("organization").filter(id=pair.team_id).first()
+ user = User.objects.filter(id=pair.user_id, is_active=True).first()
+ if team is None or user is None or not is_team_enrolled(config, team):
+ return PersonalRankingPairResult(status="not_enrolled")
+ lock_key = build_lock_key(team.id, user.id)
+ if not cache.add(lock_key, "1", timeout=BUILD_LOCK_TTL_SECONDS):
+ return PersonalRankingPairResult(status="running")
+ try:
+ return _check_and_build(team, user, config)
+ finally:
+ cache.delete(lock_key)
+
+
+def capture_tick(result: PersonalRankingTickResult) -> None:
+ properties: dict[str, Any] = {
+ "deployment": settings.CLOUD_DEPLOYMENT,
+ "due_pairs": result.due_pairs,
+ "due_without_list": result.due_without_list,
+ "picked": result.picked,
+ **{status: result.statuses.get(status, 0) for status in PAIR_STATUSES},
+ "errored": result.errored,
+ "stop_reason": result.stop_reason,
+ "oldest_due_hours": result.oldest_due_hours,
+ "fresh_share": result.fresh_share,
+ "qualifying_organizations": result.qualifying_organizations,
+ "enrolled_organizations": result.enrolled_organizations,
+ "enrolled_teams": result.enrolled_teams,
+ "duration_seconds": result.duration_seconds,
+ }
+ logger.info("personal_inbox_ranking_tick_finished", **properties)
+ ph_background_capture()(distinct_id=PIPELINE_DISTINCT_ID, event=TICK_EVENT, properties=properties)
+
+
+@scoped_temporal()
+@close_db_connections
+async def plan_personal_ranking_tick_activity(input: PersonalRankingTickInput) -> PersonalRankingTickPlan:
+ return await database_sync_to_async(plan_tick, thread_sensitive=False)(input.limit, timezone.now())
+
+
+@scoped_temporal()
+@close_db_connections
+async def build_personal_ranking_pair_activity(pair: TickPair) -> PersonalRankingPairResult:
+ async with Heartbeater():
+ return await database_sync_to_async(run_pair, thread_sensitive=False)(pair)
+
+
+@scoped_temporal()
+@close_db_connections
+async def capture_personal_ranking_tick_activity(result: PersonalRankingTickResult) -> None:
+ await database_sync_to_async(capture_tick, thread_sensitive=False)(result)
+
+
[email protected](name=TICK_WORKFLOW_NAME)
+class PersonalRankingTickWorkflow:
+ @workflow.run
+ async def run(self, inputs: PersonalRankingTickInput) -> PersonalRankingTickResult:
+ started = workflow.now()
+ plan = await workflow.execute_activity(
+ plan_personal_ranking_tick_activity,
+ inputs,
+ start_to_close_timeout=_PLAN_TIMEOUT,
+ retry_policy=RetryPolicy(maximum_attempts=2),
+ )
+ if not plan.enabled:
+ return PersonalRankingTickResult(stop_reason=plan.stop_reason)
+
+ deadline = started + datetime.timedelta(seconds=plan.time_budget_seconds)
+ statuses: dict[str, int] = {}
+ errored = 0
+ # Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.
+ calls = 0
+ failures = 0
+ picked = 0
+ stop_reason = plan.stop_reason
+ concurrency = max(1, plan.max_concurrency)
+ index = 0
+ while index < len(plan.pairs):
+ # The pairs not attempted yet stay due, so the next tick picks them.
+ if calls >= _MIN_ERROR_RATE_SAMPLE and failures >= plan.max_error_rate * calls:
+ stop_reason = STOP_ERROR_RATE
+ break
+ if workflow.now() >= deadline:
+ stop_reason = STOP_TIME_BUDGET
+ break
+ wave = plan.pairs[index : index + concurrency]
+ index += len(wave)
+ picked += len(wave)
+ results = await asyncio.gather(
+ *(
+ workflow.execute_activity(
+ build_personal_ranking_pair_activity,
+ pair,
+ start_to_close_timeout=_PAIR_TIMEOUT,
+ heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,
+ retry_policy=_PAIR_RETRY_POLICY,
+ )
+ for pair in wave
+ ),
+ return_exceptions=True,
+ )
+ limit_stop: str | None = None
+ for outcome in results:
+ if isinstance(outcome, BaseException):
+ # The attempt can have called the model, so it counts as a failed call.
+ errored += 1
+ calls += 1
+ failures += 1
+ continue
+ statuses[outcome.status] = statuses.get(outcome.status, 0) + 1
+ calls += outcome.model_called
+ failures += outcome.status == "failed"
+ limit_stop = limit_stop or _LIMIT_STOPS.get(outcome.status)
+ if limit_stop is not None:
+ stop_reason = limit_stop
+ break
+
+ result = PersonalRankingTickResult(
+ enabled=True,
+ due_pairs=plan.due_pairs,
+ due_without_list=plan.due_without_list,
+ picked=picked,
+ statuses=statuses,
+ errored=errored,
+ stop_reason=stop_reason,
+ oldest_due_hours=plan.oldest_due_hours,
+ fresh_share=plan.fresh_share,
+ qualifying_organizations=plan.qualifying_organizations,
+ enrolled_organizations=plan.enrolled_organizations,
+ enrolled_teams=plan.enrolled_teams,
+ duration_seconds=(workflow.now() - started).total_seconds(),
+ )
+ await workflow.execute_activity(
+ capture_personal_ranking_tick_activity,
+ result,
+ start_to_close_timeout=datetime.timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=2),
+ )
+ return result
diff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.py
index 3bdaae305790..314135814091 100644
--- a/products/signals/backend/temporal/__init__.py
+++ b/products/signals/backend/temporal/__init__.py
@@ -14,6 +14,12 @@ from products.signals.backend.anomaly_scores.tick import (
plan_anomaly_scoring_tick_activity,
score_anomaly_insight_activity,
)
+from products.signals.backend.personal_ranking.tick import (
+ PersonalRankingTickWorkflow,
+ build_personal_ranking_pair_activity,
+ capture_personal_ranking_tick_activity,
+ plan_personal_ranking_tick_activity,
+)
from products.signals.backend.ranking.sweep import InboxRankingScoringWorkflow, score_inbox_reports_activity
from products.signals.backend.temporal.agentic.report import run_agentic_report_activity
from products.signals.backend.temporal.agentic.scout_coordinator import (
@@ -251,6 +257,7 @@ ACTIVITIES = [
SELF_DRIVING_WORKFLOWS = [
InboxRankingScoringWorkflow,
ActivitySummaryTickWorkflow,
+ PersonalRankingTickWorkflow,
AnomalyScoringTickWorkflow,
AnomalyScoringDueListRefreshWorkflow,
]
@@ -259,6 +266,9 @@ SELF_DRIVING_ACTIVITIES = [
plan_activity_summary_tick_activity,
update_activity_summary_pair_activity,
capture_activity_summary_tick_activity,
+ plan_personal_ranking_tick_activity,
+ build_personal_ranking_pair_activity,
+ capture_personal_ranking_tick_activity,
plan_anomaly_scoring_tick_activity,
score_anomaly_insight_activity,
capture_anomaly_scoring_tick_activity,
diff --git a/products/signals/backend/test/test_personal_ranking.py b/products/signals/backend/test/test_personal_ranking.py
index c3ac114d0f08..d367991ca051 100644
--- a/products/signals/backend/test/test_personal_ranking.py
+++ b/products/signals/backend/test/test_personal_ranking.py
@@ -39,6 +39,7 @@ from products.signals.backend.personal_ranking.candidates import Candidate
from products.signals.backend.personal_ranking.config import (
CandidatePool,
PersonalRankingConfig,
+ PersonalRankingTickConfig,
ReasonLimits,
parse_config,
)
@@ -364,6 +365,45 @@ class TestConfig(SimpleTestCase):
),
("reasoning_effort", {"reasoning_effort": "high"}, PersonalRankingConfig(reasoning_effort="high")),
("invalid_reasoning_effort", {"reasoning_effort": "max"}, PersonalRankingConfig()),
+ ("tick_without_enabled_stays_off", {"tick": {"organization_ids": ["org"]}}, PersonalRankingConfig()),
+ (
+ "tick_drops_invalid_values_and_caps_large_ones",
+ {
+ "tick": {
+ "enabled": True,
+ "organization_ids": ["org", 3, ""],
+ "skip_team_ids": [1, True, "2"],
+ "self_driving_organization_percentage": 101,
+ "max_pairs_per_tick": 10_000,
+ "max_tick_builds_per_day": -1,
+ "max_org_share": 2,
+ }
+ },
+ PersonalRankingConfig(
+ tick=PersonalRankingTickConfig(
+ enabled=True, organization_ids=("org",), skip_team_ids=(1,), max_pairs_per_tick=500
+ )
+ ),
+ ),
+ (
+ "enabled_tick_raises_freshness_and_leaves_a_shared_reserve",
+ {
+ "max_builds_per_day": 100,
+ "tick": {"enabled": True, "target_interval_minutes": 180, "max_tick_builds_per_day": 100},
+ },
+ PersonalRankingConfig(
+ max_builds_per_day=100,
+ freshness_minutes=180,
+ tick=PersonalRankingTickConfig(
+ enabled=True, target_interval_minutes=180, max_tick_builds_per_day=80
+ ),
+ ),
+ ),
+ (
+ "disabled_tick_leaves_freshness_alone",
+ {"tick": {"target_interval_minutes": 180}},
+ PersonalRankingConfig(),
+ ),
]
)
def test_parse_config(self, _name: str, payload: Any, expected: PersonalRankingConfig) -> None:
@@ -889,6 +929,23 @@ class TestForMeEndpoint(APIBaseTest):
assert [args[1] for args in self.queued] == [self.user.id, third.id]
+ def test_a_list_the_tick_checked_stays_fresh_past_its_build_time(self) -> None:
+ self._claimed_report("First")
+ with time_machine.travel(timezone.now(), tick=False) as traveller:
+ self._get()
+ self._run_builds()
+ traveller.shift(datetime.timedelta(hours=2))
+ assert self._get()["stale"] is True
+ self._run_builds()
+
+ traveller.shift(datetime.timedelta(hours=2))
+ PersonalInboxRanking.objects.for_team(self.team.id).filter(user_id=self.user.id).update(
+ checked_at=timezone.now()
+ )
+ body = self._get()
+
+ assert (self._state(body), self.queued) == ((True, False, False), [])
+
def test_another_user_never_reads_your_list(self) -> None:
self._get()
self._run_builds()
diff --git a/products/signals/backend/test/test_personal_ranking_tick.py b/products/signals/backend/test/test_personal_ranking_tick.py
new file mode 100644
index 000000000000..489cf4360cbd
--- /dev/null
+++ b/products/signals/backend/test/test_personal_ranking_tick.py
@@ -0,0 +1,377 @@
+import json
+import uuid
+import asyncio
+import datetime
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+import time_machine
+from posthog.test.base import BaseTest
+from unittest.mock import MagicMock, patch
+
+from django.core.cache import cache
+from django.utils import timezone
+
+from parameterized import parameterized
+from temporalio import activity
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+
+from posthog.models import Organization, Team, User
+from posthog.redis import TEST_clear_clients
+from posthog.token_bucket import TEST_reset_scripts
+
+from products.signals.backend.activity_summary.config import ActivitySummaryConfig
+from products.signals.backend.activity_summary.read import ActivitySummaryStatus, ActivitySummaryView
+from products.signals.backend.models import (
+ PersonalInboxRanking,
+ SignalReport,
+ SignalReportAssignment,
+ UserActivitySummary,
+ UserActivitySummaryQueueState,
+)
+from products.signals.backend.personal_ranking.build_budget import spend_shared_build, spend_tick_build
+from products.signals.backend.personal_ranking.config import parse_config
+from products.signals.backend.personal_ranking.tick import (
+ STOP_ERROR_RATE,
+ STOP_MAX_BUILDS_PER_DAY,
+ STOP_MAX_TICK_BUILDS_PER_DAY,
+ STOP_TIME_BUDGET,
+ PersonalRankingPairResult,
+ PersonalRankingTickInput,
+ PersonalRankingTickPlan,
+ PersonalRankingTickResult,
+ PersonalRankingTickWorkflow,
+ TickPair,
+ enrollment,
+ is_team_enrolled,
+ run_pair,
+ select_pairs,
+)
+from products.signals.backend.personal_ranking_tasks import build_lock_key
+
+NOW = datetime.datetime(2026, 10, 20, 12, 0, tzinfo=datetime.UTC)
+_TICK = "products.signals.backend.personal_ranking.tick"
+
+
+class _TickTest(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.tick: dict[str, Any] = {"enabled": True, "organization_ids": [str(self.organization.id)]}
+ self.payload: dict[str, Any] = {}
+ self.source_team_ids: set[int] = set()
+ for target, value in (
+ (
+ "products.signals.backend.personal_ranking.config.read_flag_payload",
+ MagicMock(side_effect=lambda *_: {**self.payload, "tick": self.tick}),
+ ),
+ (
+ f"{_TICK}.read_activity_summary_config",
+ MagicMock(return_value=ActivitySummaryConfig(enabled=True, enroll_all_approved=True)),
+ ),
+ (f"{_TICK}.team_ids_with_enabled_source", MagicMock(side_effect=lambda: self.source_team_ids)),
+ (f"{_TICK}.has_enabled_source", MagicMock(side_effect=lambda team_id: team_id in self.source_team_ids)),
+ ):
+ patcher = patch(target, value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
+ def _config(self):
+ return parse_config({**self.payload, "tick": self.tick})
+
+ def _pair(
+ self,
+ team: Team,
+ email: str,
+ *,
+ active_hours_ago: float,
+ list_hours_ago: float | None = None,
+ checked_hours_ago: float | None = None,
+ ) -> User:
+ now = timezone.now()
+ user = User.objects.create_and_join(team.organization, email, None)
+ UserActivitySummary.objects.for_team(team.id).create(
+ team_id=team.id, user_id=user.id, status=UserActivitySummary.Status.OK, summary={}, generated_at=now
+ )
+ UserActivitySummaryQueueState.objects.for_team(team.id).create(
+ team_id=team.id, user_id=user.id, last_activity_at=now - datetime.timedelta(hours=active_hours_ago)
+ )
+ if list_hours_ago is not None:
+ row = PersonalInboxRanking.objects.for_team(team.id).create(
+ team_id=team.id, user_id=user.id, input_hash="old", model="m", prompt_version=1, status="ok"
+ )
+ PersonalInboxRanking.objects.for_team(team.id).filter(id=row.id).update(
+ created_at=now - datetime.timedelta(hours=list_hours_ago),
+ checked_at=None if checked_hours_ago is None else now - datetime.timedelta(hours=checked_hours_ago),
+ )
+ return user
+
+
+class TestTickEnrollment(_TickTest):
+ @parameterized.expand(
+ [
+ ("nothing_at_zero_without_a_list", {"organization_ids": []}, True, True, False),
+ ("a_listed_organization", {}, False, True, True),
+ ("a_listed_organization_without_approval", {}, False, False, False),
+ ("a_denied_organization", {"deny": True}, False, True, False),
+ ("a_skipped_team", {"skip": True}, False, True, False),
+ ("every_team_with_a_source_at_100", {"organization_ids": [], "percentage": 100}, True, True, True),
+ ("no_team_without_a_source_at_100", {"organization_ids": [], "percentage": 100}, False, True, False),
+ ("the_tick_off", {"enabled": False}, True, True, False),
+ ]
+ )
+ def test_enrollment(
+ self, _name: str, overrides: dict[str, Any], has_source: bool, approved: bool, expected: bool
+ ) -> None:
+ self.organization.is_ai_data_processing_approved = approved
+ self.organization.save()
+ if has_source:
+ self.source_team_ids = {self.team.id}
+ self.tick = {
+ **self.tick,
+ "enabled": overrides.get("enabled", True),
+ "organization_ids": overrides.get("organization_ids", self.tick["organization_ids"]),
+ "self_driving_organization_percentage": overrides.get("percentage", 0),
+ "deny_organization_ids": [str(self.organization.id)] if overrides.get("deny") else [],
+ "skip_team_ids": [self.team.id] if overrides.get("skip") else [],
+ }
+ config = self._config()
+ self.team.refresh_from_db()
+
+ assert (enrollment(config).team_ids, is_team_enrolled(config, self.team)) == (
+ [self.team.id] if expected else [],
+ expected,
+ )
+
+
+class TestTickSelection(_TickTest):
+ def test_selects_due_pairs_paying_first_then_by_activity_and_oldest_list_within_the_org_share(self) -> None:
+ paying = Team.objects.create(
+ organization=Organization.objects.create(name="Paying", has_active_subscription=True), name="Paying"
+ )
+ free = Team.objects.create(organization=Organization.objects.create(name="Free"), name="Free")
+ with time_machine.travel(NOW, tick=False):
+ users = {
+ "paying": self._pair(paying, "[email protected]", active_hours_ago=30),
+ "recent_no_list": self._pair(free, "[email protected]", active_hours_ago=1),
+ "recent_old_list": self._pair(free, "[email protected]", active_hours_ago=1, list_hours_ago=3),
+ # Past this organization's share of the tick.
+ "older": self._pair(free, "[email protected]", active_hours_ago=2),
+ "oldest_activity": self._pair(self.team, "[email protected]", active_hours_ago=3, list_hours_ago=5),
+ "fresh_list": self._pair(self.team, "[email protected]", active_hours_ago=1, list_hours_ago=0.2),
+ "checked_recently": self._pair(
+ self.team, "[email protected]", active_hours_ago=1, list_hours_ago=3, checked_hours_ago=0.2
+ ),
+ "inactive": self._pair(self.team, "[email protected]", active_hours_ago=24 * 10),
+ }
+ selection = select_pairs(
+ parse_config({"tick": {"enabled": True, "max_org_share": 0.5}}),
+ NOW,
+ 4,
+ [paying.id, free.id, self.team.id],
+ )
+
+ assert [pair.user_id for pair in selection.pairs] == [
+ users[name].id for name in ("paying", "recent_no_list", "recent_old_list", "oldest_activity")
+ ]
+ assert (selection.due_pairs, selection.due_without_list, selection.oldest_due_hours) == (5, 3, 5.0)
+ assert selection.fresh_share == 2 / 7
+
+
+class TestTickPair(_TickTest):
+ def setUp(self) -> None:
+ super().setUp()
+ TEST_clear_clients()
+ TEST_reset_scripts()
+ cache.delete(build_lock_key(self.team.id, self.user.id))
+ self.malformed_answers = 0
+ self.llm = MagicMock()
+ self.create = self.llm.with_options.return_value.chat.completions.create
+ self.create.side_effect = lambda **_: SimpleNamespace(
+ choices=[SimpleNamespace(finish_reason="stop", message=SimpleNamespace(content=self._answer()))],
+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5),
+ )
+ scoped_capture = MagicMock()
+ self.capture = scoped_capture.return_value.__enter__.return_value
+ summary = ActivitySummaryView(status=ActivitySummaryStatus.READY, version=1)
+ for target, value in (
+ ("products.signals.backend.personal_ranking.logic.build_openai_client", MagicMock(return_value=self.llm)),
+ ("products.signals.backend.personal_ranking.logic.get_summary", MagicMock(return_value=summary)),
+ ("products.signals.backend.personal_ranking.serve.get_summary", MagicMock(return_value=summary)),
+ ("products.signals.backend.personal_ranking.logic.ph_scoped_capture", scoped_capture),
+ (
+ "products.signals.backend.personal_ranking.candidates.fetch_source_products_for_reports",
+ MagicMock(return_value={}),
+ ),
+ ):
+ patcher = patch(target, value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ report = SignalReport.objects.create(
+ team=self.team, title="Claimed", summary="Claimed", signal_count=1, total_weight=1.0, status="ready"
+ )
+ SignalReportAssignment.all_teams.create(team=self.team, report=report, actor_kind="user", actor_user=self.user)
+ UserActivitySummary.objects.for_team(self.team.id).create(
+ team_id=self.team.id,
+ user_id=self.user.id,
+ status=UserActivitySummary.Status.OK,
+ summary={},
+ generated_at=timezone.now(),
+ )
+ UserActivitySummaryQueueState.objects.for_team(self.team.id).create(
+ team_id=self.team.id, user_id=self.user.id, last_activity_at=timezone.now()
+ )
+
+ def _answer(self) -> str:
+ if self.malformed_answers:
+ self.malformed_answers -= 1
+ return "not json"
+ return json.dumps(
+ {
+ "ranked": [{"ref": "c1", "rank": 1, "tier": "top", "reason": "r", "matched": None, "next_step": None}],
+ "folded": [],
+ }
+ )
+
+ def _run(self) -> PersonalRankingPairResult:
+ return run_pair(TickPair(team_id=self.team.id, user_id=self.user.id))
+
+ def _rows(self):
+ return PersonalInboxRanking.objects.for_team(self.team.id).filter(user_id=self.user.id)
+
+ @parameterized.expand(
+ [
+ ("a_ranked_list_is_kept", 0, "built", "unchanged", 1),
+ ("a_failed_model_call_is_retried", 1, "failed", "built", 2),
+ ]
+ )
+ def test_an_unchanged_pair_calls_no_model_unless_its_last_build_failed(
+ self, _name: str, malformed_answers: int, first: str, second: str, model_calls: int
+ ) -> None:
+ self.malformed_answers = malformed_answers
+ assert self._run() == PersonalRankingPairResult(status=first, model_called=True) # type: ignore[arg-type]
+ assert self.capture.call_args.kwargs["properties"]["trigger"] == "tick"
+ self._rows().update(created_at=timezone.now() - datetime.timedelta(hours=2))
+ assert len(select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs) == 1
+
+ assert self._run() == PersonalRankingPairResult(
+ status=second, # type: ignore[arg-type]
+ model_called=second == "built",
+ )
+
+ assert self.create.call_count == model_calls
+ assert select_pairs(self._config(), timezone.now(), 10, [self.team.id]).pairs == []
+
+ @parameterized.expand([("running",), ("tick_limited",), ("shared_limited",), ("not_enrolled",)])
+ def test_a_pair_that_may_not_build_calls_no_model(self, status: str) -> None:
+ self.payload = {"max_builds_per_day": 1}
+ self.tick = {**self.tick, "max_tick_builds_per_day": 1}
+ if status == "running":
+ cache.add(build_lock_key(self.team.id, self.user.id), "1")
+ elif status == "tick_limited":
+ assert spend_tick_build(self._config())
+ elif status == "shared_limited":
+ assert spend_shared_build(self._config()) is None
+ else:
+ self.tick = {**self.tick, "organization_ids": []}
+
+ assert self._run() == PersonalRankingPairResult(status=status) # type: ignore[arg-type]
+
+ assert (self.create.call_count, self._rows().count()) == (0, 0)
+ if status == "shared_limited":
+ # The tick's own limit gets its build back.
+ assert spend_tick_build(self._config())
+
+
+def _plan(pairs: int, **overrides: Any) -> PersonalRankingTickPlan:
+ fields: dict[str, Any] = {
+ "enabled": True,
+ "pairs": [TickPair(team_id=1, user_id=user_id) for user_id in range(pairs)],
+ "max_concurrency": 2,
+ "max_error_rate": 0.5,
+ "time_budget_seconds": 600.0,
+ "stop_reason": None,
+ "due_pairs": pairs,
+ "enrolled_teams": 3,
+ }
+ return PersonalRankingTickPlan(**{**fields, **overrides})
+
+
+async def _run_tick(
+ plan: PersonalRankingTickPlan, pair_status: str
+) -> tuple[PersonalRankingTickResult, int, list[PersonalRankingTickResult]]:
+ running = peak = 0
+ captured: list[PersonalRankingTickResult] = []
+
+ @activity.defn(name="plan_personal_ranking_tick_activity")
+ async def fake_plan(input: PersonalRankingTickInput) -> PersonalRankingTickPlan:
+ return plan
+
+ @activity.defn(name="build_personal_ranking_pair_activity")
+ async def fake_pair(pair: TickPair) -> PersonalRankingPairResult:
+ nonlocal running, peak
+ running += 1
+ peak = max(peak, running)
+ await asyncio.sleep(0.01)
+ running -= 1
+ return PersonalRankingPairResult(
+ status=pair_status, # type: ignore[arg-type]
+ model_called=pair_status in ("built", "failed"),
+ )
+
+ @activity.defn(name="capture_personal_ranking_tick_activity")
+ async def fake_capture(result: PersonalRankingTickResult) -> None:
+ captured.append(result)
+
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue="personal-ranking-tick-test",
+ workflows=[PersonalRankingTickWorkflow],
+ activities=[fake_plan, fake_pair, fake_capture],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ result = await env.client.execute_workflow(
+ PersonalRankingTickWorkflow.run,
+ PersonalRankingTickInput(),
+ id=f"personal-ranking-tick-{uuid.uuid4()}",
+ task_queue="personal-ranking-tick-test",
+ )
+ return result, peak, captured
+
+
+ "plan,pair_status,picked,stop_reason",
+ [
+ (_plan(5), "built", 5, None),
+ (_plan(5), "unchanged", 5, None),
+ # The wave that met the limit finishes, and no later wave starts.
+ (_plan(5), "tick_limited", 2, STOP_MAX_TICK_BUILDS_PER_DAY),
+ (_plan(5), "shared_limited", 2, STOP_MAX_BUILDS_PER_DAY),
+ # The breaker waits for a sample of 10 model calls, which is five waves.
+ (_plan(20), "failed", 10, STOP_ERROR_RATE),
+ (_plan(5, time_budget_seconds=0.0), "built", 0, STOP_TIME_BUDGET),
+ ],
+)
+async def test_the_tick_bounds_concurrency_and_stops_at_each_limit(
+ plan: PersonalRankingTickPlan, pair_status: str, picked: int, stop_reason: str | None
+) -> None:
+ result, peak, captured = await _run_tick(plan, pair_status)
+
+ assert peak <= plan.max_concurrency
+ assert (result.picked, result.statuses, result.stop_reason) == (
+ picked,
+ {pair_status: picked} if picked else {},
+ stop_reason,
+ )
+ assert (result.due_pairs, result.enrolled_teams) == (plan.due_pairs, plan.enrolled_teams)
+ assert captured == [result]
+
+
+async def test_a_disabled_tick_runs_no_pair() -> None:
+ result, peak, captured = await _run_tick(_plan(0, enabled=False), "built")
+
+ assert (result, peak, captured) == (PersonalRankingTickResult(), 0, [])