Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): run one background scout lane per skill (#126758)

posthog[bot]0a91a8141994
diff --git a/products/signals/backend/migrations/0224_signalscoutbackgroundband_audience.py b/products/signals/backend/migrations/0224_signalscoutbackgroundband_audience.py
new file mode 100644
index 000000000000..43990bfce79a
--- /dev/null
+++ b/products/signals/backend/migrations/0224_signalscoutbackgroundband_audience.py
@@ -0,0 +1,22 @@
+# Generated by Django 6.1.2 on 2026-10-31 10:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0223_personalinboxranking"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="signalscoutbackgroundband",
+ name="audience",
+ field=models.CharField(
+ choices=[("not_set_up", "Not set up"), ("set_up", "Set up")],
+ db_default="not_set_up",
+ default="not_set_up",
+ max_length=20,
+ ),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 360907deb011..d205921321cb 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0223_personalinboxranking
+0224_signalscoutbackgroundband_audience
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index f46c1b41cbe4..37e0bd0c3538 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -4015,10 +4015,16 @@ class SignalScoutBackgroundBand(TeamScopedRootMixin, UUIDModel):
"""The activity band of one project that can get a background scout, one row per team.
A nightly job (`scout_harness/background_bands.py`) writes the full set and deletes every row
- for a project that is no longer eligible. The coordinator samples a percentage of each band
- from the `background.bands` block of the `signals-scout` flag payload.
+ for a project that is no longer eligible. Each background lane samples a percentage of each band
+ from the rows of its own `audience` (see `team_limits.parse_background_lanes`).
"""
+ class Audience(models.TextChoices):
+ """The background lanes that can sample the project. A project is in exactly one audience."""
+
+ NOT_SET_UP = "not_set_up", "Not set up"
+ SET_UP = "set_up", "Set up"
+
# See SignalScoutConfig.all_teams for rationale.
all_teams = models.Manager() # noqa: DJ012
@@ -4031,6 +4037,14 @@ class SignalScoutBackgroundBand(TeamScopedRootMixin, UUIDModel):
related_name="+",
)
band = models.PositiveSmallIntegerField()
+ # `db_default` alongside `default` keeps the AddField non-blocking and fills the existing rows,
+ # which the nightly job wrote for projects that did not set Signals up.
+ audience = models.CharField(
+ max_length=20,
+ choices=Audience.choices,
+ default=Audience.NOT_SET_UP,
+ db_default=Audience.NOT_SET_UP,
+ )
computed_at = models.DateTimeField()
class Meta:
diff --git a/products/signals/backend/scout_harness/AGENTS.md b/products/signals/backend/scout_harness/AGENTS.md
index 84fe51f1781f..248f020a3b9a 100644
--- a/products/signals/backend/scout_harness/AGENTS.md
+++ b/products/signals/backend/scout_harness/AGENTS.md
@@ -102,10 +102,11 @@ In production it is driven by `SignalsScoutCoordinatorWorkflow` (periodic tick e
Also the failure-streak circuit breaker's knobs: `failure_streak_pause_threshold(runs_in_tolerance_window)` — consecutive failed runs before the breaker pauses the lane, derived per-lane so the same wall-clock tolerance holds at every cadence (one failure past the runs the schedule fits into `FAILURE_STREAK_MIN_SPAN_MINUTES` of continuous failure, `FAILURE_STREAK_MIN_RUNS` floor, capped at `FAILURE_STREAK_MAX_RUNS` + 1), with `interval_runs_in_tolerance_window` sizing the evenly spaced case — and `AUTO_PAUSE_PROBE_INTERVAL_S` (how long a paused lane holds before the coordinator dispatches one probe).
- `team_limits.py`
Single source of truth for a team's effective scout caps + metadata, resolved from the `signals-scout` flag payload in one read. The same three-layer cap resolution (`team_configs[team]` → `default_team_config` → code constant) the coordinator enforces at dispatch — for the run budgets and for `max_enabled_scouts`, the ceiling on how many scouts a project may have switched on, which `resolve_max_enabled_scouts` / `max_enabled_scouts_for_team` serve to the config write surfaces, auto-registration and the system resume alike, plus enrollment (`_parse_enrollment` → `guaranteed_team_ids` / `skip_team_ids`, with a cloud/dev-gated fallback) and the editorial banner string. `guaranteed_team_ids` may contain the `"*"` wildcard (`ENROLL_ALL_TOKEN`): with it, enrollment inverts from an explicit allowlist to "every team that has enabled scout configs" — the self-serve gate, where the product-autonomy- gated UI creates the configs; explicit ids alongside `"*"` are still force-provisioned and `skip_team_ids` still hard-excludes. The global per-tick ceiling is flag-tunable too (`_resolve_global_max_runs_per_tick` ← `max_runs_per_tick_global`, default `MAX_RUNS_PER_TICK`).
- The `background` block (`_parse_background` → `BackgroundEnrollment`: `enabled`, `skill_name`, `team_ids`, `interval_minutes`, `max_new_teams_per_tick`, `bands`) is the pilot cohort that runs one scout without a person asking. The coordinator creates one `managed_by=background` config per listed team whose organization approved AI data processing, seeding only that skill, and pauses a background config with `background_removed` when its team leaves the list. Background configs dispatch only through `_collect_background_runs`, which checks consent again, so the kill switch (`enabled: false`) and the consent check gate every background run. A malformed block parses to `None`, and the coordinator then does nothing, so a typo in `team_ids` cannot pause the whole cohort.
- `background.bands` samples a percentage of each activity band on top of the hand-picked `team_ids`. `background_bands.py` writes the bands once a day to `SignalScoutBackgroundBand` (the `refresh_signal_scout_background_bands` Celery task). Eligibility, the band rules, and the one fleet-wide ClickHouse read are in its module docstring. A project in band `n` is sampled when `background_sample_bucket(team_id) < bands[n].percent`. The bucket is a stable digest, so a higher percent keeps every project a lower one sampled. A sampled project gets its config through the same creation path and `max_new_teams_per_tick` cap, with the band's `interval_minutes`, and `SignalScoutConfig.background_band` records the band. A lower percent, a band at 0, or a project that leaves its band removes the team from the listed set, so the existing `background_removed` pause applies, and a `managed_by=team` config never pauses. A malformed band entry counts as 0 percent for that band alone. A hand-picked team keeps no band. The runner stamps `background_band` on the run metadata and on both lifecycle events, next to `managed_by`.
- `background.backoff` (`enabled`, `factor`, `max_interval_minutes`) slows down a background config that nobody engages with. `SignalScoutConfig.background_backoff_level` counts the runs in a row with no engagement, and the coordinator makes the config due at `min(run_interval_minutes × factor^level, max_interval_minutes)`, so `run_interval_minutes` keeps the nominal band cadence. The stamp moves the level when it dispatches a run: up by one when the previous dispatch created a `SignalScoutRun` and no member engaged since, and back to 0 when one did. A previous dispatch that the child skipped (quota, daily report limit, an active run) keeps the level. Planning reads the same engagement, so engagement makes a backed-off config due at the band interval at once. Engagement is a `SignalReportAction` row of any type or a person-attributed dismissal or log artefact on the team's reports, read for all due teams together (`_engaged_team_ids`). A login is not engagement. A config above level 0 is stamped at the tick start rather than at its slot anchor, because a slot on a changing period would dispatch it early. Without the block, nothing reads or moves the level, and a `managed_by=team` config never backs off. The runner stamps `background_backoff_level` and `background_effective_interval_minutes` next to `background_band`; both describe the cadence after the run.
- `background.budget` (`max_runs_per_day`, optional `max_runs_per_tick`) caps the background runs one region dispatches in a trailing 24h window. The coordinator counts the `SignalScoutRun` rows stamped with the background origin (`BACKGROUND_RUN_METADATA`), and each tick admits at most `min(max_runs_per_day - runs_last_24h, max_runs_per_tick)` due background runs, including breaker probes. Without `max_runs_per_tick`, the day spreads evenly across the ticks of one day. The order is never-run first, then backoff level 0, then most overdue, then band and config pk. The rest are not stamped, so they stay due for the next tick, and their backoff level does not move. `0` stops background dispatch and keeps every config as it is. A missing or malformed block means no lane budget. A `managed_by=team` config and a scout a person enabled never count against the budget. Each tick that runs the lane logs and captures `signals_scout_background_tick` with `due`, `dispatched`, `deferred`, `runs_last_24h`, `max_runs_per_day`, `never_run_due`, and `oldest_due_age_s`.
+ `background_lanes` (`parse_background_lanes` → a tuple of `BackgroundEnrollment`, one per skill: `skill_name`, `enabled`, `audience`, `team_ids`, `interval_minutes`, `max_new_teams_per_tick`, `bands`, `backoff`, `budget`, `require_precheck`) runs scouts without a person asking. The older single `background` block parses as one lane with the default skill `signals-scout-general`, and applies only when `background_lanes` is absent. A lane is its skill: every background query below also filters on `skill_name`, so a lane never pauses, samples, budgets, or backs off another lane's configs. For each lane the coordinator creates one `managed_by=background` config per listed team whose organization approved AI data processing, seeding only that skill, and pauses a background config with `background_removed` when its team leaves the lane or its skill leaves the list. Background configs dispatch only through `_collect_background_runs`, which checks consent again, so each lane's kill switch (`enabled: false`) and the consent check gate every background run. A malformed lane, or two lanes with one `skill_name`, parses the whole list to `None`, and the coordinator then does nothing, so a typo cannot pause a cohort. A project can be in more than one lane; the shared tick caps, the enabled-scout cap, and `withheld_skills` apply to every lane.
+ `require_precheck` makes a lane's configs use the skill's default pre-check (`scout-precheck-query`) whatever `precheck_default_rollout` says (`precheck.resolve_effective_precheck`). A lane whose skill ships no default dispatches nothing and logs it.
+ A lane's `bands` sample a percentage of each activity band on top of the hand-picked `team_ids`. `background_bands.py` writes the bands once a day to `SignalScoutBackgroundBand` (the `refresh_signal_scout_background_bands` Celery task), with an `audience` of `not_set_up` or `set_up` per project, and a lane samples only the rows of its own `audience`. Eligibility, the band rules, and the one fleet-wide ClickHouse read are in its module docstring. A project in band `n` is sampled when `background_sample_bucket(team_id, skill_name) < bands[n].percent`. The bucket is a stable digest salted with the skill, so each lane samples on its own, and a higher percent keeps every project a lower one sampled. `signals-scout-general` keeps the salt it had before lanes, so its cohort did not move. A sampled project gets its config through the same creation path and `max_new_teams_per_tick` cap, with the band's `interval_minutes`, and `SignalScoutConfig.background_band` records the band. A lower percent, a band at 0, or a project that leaves its band removes the team from the listed set, so the existing `background_removed` pause applies, and a `managed_by=team` config never pauses. A malformed band entry counts as 0 percent for that band alone. A hand-picked team keeps no band. The runner stamps `background_band` on the run metadata and on both lifecycle events, next to `managed_by`.
+ A lane's `backoff` (`enabled`, `factor`, `max_interval_minutes`) slows down a background config that nobody engages with. `SignalScoutConfig.background_backoff_level` counts the runs in a row with no engagement, and the coordinator makes the config due at `min(run_interval_minutes × factor^level, max_interval_minutes)`, so `run_interval_minutes` keeps the nominal band cadence. The stamp moves the level when it dispatches a run: up by one when the previous dispatch created a `SignalScoutRun` and no member engaged since, and back to 0 when one did. A previous dispatch that the child skipped (quota, daily report limit, an active run) keeps the level. Planning reads the same engagement, so engagement makes a backed-off config due at the band interval at once. Engagement is a `SignalReportAction` row of any type or a person-attributed dismissal or log artefact on the team's reports, read for all due teams together (`_engaged_team_ids`). A login is not engagement. A config above level 0 is stamped at the tick start rather than at its slot anchor, because a slot on a changing period would dispatch it early. Without the block, nothing reads or moves the level, and a `managed_by=team` config never backs off. The runner stamps `background_backoff_level` and `background_effective_interval_minutes` next to `background_band`; both describe the cadence after the run.
+ A lane's `budget` (`max_runs_per_day`, optional `max_runs_per_tick`) caps the runs of that lane one region dispatches in a trailing 24h window. The coordinator counts the `SignalScoutRun` rows stamped with the background origin (`BACKGROUND_RUN_METADATA`), and each tick admits at most `min(max_runs_per_day - runs_last_24h, max_runs_per_tick)` due background runs, including breaker probes. Without `max_runs_per_tick`, the day spreads evenly across the ticks of one day. The order is never-run first, then backoff level 0, then most overdue, then band and config pk. The rest are not stamped, so they stay due for the next tick, and their backoff level does not move. `0` stops background dispatch and keeps every config as it is. A missing or malformed block means no lane budget. A `managed_by=team` config and a scout a person enabled never count against the budget. Each tick logs and captures one `signals_scout_background_tick` per enabled lane with `skill_name`, `due`, `dispatched`, `deferred`, `runs_last_24h`, `max_runs_per_day`, `never_run_due`, and `oldest_due_age_s`.
Kept free of the temporalio stack so it stays cheap to import on the API path; both the coordinator and the metadata endpoint import from here so the reported caps never drift from what dispatch allows. `resolve_team_metadata()` backs the metadata viewset;
`seed_config_layers_for_team()` lets the on-demand `sync` endpoint seed the same launch posture.
- `derived_metadata.py`
diff --git a/products/signals/backend/scout_harness/background_bands.py b/products/signals/backend/scout_harness/background_bands.py
index 19dc670451a7..900bb72c9a63 100644
--- a/products/signals/backend/scout_harness/background_bands.py
+++ b/products/signals/backend/scout_harness/background_bands.py
@@ -1,13 +1,15 @@
-"""Nightly activity bands for the background scout lane.
+"""Nightly activity bands for the background scout lanes.
-The coordinator samples a percentage of each band from the `background.bands` block of the
-`signals-scout` flag payload (see `team_limits._parse_background`). This module decides which
-projects are eligible and which band each one is in, once a day, so the 30-minute coordinator tick
-reads one small table instead of ClickHouse.
+Each lane samples a percentage of each band from its `bands` block in the `signals-scout` flag
+payload (see `team_limits.parse_background_lanes`). This module decides which projects are
+eligible, which band each one is in, and which audience it belongs to, once a day, so the
+30-minute coordinator tick reads one small table instead of ClickHouse.
+
+The audience is `set_up` when the project set Signals up (`suggestions.set_up_team_q`) and
+`not_set_up` otherwise. A lane samples only the rows of its own audience.
A project is eligible only when all of these hold:
-- it did not set Signals up (`suggestions.set_up_team_q`)
- its organization approved AI data processing
- it is a root project, not a demo, and its organization is not the internal-metrics one
- it is the project where the most recently active member of its organization last worked
@@ -68,6 +70,18 @@ class _OrgActivity:
last_login: datetime
+@frozen
+class TeamBand:
+ band: int
+ audience: SignalScoutBackgroundBand.Audience
+
+
+@frozen
+class _Eligibility:
+ paying: bool
+ set_up: bool
+
+
def _id_batches(ids: Collection[int]) -> Iterator[tuple[int, ...]]:
return batched(sorted(ids), ID_BATCH_SIZE, strict=False)
@@ -96,22 +110,22 @@ def _most_active_project_by_org(login_cutoff: datetime) -> dict[int, _OrgActivit
return by_org
-def _eligible_teams(team_ids: set[int]) -> dict[int, bool]:
- """`team_id -> paying` for each project in `team_ids` that can get a background scout."""
- eligible: dict[int, bool] = {}
+def _eligible_teams(team_ids: set[int]) -> dict[int, _Eligibility]:
+ """Whether each project in `team_ids` that can get a background scout is paying and set up."""
+ eligible: dict[int, _Eligibility] = {}
for batch in _id_batches(team_ids):
- rows = (
- Team.objects.filter(
- Q(parent_team_id__isnull=True) | Q(parent_team_id=F("id")),
- id__in=batch,
- is_demo=False,
- organization__is_ai_data_processing_approved=True,
- organization__for_internal_metrics=False,
- )
- .exclude(set_up_team_q())
- .values_list("id", "organization__has_active_subscription")
+ teams = Team.objects.filter(
+ Q(parent_team_id__isnull=True) | Q(parent_team_id=F("id")),
+ id__in=batch,
+ is_demo=False,
+ organization__is_ai_data_processing_approved=True,
+ organization__for_internal_metrics=False,
+ )
+ set_up = set(teams.filter(set_up_team_q()).values_list("id", flat=True))
+ eligible.update(
+ (team_id, _Eligibility(paying=paying is True, set_up=team_id in set_up))
+ for team_id, paying in teams.values_list("id", "organization__has_active_subscription")
)
- eligible.update((team_id, paying is True) for team_id, paying in rows)
return eligible
@@ -153,13 +167,13 @@ def band_for(*, paying: bool, events_per_day: float, last_login: datetime, now:
return 1 if events_per_day >= HIGH_VOLUME_EVENTS_PER_DAY else 2
-def compute_background_bands(now: datetime | None = None) -> dict[int, int]:
- """`team_id -> band` for every eligible project."""
+def compute_background_bands(now: datetime | None = None) -> dict[int, TeamBand]:
+ """The band and audience of every eligible project."""
now = now or timezone.now()
activity_by_org = _most_active_project_by_org(now - timedelta(days=MAX_LOGIN_DAYS))
eligible = _eligible_teams({activity.team_id for activity in activity_by_org.values()})
# The org's most active project must itself be eligible. The org never falls back to a
- # quieter project, so a set-up org does not get a background scout next to its own setup.
+ # quieter project, so a set-up org does not get a `not_set_up` scout next to its own setup.
activity_by_org = {
organization_id: activity
for organization_id, activity in activity_by_org.items()
@@ -169,16 +183,22 @@ def compute_background_bands(now: datetime | None = None) -> dict[int, int]:
return {}
day_end = now.replace(hour=0, minute=0, second=0, microsecond=0)
events_by_org = _events_per_day_by_org(activity_by_org.keys(), day_end - timedelta(days=EVENT_WINDOW_DAYS), day_end)
- bands: dict[int, int] = {}
+ bands: dict[int, TeamBand] = {}
for organization_id, activity in activity_by_org.items():
+ eligibility = eligible[activity.team_id]
band = band_for(
- paying=eligible[activity.team_id],
+ paying=eligibility.paying,
events_per_day=events_by_org.get(organization_id, 0.0),
last_login=activity.last_login,
now=now,
)
if band is not None:
- bands[activity.team_id] = band
+ audience = (
+ SignalScoutBackgroundBand.Audience.SET_UP
+ if eligibility.set_up
+ else SignalScoutBackgroundBand.Audience.NOT_SET_UP
+ )
+ bands[activity.team_id] = TeamBand(band=band, audience=audience)
return bands
@@ -193,10 +213,13 @@ def refresh_background_bands(now: datetime | None = None) -> BandOutcome:
deleted, _ = SignalScoutBackgroundBand.all_teams.filter(team_id__in=batch).delete()
removed += deleted
SignalScoutBackgroundBand.all_teams.bulk_create(
- [SignalScoutBackgroundBand(team_id=team_id, band=band, computed_at=now) for team_id, band in bands.items()],
+ [
+ SignalScoutBackgroundBand(team_id=team_id, band=entry.band, audience=entry.audience, computed_at=now)
+ for team_id, entry in bands.items()
+ ],
update_conflicts=True,
unique_fields=["team"],
- update_fields=["band", "computed_at"],
+ update_fields=["band", "audience", "computed_at"],
batch_size=1000,
)
return BandOutcome(written=len(bands), removed=removed)
diff --git a/products/signals/backend/scout_harness/precheck.py b/products/signals/backend/scout_harness/precheck.py
index 0db07429aa76..6ece739c9b48 100644
--- a/products/signals/backend/scout_harness/precheck.py
+++ b/products/signals/backend/scout_harness/precheck.py
@@ -30,6 +30,8 @@ The query comes from the first source that applies (`resolve_effective_precheck`
3. The default the canonical skill ships (`scout-precheck-query` frontmatter), for a project
inside that skill's rollout (`precheck_default_rollout` in the `signals-scout` flag payload).
Projects outside the rollout run as if there were no default, and form the control group.
+ A background config whose lane sets `require_precheck` always uses the default, whatever the
+ rollout says.
"""
from __future__ import annotations
@@ -67,6 +69,7 @@ from products.signals.backend.scout_harness.limits import MAX_PRECHECK_ROWS_BYTE
from products.signals.backend.scout_harness.team_limits import (
precheck_default_rollout_bucket,
precheck_default_rollout_percent,
+ resolve_background_lane,
)
from products.skills.backend.models.skills import LLMSkill
@@ -150,11 +153,14 @@ def _placeholders(*, since: datetime, now: datetime, interval_minutes: int) -> d
}
-def resolve_effective_precheck(config: SignalScoutConfig, *, is_canonical: bool) -> EffectivePrecheck:
+def resolve_effective_precheck(
+ config: SignalScoutConfig, *, is_canonical: bool, require_default: bool = False
+) -> EffectivePrecheck:
"""The pre-check query a scheduled run of this scout uses, and where it came from.
`is_canonical` says whether the project's skill row is the one the harness ships. A team's own
- skill can share a canonical name, and it inherits no default from disk.
+ skill can share a canonical name, and it inherits no default from disk. `require_default` skips
+ the rollout check, for a background lane that dispatches only when the default finds something.
"""
if config.precheck_disabled:
return EffectivePrecheck(query=None, source="off")
@@ -164,11 +170,18 @@ def resolve_effective_precheck(config: SignalScoutConfig, *, is_canonical: bool)
if not default:
return EffectivePrecheck(query=None, source="off")
bucket = precheck_default_rollout_bucket(config.team_id, config.skill_name)
- if bucket >= precheck_default_rollout_percent(config.skill_name):
+ if not require_default and bucket >= precheck_default_rollout_percent(config.skill_name):
return EffectivePrecheck(query=None, source="off", rollout_bucket=bucket)
return EffectivePrecheck(query=default, source="skill_default", rollout_bucket=bucket)
+def _background_lane_requires_precheck(config: SignalScoutConfig) -> bool:
+ if config.managed_by != SignalScoutConfig.ManagedBy.BACKGROUND:
+ return False
+ lane = resolve_background_lane(config.skill_name)
+ return lane is not None and lane.require_precheck
+
+
def scout_skill_is_canonical(team_id: int, skill_name: str) -> bool:
"""Whether the project's latest skill row for this scout is the one the harness ships."""
metadata = (
@@ -211,6 +224,7 @@ def evaluate_scout_precheck(team_id: int, skill_name: str, now: datetime | None
"skill_name",
"created_at",
"status",
+ "managed_by",
"enabled_by",
"created_by",
"precheck_query",
@@ -224,7 +238,11 @@ def evaluate_scout_precheck(team_id: int, skill_name: str, now: datetime | None
# succeed. A skip would hold the lane paused for as long as it stays quiet.
if config is None or config.status not in SignalScoutConfig.RUNNABLE_STATUSES:
return None
- effective = resolve_effective_precheck(config, is_canonical=scout_skill_is_canonical(team_id, skill_name))
+ effective = resolve_effective_precheck(
+ config,
+ is_canonical=scout_skill_is_canonical(team_id, skill_name),
+ require_default=_background_lane_requires_precheck(config),
+ )
if effective.query is None:
return None
diff --git a/products/signals/backend/scout_harness/runner.py b/products/signals/backend/scout_harness/runner.py
index 6563c2c202db..bc8fbf83f219 100644
--- a/products/signals/backend/scout_harness/runner.py
+++ b/products/signals/backend/scout_harness/runner.py
@@ -533,9 +533,10 @@ async def _arun_signals_scout(
business_knowledge_maintained = await database_sync_to_async(
_business_knowledge_maintained_for_team, thread_sensitive=False
)(team)
- # Resolved here for the same reason, so every lifecycle event reports the same cadence.
+ # Resolved here for the same reason, so every lifecycle event reports the same cadence. The
+ # backoff comes from the lane that runs this config's skill.
backoff = (
- await asyncio.to_thread(resolve_background_backoff)
+ await asyncio.to_thread(resolve_background_backoff, config.skill_name)
if config.managed_by == SignalScoutConfig.ManagedBy.BACKGROUND
else None
)
diff --git a/products/signals/backend/scout_harness/team_limits.py b/products/signals/backend/scout_harness/team_limits.py
index 3572c7845718..7cb7406c57d1 100644
--- a/products/signals/backend/scout_harness/team_limits.py
+++ b/products/signals/backend/scout_harness/team_limits.py
@@ -29,7 +29,7 @@ from posthog.exceptions_capture import capture_exception
from posthog.models import Team
from products.signals.backend.github_optional import ResearchAccessPolicy
-from products.signals.backend.models import SignalScoutRun
+from products.signals.backend.models import SignalScoutBackgroundBand, SignalScoutRun
from products.signals.backend.scout_harness.limits import (
COORDINATOR_INTERVAL_MINUTES,
MAX_DISPATCH_SMEAR_SECONDS,
@@ -229,9 +229,11 @@ def _parse_enrollment(payload: dict | None) -> Enrollment:
return Enrollment(wildcard=wildcard, explicit=explicit, skip=skip, drained=payload.get("guaranteed_team_ids") == [])
-# Flag payload block that enrolls a pilot cohort in one scout that PostHog sets up without a person
-# asking. Separate from `guaranteed_team_ids`: a background team gets one config for one skill, not
-# the full catalog seed.
+# Flag payload blocks that enroll a cohort in scouts that PostHog sets up without a person asking.
+# Separate from `guaranteed_team_ids`: a background team gets one config per lane skill, not the
+# full catalog seed. `background_lanes` holds one lane per skill. The older single `background`
+# block parses as one lane and applies only when `background_lanes` is absent.
+BACKGROUND_LANES_KEY = "background_lanes"
BACKGROUND_KEY = "background"
DEFAULT_BACKGROUND_SKILL_NAME = "signals-scout-general"
# Bounds how many teams get a new background config in one tick, so a large list starts over
@@ -292,9 +294,12 @@ class BackgroundBudget:
@dataclass(frozen=True)
class BackgroundEnrollment:
- """Parsed `background` block from the `signals-scout` flag payload.
+ """One background lane: an entry of `background_lanes`, or the older `background` block.
- `enabled` → the coordinator creates and dispatches background configs. A valid block with
+ A lane is its skill. Every config, sample, budget, and backoff of the lane belongs to
+ `skill_name` alone, so two lanes never change each other's configs.
+
+ `enabled` → the coordinator creates and dispatches background configs. A valid lane with
`enabled` off still pauses configs whose team left `team_ids`. `interval_minutes` is `None`
when unset, so the created row keeps the model default.
"""
@@ -310,6 +315,11 @@ class BackgroundEnrollment:
backoff: BackgroundBackoff | None = None
# `None` means no lane budget, so every due background run goes on to the shared tick caps.
budget: BackgroundBudget | None = None
+ # The band rows the lane samples from: projects that did not set Signals up, or ones that did.
+ audience: str = SignalScoutBackgroundBand.Audience.NOT_SET_UP
+ # Dispatch only when the skill's default pre-check finds something, whatever the skill's
+ # `precheck_default_rollout` says. Keeps a short interval cheap.
+ require_precheck: bool = False
def band_interval_minutes(self, band: int | None) -> int | None:
entry = self.bands.get(band) if band is not None else None
@@ -373,19 +383,33 @@ def _parse_background_budget(raw: object) -> BackgroundBudget | None:
)
-def resolve_background_backoff() -> BackgroundBackoff | None:
- """One flag-payload read of `background.backoff`, for callers outside the coordinator."""
- background = _parse_background(_read_flag_payload())
- return background.backoff if background is not None else None
+def background_lane_for(lanes: Sequence[BackgroundEnrollment] | None, skill_name: str) -> BackgroundEnrollment | None:
+ return next((lane for lane in lanes or () if lane.skill_name == skill_name), None)
+
+
+def resolve_background_lane(skill_name: str) -> BackgroundEnrollment | None:
+ """One flag-payload read of the lane that runs `skill_name`, for callers outside the coordinator."""
+ return background_lane_for(parse_background_lanes(_read_flag_payload()), skill_name)
-def background_sample_bucket(team_id: int) -> int:
- """The stable 0-99 bucket of a project. Python's `hash()` is salted per process, so it cannot be used.
+def resolve_background_backoff(skill_name: str) -> BackgroundBackoff | None:
+ lane = resolve_background_lane(skill_name)
+ return lane.backoff if lane is not None else None
+
+
+def background_sample_bucket(team_id: int, skill_name: str = DEFAULT_BACKGROUND_SKILL_NAME) -> int:
+ """The stable 0-99 bucket of a project in one lane. Python's `hash()` is salted per process, so it cannot be used.
A project is sampled when its bucket is below its band percent, so a higher percent keeps every
- project that a lower percent sampled.
+ project that a lower percent sampled. The skill name salts the hash, so each lane samples on
+ its own. The general scout keeps the salt it had before lanes, so its cohort does not move.
"""
- digest = hashlib.sha256(f"signals-scout-background:{team_id}".encode()).digest()
+ salt = (
+ "signals-scout-background"
+ if skill_name == DEFAULT_BACKGROUND_SKILL_NAME
+ else f"signals-scout-background:{skill_name}"
+ )
+ digest = hashlib.sha256(f"{salt}:{team_id}".encode()).digest()
return int.from_bytes(digest[:8], "big") % 100
@@ -422,26 +446,70 @@ def precheck_default_rollout_bucket(team_id: int, skill_name: str) -> int:
return int.from_bytes(digest[:8], "big") % 100
-def _parse_background(payload: dict | None) -> BackgroundEnrollment | None:
- """Parse the `background` block, or return `None` when the coordinator must not act on it.
-
- `None` for a missing payload, a missing block, a block that is not an object, a `skill_name`
- that is not a non-empty string, or a `team_ids` that is not a list of integer ids. A malformed
- `team_ids` must not read as an empty list, because an empty list pauses every background
- config. `enabled` is on only for a literal `true`. An absent or malformed `interval_minutes`,
- `max_new_teams_per_tick`, `bands` entry, `backoff`, or `budget` falls back to its default and does
- not invalidate the block.
+def parse_background_lanes(payload: dict | None) -> tuple[BackgroundEnrollment, ...] | None:
+ """Parse the background lanes, or return `None` when the coordinator must not act on them.
+
+ `background_lanes` wins over `background` when both are present. `None` for a missing payload,
+ no lane key at all, a `background_lanes` value that is not a list, any malformed lane, or two
+ lanes with the same `skill_name`. One bad lane fails the whole list, because a lane that drops
+ out reads as a removed lane, and a removed lane pauses its configs. An empty list is valid and
+ removes every lane.
"""
if payload is None:
return None
- raw = payload.get(BACKGROUND_KEY)
+ if BACKGROUND_LANES_KEY not in payload:
+ lane = _parse_background(payload)
+ return (lane,) if lane is not None else None
+
+ if BACKGROUND_KEY in payload:
+ logger.info("signals_scout: background_lanes is set, ignoring the background block")
+ raw_lanes = payload[BACKGROUND_LANES_KEY]
+ if not isinstance(raw_lanes, list):
+ logger.warning("signals_scout: background_lanes is not a list, skipping background lanes")
+ return None
+ lanes: list[BackgroundEnrollment] = []
+ for index, raw in enumerate(raw_lanes):
+ lane = _parse_background_lane(raw, default_skill_name=None)
+ if lane is None:
+ logger.warning("signals_scout: malformed background lane, skipping background lanes", index=index)
+ return None
+ lanes.append(lane)
+ skill_names = [lane.skill_name for lane in lanes]
+ if len(set(skill_names)) != len(skill_names):
+ logger.warning("signals_scout: duplicate background lane skill, skipping background lanes", skills=skill_names)
+ return None
+ return tuple(lanes)
+
+
+def _parse_background(payload: dict | None) -> BackgroundEnrollment | None:
+ """The older single `background` block as one lane. `None` when it is missing or malformed."""
+ if payload is None:
+ return None
+ return _parse_background_lane(payload.get(BACKGROUND_KEY), default_skill_name=DEFAULT_BACKGROUND_SKILL_NAME)
+
+
+def _parse_background_lane(raw: object, *, default_skill_name: str | None) -> BackgroundEnrollment | None:
+ """Parse one lane, or return `None` when the coordinator must not act on it.
+
+ `None` for a lane that is not an object, a `skill_name` that is not a non-empty string (a
+ `background_lanes` entry has no default skill), a `team_ids` that is not a list of integer ids,
+ or an `audience` that is not a known value. A malformed `team_ids` must not read as an empty
+ list, because an empty list pauses every config of the lane. `enabled` and `require_precheck`
+ are on only for a literal `true`. An absent or malformed `interval_minutes`,
+ `max_new_teams_per_tick`, `bands` entry, `backoff`, or `budget` falls back to its default and
+ does not invalidate the lane.
+ """
if not isinstance(raw, dict):
return None
- skill_name = raw.get("skill_name", DEFAULT_BACKGROUND_SKILL_NAME)
+ skill_name = raw.get("skill_name", default_skill_name)
if not isinstance(skill_name, str) or not skill_name.strip():
return None
+ audience = raw.get("audience", SignalScoutBackgroundBand.Audience.NOT_SET_UP)
+ if audience not in SignalScoutBackgroundBand.Audience.values:
+ return None
+
raw_team_ids = raw.get("team_ids", [])
if not isinstance(raw_team_ids, list) or not all(
isinstance(team_id, int) and not isinstance(team_id, bool) for team_id in raw_team_ids
@@ -458,6 +526,8 @@ def _parse_background(payload: dict | None) -> BackgroundEnrollment | None:
bands=_parse_background_bands(raw.get("bands")),
backoff=_parse_background_backoff(raw.get("backoff")),
budget=_parse_background_budget(raw.get("budget")),
+ audience=SignalScoutBackgroundBand.Audience(audience),
+ require_precheck=raw.get("require_precheck") is True,
)
diff --git a/products/signals/backend/temporal/agentic/scout_coordinator.py b/products/signals/backend/temporal/agentic/scout_coordinator.py
index f8205ad352e4..c7a958657565 100644
--- a/products/signals/backend/temporal/agentic/scout_coordinator.py
+++ b/products/signals/backend/temporal/agentic/scout_coordinator.py
@@ -4,6 +4,7 @@ import json
import asyncio
import hashlib
from collections import Counter
+from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta, tzinfo
from typing import Any
@@ -11,7 +12,7 @@ from uuid import UUID
from django.conf import settings
from django.db import transaction
-from django.db.models import Case, Count, DateTimeField, Max, PositiveSmallIntegerField, Q, Value, When
+from django.db.models import Case, Count, DateTimeField, Max, PositiveSmallIntegerField, Q, QuerySet, Value, When
from django.utils import timezone
import structlog
@@ -50,6 +51,7 @@ from products.signals.backend.scout_harness.config_registry import (
from products.signals.backend.scout_harness.lazy_seed import (
canonical_config_tags_for,
canonical_display_name_for,
+ canonical_precheck_query_for,
canonical_skill_names,
discover_canonical_skills,
sync_canonical_skills,
@@ -77,7 +79,6 @@ from products.signals.backend.scout_harness.team_limits import (
Enrollment,
_canonicalize_team_config_keys,
_default_team_config,
- _parse_background,
_parse_enrollment,
_read_flag_payload,
_resolve_child_start_delay,
@@ -90,6 +91,7 @@ from products.signals.backend.scout_harness.team_limits import (
_runs_today_by_team,
_team_configs,
background_sample_bucket,
+ parse_background_lanes,
resolve_max_enabled_scouts,
)
from products.signals.backend.temporal.agentic.scout_scheduler import RunSignalsScoutInput, RunSignalsScoutWorkflow
@@ -114,7 +116,7 @@ MAX_RUNS_PER_TICK = 1000
# size lanes off the cadence dispatch actually produces; it is imported above.
TICK_SECONDS = COORDINATOR_INTERVAL_MINUTES * 60
-# One event per region per tick that runs the background lane, to chart its backlog and time to first run.
+# One event per region per tick and lane, to chart each lane's backlog and time to first run.
BACKGROUND_TICK_EVENT = "signals_scout_background_tick"
# One event per region per tick, and one per project with due runs that tick. Each carries the limits
# the flag payload resolved next to the usage, so a chart never needs its own copy of the limits.
@@ -235,7 +237,7 @@ async def fetch_enabled_signals_scout_runs_activity(
# configs are derived from the same snapshot so they can't disagree across two reads.
payload = await asyncio.to_thread(_read_flag_payload)
enrollment = _parse_enrollment(payload)
- background = _parse_background(payload)
+ background_lanes = parse_background_lanes(payload)
team_configs = _team_configs(payload)
default_team_config = _default_team_config(payload)
# The global per-tick ceiling is flag-tunable (no deploy): resolve it here off the same
@@ -253,7 +255,7 @@ async def fetch_enabled_signals_scout_runs_activity(
team_configs,
default_team_config,
global_max_runs_per_tick,
- background,
+ background_lanes,
start_delay_spread_seconds=smear_seconds if child_start_delay else 0,
)
logger.info("signals_scout coordinator: planned runs", count=len(planned))
@@ -282,12 +284,12 @@ async def stamp_dispatched_signals_scout_runs_activity(
# `database_sync_to_async`'s pool is sized for DB-bound work.
payload = await asyncio.to_thread(_read_flag_payload)
slot_aligned = _resolve_slot_aligned_dispatch(payload)
- background = _parse_background(payload)
+ backoffs = {lane.skill_name: lane.backoff for lane in parse_background_lanes(payload) or () if lane.backoff}
await database_sync_to_async(_stamp_dispatched_runs, thread_sensitive=False)(
stamp_input.dispatched_runs,
slot_aligned=slot_aligned,
dispatched_at=stamp_input.dispatched_at,
- backoff=background.backoff if background is not None else None,
+ backoffs=backoffs,
)
# Counted here rather than in the workflow so a replay cannot double-count it, and after the
# stamp so a retried attempt counts the batch once.
@@ -370,7 +372,7 @@ def _stamp_dispatched_runs(
*,
slot_aligned: bool = True,
dispatched_at: datetime | None = None,
- backoff: BackgroundBackoff | None = None,
+ backoffs: Mapping[str, BackgroundBackoff] | None = None,
) -> None:
"""Sync bulk stamp. `.update()` bypasses save(), so this per-tick write never hits the
activity log.
@@ -396,8 +398,8 @@ def _stamp_dispatched_runs(
slot on a new period can sit up to that whole period back, which would dispatch it early and
undo the backoff. The tick start is already on the grid, so nothing drifts.
- With `backoff` set, this also moves the backoff level of each dispatched background config (see
- `_next_backoff_levels`).
+ `backoffs` maps a lane skill to its backoff. This also moves the backoff level of each dispatched
+ background config whose lane has one (see `_next_backoff_levels`).
"""
if not dispatched_runs:
return
@@ -416,9 +418,10 @@ def _stamp_dispatched_runs(
"managed_by",
"background_backoff_level",
"last_run_at",
+ "skill_name",
)
)
- levels = _next_backoff_levels(rows, backoff, now) if backoff is not None else {}
+ levels = _next_backoff_levels(rows, backoffs, now) if backoffs else {}
# The level and `last_run_at` commit together. After a partial write, the retried stamp would
# read the old `last_run_at` and raise the level a second time.
with transaction.atomic():
@@ -452,26 +455,29 @@ def _stamp_dispatched_runs(
)
-def _next_backoff_levels(rows: list[tuple], backoff: BackgroundBackoff, now: datetime) -> dict[UUID, int]:
- """The backoff level to store for each enabled rolling-interval background config in `rows`.
+def _next_backoff_levels(
+ rows: list[tuple], backoffs: Mapping[str, BackgroundBackoff], now: datetime
+) -> dict[UUID, int]:
+ """The backoff level to store for each enabled rolling-interval background config in `rows` whose lane backs off.
A config that a member engaged with since its previous run goes back to level 0. A config with a
previous run and no engagement goes up one level. The first run of a config keeps level 0,
because no earlier run exists for anyone to ignore. A previous dispatch that the child skipped
(quota, daily report limit, an active run) left no output to ignore, so it keeps the level.
"""
- previous_runs: dict[UUID, tuple[int, int, int | None, datetime]] = {
- pk: (team_id, interval_minutes, level, last_run_at)
- for pk, team_id, enabled, cron_schedule, interval_minutes, managed_by, level, last_run_at in rows
+ previous_runs: dict[UUID, tuple[int, int, int | None, datetime, BackgroundBackoff]] = {
+ pk: (team_id, interval_minutes, level, last_run_at, backoffs[skill_name])
+ for pk, team_id, enabled, cron_schedule, interval_minutes, managed_by, level, last_run_at, skill_name in rows
if enabled
and not cron_schedule
and managed_by == SignalScoutConfig.ManagedBy.BACKGROUND
and last_run_at is not None
+ and skill_name in backoffs
}
- engaged = _engaged_team_ids({team_id: since for team_id, _, _, since in previous_runs.values()})
- ran = _configs_with_run_between({pk: since for pk, (_, _, _, since) in previous_runs.items()}, now)
+ engaged = _engaged_team_ids({team_id: since for team_id, _, _, since, _ in previous_runs.values()})
+ ran = _configs_with_run_between({pk: since for pk, (_, _, _, since, _) in previous_runs.items()}, now)
levels: dict[UUID, int] = {}
- for pk, (team_id, interval_minutes, level, _) in previous_runs.items():
+ for pk, (team_id, interval_minutes, level, _, backoff) in previous_runs.items():
if team_id in engaged:
levels[pk] = 0
elif pk in ran:
@@ -558,22 +564,30 @@ class _BackgroundDueRun:
due_age_s: float
+@dataclass(frozen=True)
+class _BackgroundLaneTick:
+ lane: BackgroundEnrollment
+ due: list[_BackgroundDueRun]
+ runs_last_24h: int | None
+
+
def _collect_planned_runs(
enrollment: Enrollment,
team_configs: dict[int, dict] | None = None,
default_team_config: dict | None = None,
max_runs_per_tick: int | None = None,
- background: BackgroundEnrollment | None = None,
+ background_lanes: Sequence[BackgroundEnrollment] | None = None,
start_delay_spread_seconds: int = 0,
) -> list[PlannedRun]:
"""Sync DB scan. Runs in a worker thread via Django's per-thread connection mgmt.
Takes the parsed enrollment (explicit allowlist + the `"*"` wildcard), the optional per-team
config overrides, the fleet-wide default config, the resolved global per-tick ceiling, and the
- parsed `background` block — so the flag reads all stay off this DB pool.
+ parsed background lanes — so the flag reads all stay off this DB pool. `None` lanes (no lane
+ key, or a malformed list) leave every background config as it is.
Background configs are dispatched only through `_collect_background_runs`, never through the
- participating-team loop, so the consent check and the background kill switch gate every one.
+ participating-team loop, so the consent check and each lane's kill switch gate every one.
`start_delay_spread_seconds` is the window the start delays of the planned runs spread across.
0 gives every run no delay.
@@ -582,23 +596,29 @@ def _collect_planned_runs(
team_configs = _canonicalize_team_config_keys(team_configs or {})
default_team_config = default_team_config or {}
due: list[_DueRun] = []
- background_team_ids: set[int] = set()
- if background is not None:
- skipped_team_ids = _canonicalize_team_ids(enrollment.skip)
- background_team_ids = _canonicalize_team_ids(set(background.team_ids)) - skipped_team_ids
+ lane_team_ids: dict[str, set[int]] = {}
+ if background_lanes is not None:
try:
- # A hand-picked team keeps no band, so `team_ids` stays an override that bands never change.
- band_by_team = {
- team_id: band
- for team_id, band in _sampled_band_teams(background).items()
- if team_id not in background_team_ids and team_id not in skipped_team_ids
- }
- background_team_ids |= band_by_team.keys()
- _reconcile_background_configs(
- background, background_team_ids, band_by_team, team_configs, default_team_config
- )
+ _pause_removed_lane_configs({lane.skill_name for lane in background_lanes})
except Exception:
- logger.exception("signals_scout coordinator: background reconcile failed; continuing")
+ logger.exception("signals_scout coordinator: removed background lane pause failed; continuing")
+ skipped_team_ids = _canonicalize_team_ids(enrollment.skip)
+ for lane in background_lanes:
+ listed_team_ids = _canonicalize_team_ids(set(lane.team_ids)) - skipped_team_ids
+ try:
+ # A hand-picked team keeps no band, so `team_ids` stays an override that bands never change.
+ band_by_team = {
+ team_id: band
+ for team_id, band in _sampled_band_teams(lane).items()
+ if team_id not in listed_team_ids and team_id not in skipped_team_ids
+ }
+ listed_team_ids |= band_by_team.keys()
+ _reconcile_background_configs(lane, listed_team_ids, band_by_team, team_configs, default_team_config)
+ except Exception:
+ logger.exception(
+ "signals_scout coordinator: background reconcile failed; continuing", skill_name=lane.skill_name
+ )
+ lane_team_ids[lane.skill_name] = listed_team_ids
paused_by_team = _breaker_paused_configs_by_team()
skipped_without_consent = 0
reconcile_by_team = operational_configs_needing_reconcile() if enrollment.wildcard else {}
@@ -690,22 +710,26 @@ def _collect_planned_runs(
team_count=skipped_without_consent,
)
- background_due: list[_BackgroundDueRun] = []
- runs_last_24h: int | None = None
- if background is not None and background.enabled:
- background_due = _collect_background_runs(
- background,
- background_team_ids,
+ lane_ticks: list[_BackgroundLaneTick] = []
+ for lane in background_lanes or ():
+ if not lane.enabled:
+ continue
+ listed_team_ids = lane_team_ids.get(lane.skill_name, set())
+ lane_due = _collect_background_runs(
+ lane,
+ listed_team_ids,
team_configs,
default_team_config,
now,
start_delay_spread_seconds=start_delay_spread_seconds,
)
- admitted = [d.run for d in background_due]
- if background.budget is not None and background_due:
- runs_last_24h = _background_runs_since(background, background_team_ids, now - DAILY_BUDGET_WINDOW)
- admitted = _apply_background_budget(background_due, background.budget, runs_last_24h)
+ admitted = [d.run for d in lane_due]
+ runs_last_24h: int | None = None
+ if lane.budget is not None and lane_due:
+ runs_last_24h = _background_runs_since(lane, listed_team_ids, now - DAILY_BUDGET_WINDOW)
+ admitted = _apply_background_budget(lane_due, lane.budget, runs_last_24h)
due.extend(admitted)
+ lane_ticks.append(_BackgroundLaneTick(lane=lane, due=lane_due, runs_last_24h=runs_last_24h))
selected: list[_DueRun] = []
runs_today: dict[int, int] = {}
@@ -720,8 +744,8 @@ def _collect_planned_runs(
runs_today = _runs_today_by_team(capped_team_ids, now - DAILY_BUDGET_WINDOW)
selected = _allocate_tick_budget(due, team_configs, default_team_config, runs_today, max_runs_per_tick)
_capture_capacity_tick(due, selected, team_configs, default_team_config, runs_today, max_runs_per_tick)
- if background is not None and background.enabled:
- _capture_background_tick(background_due, selected, background.budget, runs_last_24h)
+ for lane_tick in lane_ticks:
+ _capture_background_tick(lane_tick, selected)
planned = [
PlannedRun(team_id=d.team_id, skill_name=d.skill_name, start_delay_seconds=d.start_delay_s) for d in selected
]
@@ -731,7 +755,7 @@ def _collect_planned_runs(
def _background_runs_since(background: BackgroundEnrollment, listed_team_ids: set[int], since: datetime) -> int:
- """Background runs that started since `since` in this region, in one aggregate query.
+ """Background runs of one lane that started since `since` in this region, in one aggregate query.
Counts the bridge rows stamped with the background origin, so a run still counts after a person
takes its config over. The team and skill filters keep the read on the per-scout run index. A
@@ -766,6 +790,7 @@ def _apply_background_budget(
if len(ordered) > allowance:
logger.info(
"signals_scout coordinator: background budget defers due runs",
+ skill_name=ordered[0].run.skill_name,
due=len(ordered),
allowance=allowance,
runs_last_24h=runs_last_24h,
@@ -774,25 +799,23 @@ def _apply_background_budget(
return [d.run for d in ordered[:allowance]]
-def _capture_background_tick(
- background_due: list[_BackgroundDueRun],
- selected: list[_DueRun],
- budget: BackgroundBudget | None,
- runs_last_24h: int | None,
-) -> None:
- """Log and capture the backlog of the background lane for this tick.
+def _capture_background_tick(lane_tick: _BackgroundLaneTick, selected: list[_DueRun]) -> None:
+ """Log and capture the backlog of one background lane for this tick.
`dispatched` counts the background runs that also pass the shared tick caps, so `due`,
`dispatched`, and `deferred` always add up.
"""
selected_pks = {d.config_pk for d in selected}
+ background_due = lane_tick.due
+ budget = lane_tick.lane.budget
dispatched = sum(1 for d in background_due if d.run.config_pk in selected_pks)
properties = {
"region": settings.CLOUD_DEPLOYMENT,
+ "skill_name": lane_tick.lane.skill_name,
"due": len(background_due),
"dispatched": dispatched,
"deferred": len(background_due) - dispatched,
- "runs_last_24h": runs_last_24h,
+ "runs_last_24h": lane_tick.runs_last_24h,
"max_runs_per_day": budget.max_runs_per_day if budget is not None else None,
"never_run_due": sum(1 for d in background_due if d.never_run),
"oldest_due_age_s": max((d.due_age_s for d in background_due), default=0.0),
@@ -1198,7 +1221,7 @@ def _ai_data_processing_approved(team: Team) -> bool:
def _sampled_band_teams(background: BackgroundEnrollment) -> dict[int, int]:
- """`team_id -> band` for each project that its band percent samples into the background lane.
+ """`team_id -> band` for each project of the lane's audience that its band percent samples into the lane.
Reads nothing when every band is at 0 percent, which is the rollout default.
"""
@@ -1207,10 +1230,10 @@ def _sampled_band_teams(background: BackgroundEnrollment) -> dict[int, int]:
return {}
return {
team_id: band
- for team_id, band in SignalScoutBackgroundBand.all_teams.filter(band__in=percents).values_list(
- "team_id", "band"
- )
- if background_sample_bucket(team_id) < percents[band]
+ for team_id, band in SignalScoutBackgroundBand.all_teams.filter(
+ band__in=percents, audience=background.audience
+ ).values_list("team_id", "band")
+ if background_sample_bucket(team_id, background.skill_name) < percents[band]
}
@@ -1221,7 +1244,7 @@ def _reconcile_background_configs(
team_configs: dict[int, dict],
default_team_config: dict,
) -> None:
- """Move background configs toward the `background` block of the flag payload.
+ """Move the background configs of one lane toward that lane in the flag payload.
`listed_team_ids` holds the hand-picked `team_ids` and the band-sampled teams in `band_by_team`.
Runs on a valid block only, because an unreadable block must never read as an empty list. The
@@ -1229,7 +1252,7 @@ def _reconcile_background_configs(
stopping the pilot. A lower band percent pauses through the same path, because the team leaves
the listed set. Creation, resume, and band updates run only with `enabled` on.
"""
- _pause_departed_background_configs(listed_team_ids)
+ _pause_departed_background_configs(background.skill_name, listed_team_ids)
if not background.enabled:
return
if background.skill_name not in canonical_skill_names():
@@ -1249,23 +1272,40 @@ def _reconcile_background_configs(
_create_background_configs(background, eligible, band_by_team, team_configs, default_team_config)
-def _pause_departed_background_configs(listed_team_ids: set[int]) -> None:
- """Pause every runnable background config whose team is no longer listed.
+def _pause_removed_lane_configs(lane_skill_names: set[str]) -> None:
+ """Pause every runnable background config whose skill has no lane any more."""
+ _pause_background_configs(
+ SignalScoutConfig.all_teams.filter(
+ managed_by=SignalScoutConfig.ManagedBy.BACKGROUND,
+ status__in=SignalScoutConfig.RUNNABLE_STATUSES,
+ ).exclude(skill_name__in=lane_skill_names)
+ )
+
+
+def _pause_departed_background_configs(skill_name: str, listed_team_ids: set[int]) -> None:
+ """Pause every runnable background config of one lane whose team is no longer listed."""
+ _pause_background_configs(
+ SignalScoutConfig.all_teams.filter(
+ skill_name=skill_name,
+ managed_by=SignalScoutConfig.ManagedBy.BACKGROUND,
+ status__in=SignalScoutConfig.RUNNABLE_STATUSES,
+ ).exclude(team_id__in=listed_team_ids)
+ )
+
+
+def _pause_background_configs(configs: QuerySet[SignalScoutConfig]) -> None:
+ """Pause `configs` with `background_removed`.
`transition_status_by_system` refuses a row a person paused or another writer owns, so this
only ever pauses what the background coordinator may still change.
"""
- departed = SignalScoutConfig.all_teams.filter(
- managed_by=SignalScoutConfig.ManagedBy.BACKGROUND,
- status__in=SignalScoutConfig.RUNNABLE_STATUSES,
- ).exclude(team_id__in=listed_team_ids)
- for config in departed:
+ for config in configs:
if config.transition_status_by_system(
SignalScoutConfig.Status.PAUSED_BY_SYSTEM,
pause_reason=SignalScoutConfig.PauseReason.BACKGROUND_REMOVED,
):
logger.info(
- "signals_scout coordinator: paused background scout for a team that left the list",
+ "signals_scout coordinator: paused background scout that left its lane",
team_id=config.team_id,
skill_name=config.skill_name,
)
@@ -1419,6 +1459,12 @@ def _collect_background_runs(
"""
if not listed_team_ids:
return []
+ if background.require_precheck and not canonical_precheck_query_for(background.skill_name):
+ logger.warning(
+ "signals_scout coordinator: background lane requires a pre-check its skill does not ship; skipping it",
+ skill_name=background.skill_name,
+ )
+ return []
configs = list(
SignalScoutConfig.all_teams.filter(
Q(enabled=True)
diff --git a/products/signals/backend/test/test_scout_background_bands.py b/products/signals/backend/test/test_scout_background_bands.py
index 329d5fc28004..992c9af41a65 100644
--- a/products/signals/backend/test/test_scout_background_bands.py
+++ b/products/signals/backend/test/test_scout_background_bands.py
@@ -69,7 +69,18 @@ class TestRefreshBackgroundBands:
with patch(_SYNC_EXECUTE, return_value=rows):
outcome = refresh_background_bands(_NOW)
- bands = dict(SignalScoutBackgroundBand.all_teams.values_list("team_id", "band"))
- assert bands == {paying.id: 1, free.id: 3, second_project.id: 1}
+ not_set_up, set_up_audience = SignalScoutBackgroundBand.Audience
+ bands = {
+ team_id: (band, audience)
+ for team_id, band, audience in SignalScoutBackgroundBand.all_teams.values_list(
+ "team_id", "band", "audience"
+ )
+ }
+ assert bands == {
+ paying.id: (1, not_set_up),
+ free.id: (3, not_set_up),
+ second_project.id: (1, not_set_up),
+ set_up.id: (1, set_up_audience),
+ }
assert not SignalScoutBackgroundBand.all_teams.filter(pk=stale.pk).exists()
- assert outcome.written == 3
+ assert outcome.written == 4
diff --git a/products/signals/backend/test/test_scout_coordinator.py b/products/signals/backend/test/test_scout_coordinator.py
index 72ec672d7281..01f2c6212796 100644
--- a/products/signals/backend/test/test_scout_coordinator.py
+++ b/products/signals/backend/test/test_scout_coordinator.py
@@ -1,6 +1,8 @@
from __future__ import annotations
import random
+import hashlib
+import dataclasses
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -72,6 +74,7 @@ from products.signals.backend.scout_harness.team_limits import (
_team_configs,
background_sample_bucket,
max_enabled_scouts_for_team,
+ parse_background_lanes,
resolve_max_enabled_scouts,
)
from products.signals.backend.temporal.agentic.scout_coordinator import (
@@ -772,6 +775,63 @@ def test_parse_background(payload, expected):
assert _parse_background(payload) == expected
+_ANOMALY = "signals-scout-anomaly-detection"
+_LEGACY_LANE = BackgroundEnrollment(True, _GENERAL, frozenset({1}), None, DEFAULT_BACKGROUND_MAX_NEW_TEAMS_PER_TICK)
+_ANOMALY_LANE = BackgroundEnrollment(
+ True,
+ _ANOMALY,
+ frozenset({2}),
+ 360,
+ DEFAULT_BACKGROUND_MAX_NEW_TEAMS_PER_TICK,
+ audience=SignalScoutBackgroundBand.Audience.SET_UP,
+ require_precheck=True,
+)
+
+
+ "payload,expected",
+ [
+ ({}, None),
+ ({"background": {"enabled": True, "team_ids": [1]}}, (_LEGACY_LANE,)),
+ (
+ {
+ "background": {"enabled": True, "team_ids": [9]},
+ "background_lanes": [
+ {"skill_name": _GENERAL, "enabled": True, "team_ids": [1]},
+ {
+ "skill_name": _ANOMALY,
+ "enabled": True,
+ "audience": "set_up",
+ "team_ids": [2],
+ "interval_minutes": 360,
+ "require_precheck": True,
+ },
+ ],
+ },
+ (_LEGACY_LANE, _ANOMALY_LANE),
+ ),
+ ({"background_lanes": []}, ()),
+ # Each of these must not read as removed lanes, because a removed lane pauses its configs.
+ ({"background_lanes": {"skill_name": _GENERAL}}, None),
+ ({"background_lanes": [{"skill_name": _GENERAL, "team_ids": [1]}, {"team_ids": [2]}]}, None),
+ ({"background_lanes": [{"skill_name": _GENERAL, "team_ids": "nope"}]}, None),
+ ({"background_lanes": [{"skill_name": _GENERAL, "audience": "everyone"}]}, None),
+ ({"background_lanes": [{"skill_name": _GENERAL}, {"skill_name": _GENERAL}]}, None),
+ ],
+)
+def test_parse_background_lanes(payload, expected):
+ assert parse_background_lanes(payload) == expected
+
+
[email protected]("team_id", [1, 2, 12345])
+def test_background_sample_bucket_salt(team_id):
+ def bucket(salt: str) -> int:
+ return int.from_bytes(hashlib.sha256(salt.encode()).digest()[:8], "big") % 100
+
+ assert background_sample_bucket(team_id, _GENERAL) == bucket(f"signals-scout-background:{team_id}")
+ assert background_sample_bucket(team_id, _ANOMALY) == bucket(f"signals-scout-background:{_ANOMALY}:{team_id}")
+
+
@pytest.mark.parametrize(
"budget,runs_last_24h,expected",
[
@@ -798,6 +858,14 @@ def _background(
return BackgroundEnrollment(enabled, _GENERAL, frozenset(team_ids), 720, max_new, bands or {}, backoff, budget)
+def _lanes(*lanes: BackgroundEnrollment | None) -> tuple[BackgroundEnrollment, ...] | None:
+ return None if lanes == (None,) else tuple(lane for lane in lanes if lane is not None)
+
+
+def _backoffs(backoff: BackgroundBackoff | None) -> dict[str, BackgroundBackoff]:
+ return {_GENERAL: backoff} if backoff is not None else {}
+
+
_WEEK_MINUTES = 7 * 24 * 60
_BACKOFF = BackgroundBackoff(factor=2, max_interval_minutes=90 * 24 * 60)
@@ -814,7 +882,7 @@ class TestBackgroundEnrollment:
def test_creates_only_the_background_config_on_consented_teams(self):
approved, unapproved = self._team(), self._team(approved=False)
- _collect_planned_runs(_NO_ENROLLMENT, background=_background({approved.id, unapproved.id}))
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(_background({approved.id, unapproved.id})))
configs = list(SignalScoutConfig.all_teams.filter(team__in=[approved, unapproved]))
assert [(c.team_id, c.skill_name, c.managed_by, c.run_interval_minutes) for c in configs] == [
@@ -825,10 +893,10 @@ class TestBackgroundEnrollment:
teams = [self._team(), self._team()]
background = _background({t.id for t in teams}, max_new=1)
- _collect_planned_runs(_NO_ENROLLMENT, background=background)
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(background))
assert SignalScoutConfig.all_teams.filter(team__in=teams).count() == 1
- _collect_planned_runs(_NO_ENROLLMENT, background=background)
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(background))
assert SignalScoutConfig.all_teams.filter(team__in=teams).count() == 2
@pytest.mark.parametrize(
@@ -838,6 +906,7 @@ class TestBackgroundEnrollment:
(False, "enabled", False), # consent withdrawn after setup
(True, "disabled", False), # kill switch
(True, "absent", False), # explicit enrollment alone never runs a background config
+ (True, "requires_missing_precheck", False), # the general scout ships no default pre-check
],
)
def test_dispatch_is_gated_on_consent_and_the_block(self, approved, block, expect_dispatch):
@@ -845,12 +914,19 @@ class TestBackgroundEnrollment:
with team_scope(team.id, canonical=True):
_create_skill(team, _GENERAL)
_create_config(team, _GENERAL, managed_by=SignalScoutConfig.ManagedBy.BACKGROUND)
- background = None if block == "absent" else _background({team.id}, enabled=block == "enabled")
+ background = (
+ None
+ if block == "absent"
+ else dataclasses.replace(
+ _background({team.id}, enabled=block != "disabled"),
+ require_precheck=block == "requires_missing_precheck",
+ )
+ )
enrollment = Enrollment(wildcard=False, explicit={team.id}, skip=set())
with patch("products.signals.backend.temporal.agentic.scout_coordinator.register_missing_configs") as seed:
seed.return_value = {_GENERAL}
- planned = _collect_planned_runs(enrollment, background=background)
+ planned = _collect_planned_runs(enrollment, background_lanes=_lanes(background))
expected = [PlannedRun(team_id=team.id, skill_name=_GENERAL)] if expect_dispatch else []
assert planned == expected
@@ -861,7 +937,7 @@ class TestBackgroundEnrollment:
background_config = _create_config(team, _GENERAL, managed_by=SignalScoutConfig.ManagedBy.BACKGROUND)
user_config = _create_config(team, "signals-scout-errors")
- _collect_planned_runs(_NO_ENROLLMENT, background=_background(set(), enabled=False))
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(_background(set(), enabled=False)))
background_config.refresh_from_db()
user_config.refresh_from_db()
@@ -869,17 +945,68 @@ class TestBackgroundEnrollment:
assert background_config.pause_reason == SignalScoutConfig.PauseReason.BACKGROUND_REMOVED
assert user_config.status == SignalScoutConfig.Status.ACTIVE
- _collect_planned_runs(_NO_ENROLLMENT, background=_background({team.id}))
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(_background({team.id})))
background_config.refresh_from_db()
assert background_config.status == SignalScoutConfig.Status.ACTIVE
assert background_config.enabled is True
- def _banded_teams(self, band: int, count: int) -> list[Team]:
+ def test_each_lane_pauses_only_its_own_configs(self):
+ general_team, anomaly_team = self._team(), self._team()
+ with team_scope(general_team.id, canonical=True):
+ general_config = _create_config(general_team, _GENERAL, managed_by=SignalScoutConfig.ManagedBy.BACKGROUND)
+ with team_scope(anomaly_team.id, canonical=True):
+ anomaly_config = _create_config(anomaly_team, _ANOMALY, managed_by=SignalScoutConfig.ManagedBy.BACKGROUND)
+ general_lane = _background({general_team.id}, enabled=False)
+ anomaly_lane = dataclasses.replace(general_lane, skill_name=_ANOMALY, team_ids=frozenset({anomaly_team.id}))
+
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(general_lane, anomaly_lane))
+
+ general_config.refresh_from_db()
+ anomaly_config.refresh_from_db()
+ assert (general_config.status, anomaly_config.status) == (
+ SignalScoutConfig.Status.ACTIVE,
+ SignalScoutConfig.Status.ACTIVE,
+ )
+
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(general_lane))
+
+ general_config.refresh_from_db()
+ anomaly_config.refresh_from_db()
+ assert general_config.status == SignalScoutConfig.Status.ACTIVE
+ assert anomaly_config.status == SignalScoutConfig.Status.PAUSED_BY_SYSTEM
+ assert anomaly_config.pause_reason == SignalScoutConfig.PauseReason.BACKGROUND_REMOVED
+
+ @pytest.mark.parametrize("with_second_lane", [False, True])
+ def test_each_lane_samples_its_own_audience(self, with_second_lane):
+ (not_set_up_team,) = self._banded_teams(1, 1)
+ (set_up_team,) = self._banded_teams(1, 1, audience=SignalScoutBackgroundBand.Audience.SET_UP)
+ bands = {1: BackgroundBand(percent=100, interval_minutes=None)}
+ general_lane = _background(set(), bands=bands)
+ anomaly_lane = dataclasses.replace(
+ general_lane, skill_name=_ANOMALY, audience=SignalScoutBackgroundBand.Audience.SET_UP
+ )
+
+ lanes = _lanes(general_lane, anomaly_lane) if with_second_lane else _lanes(general_lane)
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=lanes)
+
+ created = set(
+ SignalScoutConfig.all_teams.filter(team__in=[not_set_up_team, set_up_team]).values_list(
+ "team_id", "skill_name"
+ )
+ )
+ expected = {(not_set_up_team.id, _GENERAL)}
+ if with_second_lane:
+ expected.add((set_up_team.id, _ANOMALY))
+ assert created == expected
+
+ def _banded_teams(
+ self, band: int, count: int, audience: str = SignalScoutBackgroundBand.Audience.NOT_SET_UP
+ ) -> list[Team]:
teams = [self._team() for _ in range(count)]
now = timezone.now()
for team in teams:
- SignalScoutBackgroundBand.all_teams.create(team=team, band=band, computed_at=now)
+ SignalScoutBackgroundBand.all_teams.create(team=team, band=band, audience=audience, computed_at=now)
return teams
def test_band_sampling_keeps_earlier_picks_as_the_percent_rises(self):
@@ -888,7 +1015,7 @@ class TestBackgroundEnrollment:
def sampled(percent: int) -> set[int]:
bands = {2: BackgroundBand(percent=percent, interval_minutes=20160)}
- _collect_planned_runs(_NO_ENROLLMENT, background=_background(set(), max_new=100, bands=bands))
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(_background(set(), max_new=100, bands=bands)))
return set(
SignalScoutConfig.all_teams.filter(
team__in=teams, status=SignalScoutConfig.Status.ACTIVE, skill_name=_GENERAL
@@ -918,7 +1045,7 @@ class TestBackgroundEnrollment:
)
bands = {1: BackgroundBand(percent=0, interval_minutes=None)}
- _collect_planned_runs(_NO_ENROLLMENT, background=_background({hand_picked.id}, bands=bands))
+ _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(_background({hand_picked.id}, bands=bands)))
for config in (banded_config, taken_over_config, hand_picked_config):
config.refresh_from_db()
@@ -943,8 +1070,8 @@ class TestBackgroundEnrollment:
self, team: Team, config: SignalScoutConfig, background: BackgroundEnrollment, at: datetime, *, runs: bool
) -> bool:
with time_machine.travel(at, tick=False):
- planned = _collect_planned_runs(_NO_ENROLLMENT, background=background)
- _stamp_dispatched_runs(planned, dispatched_at=at, backoff=background.backoff)
+ planned = _collect_planned_runs(_NO_ENROLLMENT, background_lanes=_lanes(background))
+ _stamp_dispatched_runs(planned, dispatched_at=at, backoffs=_backoffs(background.backoff))
dispatched = PlannedRun(team_id=team.id, skill_name=_GENERAL) in planned
if dispatched and runs:
self._record_run(team, config, at + timedelta(minutes=1))
@@ -1008,7 +1135,7 @@ class TestBackgroundEnrollment:
patch("products.signals.backend.temporal.agentic.scout_coordinator.posthoganalytics.capture") as capture,
):
planned = _collect_planned_runs(
- Enrollment(wildcard=False, explicit={person_team.id}, skip=set()), background=background
+ Enrollment(wildcard=False, explicit={person_team.id}, skip=set()), background_lanes=_lanes(background)
)
assert [
@@ -1022,8 +1149,10 @@ class TestBackgroundEnrollment:
if c.kwargs["event"] == "signals_scout_background_tick"
]
assert {
- key: properties[key] for key in ("due", "dispatched", "deferred", "runs_last_24h", "never_run_due")
+ key: properties[key]
+ for key in ("skill_name", "due", "dispatched", "deferred", "runs_last_24h", "never_run_due")
} == {
+ "skill_name": _GENERAL,
"due": 3,
"dispatched": sum(expected),
"deferred": 3 - sum(expected),
@@ -1093,8 +1222,10 @@ class TestBackgroundEnrollment:
return_value={_GENERAL},
),
):
- planned = _collect_planned_runs(enrollment, background=_background({team.id}, backoff=backoff))
- _stamp_dispatched_runs(planned, dispatched_at=at, backoff=backoff)
+ planned = _collect_planned_runs(
+ enrollment, background_lanes=_lanes(_background({team.id}, backoff=backoff))
+ )
+ _stamp_dispatched_runs(planned, dispatched_at=at, backoffs=_backoffs(backoff))
assert (PlannedRun(team_id=team.id, skill_name=_GENERAL) in planned) is expect_dispatch
config.refresh_from_db()
@@ -1118,10 +1249,10 @@ class TestBackgroundEnrollment:
),
pytest.raises(RuntimeError),
):
- _stamp_dispatched_runs(planned, dispatched_at=at, backoff=_BACKOFF)
+ _stamp_dispatched_runs(planned, dispatched_at=at, backoffs=_backoffs(_BACKOFF))
else:
- _stamp_dispatched_runs(planned, dispatched_at=at, backoff=_BACKOFF)
- _stamp_dispatched_runs(planned, dispatched_at=at, backoff=_BACKOFF)
+ _stamp_dispatched_runs(planned, dispatched_at=at, backoffs=_backoffs(_BACKOFF))
+ _stamp_dispatched_runs(planned, dispatched_at=at, backoffs=_backoffs(_BACKOFF))
config.refresh_from_db()
assert config.background_backoff_level == 1
diff --git a/products/signals/backend/test/test_scout_precheck.py b/products/signals/backend/test/test_scout_precheck.py
index 6efa21110599..43d994fcfb35 100644
--- a/products/signals/backend/test/test_scout_precheck.py
+++ b/products/signals/backend/test/test_scout_precheck.py
@@ -324,11 +324,28 @@ class TestSkillDefaultPrecheck(ClickhouseTestMixin, BaseTest):
assert result is not None
assert (result.outcome, result.query_source) == (outcome, "skill_default")
- def test_project_outside_the_rollout_runs_as_before(self) -> None:
- result, capture = self._evaluate(rollout_percent=0)
+ @parameterized.expand(
+ [
+ ("team_managed", SignalScoutConfig.ManagedBy.TEAM, True, False),
+ ("background_lane_without_require_precheck", SignalScoutConfig.ManagedBy.BACKGROUND, False, False),
+ ("background_lane_with_require_precheck", SignalScoutConfig.ManagedBy.BACKGROUND, True, True),
+ ]
+ )
+ def test_project_outside_the_rollout(self, _name, managed_by, require_precheck, expect_gated) -> None:
+ SignalScoutConfig.all_teams.filter(team=self.team).update(managed_by=managed_by)
+ lanes = [{"skill_name": SURVEYS_SKILL, "require_precheck": require_precheck}]
+ with patch(
+ "products.signals.backend.scout_harness.team_limits._read_flag_payload",
+ return_value={"background_lanes": lanes},
+ ):
+ result, capture = self._evaluate(rollout_percent=0)
- assert result is None
- capture.assert_not_called()
+ if expect_gated:
+ assert result is not None
+ assert (result.outcome, result.query_source) == ("skip", "skill_default")
+ else:
+ assert result is None
+ capture.assert_not_called()
# Non-atomic: ClickHouse reads `system.*` tables over its own Postgres connection, so rows must be committed.