:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aeo): run the full prompt set daily with parallel checks (#122437)
Lucas Ricoy9b9a9869af91
diff --git a/posthog/settings/web.py b/posthog/settings/web.pyindex fd90723bdcca..4f3f5e1f03d3 100644--- a/posthog/settings/web.py+++ b/posthog/settings/web.py@@ -1471,6 +1471,9 @@ AEO_TARGET_DOMAINS = get_list(get_from_env("AEO_TARGET_DOMAINS", "posthog.com"))AEO_BRAND_SENTIMENT_ENABLED = get_from_env("AEO_BRAND_SENTIMENT_ENABLED", False, type_cast=str_to_bool)AEO_ANTHROPIC_MODEL = get_from_env("AEO_ANTHROPIC_MODEL", "claude-sonnet-5")AEO_OPENAI_MODEL = get_from_env("AEO_OPENAI_MODEL", "gpt-5")+# A full daily run covers the whole prompt set, so the cap only guards against a runaway import.+AEO_MAX_PROMPTS_PER_RUN = get_from_env("AEO_MAX_PROMPTS_PER_RUN", 2000, type_cast=int)+AEO_RUN_CONCURRENCY = get_from_env("AEO_RUN_CONCURRENCY", 16, type_cast=int)EXA_API_KEY = get_from_env("EXA_API_KEY", "")CONTENT_AUTOPILOT_MODEL = get_from_env("CONTENT_AUTOPILOT_MODEL", "claude-sonnet-5")CONTENT_AUTOPILOT_SAFETY_MODEL = get_from_env("CONTENT_AUTOPILOT_SAFETY_MODEL", "claude-haiku-4-5")diff --git a/products/aeo/README.md b/products/aeo/README.mdindex 371976408e08..f8d8e72b9a70 100644--- a/products/aeo/README.md+++ b/products/aeo/README.md@@ -67,7 +67,9 @@ A prompt keeps its id, so its checks, stats and detail view stay in one place. I### Which prompts a run takes-A run takes at most 50 prompts. It takes one prompt per topic in turn, so every topic is sampled on each run. Within a topic, prompts never checked come first, then the least recently checked, so a set larger than one run rotates.+A run takes the whole prompt set, up to `AEO_MAX_PROMPTS_PER_RUN` (default 2,000), and checks `AEO_RUN_CONCURRENCY` prompts at a time (default 16).+It takes one prompt per topic in turn, so a capped run still samples every topic.+Within a topic, prompts never checked come first, then the least recently checked, so a set larger than one run rotates, and a run cut short by a worker restart resumes on the next dispatch.## The table@@ -115,7 +117,9 @@ Web analytics' Content autopilot (`/web/content-autopilot`, flag `web-analytics-## Cost-Roughly \$2–4/day at 50 prompts × 3 engines × 1 run/day: provider web-search fees + tokens (attributed as `$ai_web_search_cost_usd` and `$ai_total_cost_usd` on the `$ai_generation` event) plus Exa at \$5 per 1,000 requests (`cost_usd` on the check event).+Measured in October 2026: about \$0.06 per OpenAI check and \$0.14 per Claude check, provider web-search fees and tokens included (`$ai_web_search_cost_usd` and `$ai_total_cost_usd` on the `$ai_generation` event).+A daily run of 1,000 prompts on both engines is about \$200.+Exa adds \$5 per 1,000 requests (`cost_usd` on the check event).## Out of scopediff --git a/products/aeo/backend/engines.py b/products/aeo/backend/engines.pyindex db345841f8d2..30392e30886a 100644--- a/products/aeo/backend/engines.py+++ b/products/aeo/backend/engines.py@@ -13,6 +13,8 @@ joined back to the prompt via `aeo_prompt_id` / `aeo_run_id`.from __future__ import annotations+import time+import threadingfrom dataclasses import fieldfrom typing import Any, Protocolfrom urllib.parse import urlparse@@ -97,12 +99,68 @@ class CitationEngine(Protocol):def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: ...-def _session() -> requests.Session:+class _ThreadSessions:+ """One requests.Session per thread. A run calls engines from a thread pool, and+ requests does not document Session as safe to share across threads."""++ def __init__(self, *, trust_env: bool) -> None:+ self._local = threading.local()+ self._trust_env = trust_env++ def get(self) -> requests.Session:+ session: requests.Session | None = getattr(self._local, "session", None)+ if session is None:+ session = requests.Session()+ session.trust_env = self._trust_env+ self._local.session = session+ return session+++def _gateway_sessions() -> _ThreadSessions:# trust_env=False keeps in-cluster gateway calls off any egress proxy, and- # the session reuses connections across a run's many sequential calls.- session = requests.Session()- session.trust_env = False- return session+ # each thread's session reuses connections across its calls.+ return _ThreadSessions(trust_env=False)+++# Throttling and transient upstream errors. A full run sends many calls in parallel, so+# these get a short bounded retry instead of becoming a failed check straight away.+RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})+GATEWAY_MAX_ATTEMPTS = 3+GATEWAY_RETRY_BASE_SECONDS = 5+GATEWAY_RETRY_MAX_SECONDS = 60+++def _retry_delay_seconds(response: requests.Response, attempt: int) -> float:+ retry_after = response.headers.get("Retry-After", "")+ if retry_after.isdigit():+ return min(float(retry_after), GATEWAY_RETRY_MAX_SECONDS)+ return min(GATEWAY_RETRY_BASE_SECONDS * 2**attempt, GATEWAY_RETRY_MAX_SECONDS)+++class _SharedBackoff:+ """A pause every worker thread honours before its next gateway request. One throttled+ response pauses the whole run, so the other workers stop adding load while it backs off."""++ def __init__(self) -> None:+ self._lock = threading.Lock()+ self._resume_at = 0.0++ def pause(self, seconds: float) -> None:+ with self._lock:+ # A shorter pause from another worker must not cut a longer one short.+ self._resume_at = max(self._resume_at, time.monotonic() + seconds)++ def wait(self) -> None:+ # Another worker can extend the pause during the sleep, so read the deadline again after each one.+ while True:+ with self._lock:+ remaining = self._resume_at - time.monotonic()+ if remaining <= 0:+ return+ time.sleep(remaining)+++GATEWAY_BACKOFF = _SharedBackoff()def gateway_post_json(@@ -112,10 +170,17 @@ def gateway_post_json(payload: dict[str, Any],timeout: int = GATEWAY_TIMEOUT_SECONDS,) -> dict[str, Any]:- """POST and decode JSON; raises requests.RequestException on any failure."""- response = session.post(url, headers=headers, json=payload, timeout=timeout)- response.raise_for_status()- return response.json()+ """POST and decode JSON, retrying throttled and transient errors a bounded number of+ times; raises requests.RequestException once the attempts run out or on any other failure."""+ for attempt in range(GATEWAY_MAX_ATTEMPTS):+ GATEWAY_BACKOFF.wait()+ response = session.post(url, headers=headers, json=payload, timeout=timeout)+ if response.status_code in RETRYABLE_STATUS_CODES and attempt < GATEWAY_MAX_ATTEMPTS - 1:+ GATEWAY_BACKOFF.pause(_retry_delay_seconds(response, attempt))+ continue+ response.raise_for_status()+ return response.json()+ raise AssertionError("unreachable: the last attempt returns or raises")def _dedupe_urls(urls: list[str]) -> list[str]:@@ -349,7 +414,7 @@ class ClaudeWebSearchEngine:if gateway is None:raise ValueError("AI gateway is not configured (AI_GATEWAY_URL / AI_GATEWAY_API_KEY)")self._gateway = gateway- self._session = _session()+ self._sessions = _gateway_sessions()def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:payload = {@@ -375,7 +440,9 @@ class ClaudeWebSearchEngine:**(ai_gateway_headers(trace_id=trace_id, properties=custom_properties) or {}),}try:- body = gateway_post_json(self._session, self._gateway.url.rstrip("/") + "/messages", headers, payload)+ body = gateway_post_json(+ self._sessions.get(), self._gateway.url.rstrip("/") + "/messages", headers, payload+ )except requests.RequestException as e:return CitationCheck(engine=self.name, model=self.model, trace_id=trace_id, error=_request_error(e))parsed = parse_anthropic_citations(body)@@ -403,7 +470,7 @@ class OpenAIWebSearchEngine:if gateway is None:raise ValueError("AI gateway is not configured (AI_GATEWAY_URL / AI_GATEWAY_API_KEY)")self._gateway = gateway- self._session = _session()+ self._sessions = _gateway_sessions()def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:payload = {@@ -417,7 +484,9 @@ class OpenAIWebSearchEngine:**(ai_gateway_headers(trace_id=trace_id, properties=custom_properties) or {}),}try:- body = gateway_post_json(self._session, self._gateway.url.rstrip("/") + "/responses", headers, payload)+ body = gateway_post_json(+ self._sessions.get(), self._gateway.url.rstrip("/") + "/responses", headers, payload+ )except requests.RequestException as e:return CitationCheck(engine=self.name, model=self.model, trace_id=trace_id, error=_request_error(e))if (error := openai_response_error(body)) is not None:@@ -446,13 +515,13 @@ class ExaAnswerEngine:model = "exa-answer"def __init__(self) -> None:- self._session = requests.Session()+ self._sessions = _ThreadSessions(trust_env=True)def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:headers = {"x-api-key": settings.EXA_API_KEY}try:body = gateway_post_json(- self._session,+ self._sessions.get(),EXA_ANSWER_URL,headers,{"query": prompt, "text": False},diff --git a/products/aeo/backend/management/commands/run_aeo_citation_checks.py b/products/aeo/backend/management/commands/run_aeo_citation_checks.pyindex 39b803d44a15..17b53e44245b 100644--- a/products/aeo/backend/management/commands/run_aeo_citation_checks.py+++ b/products/aeo/backend/management/commands/run_aeo_citation_checks.py@@ -25,7 +25,7 @@ class Command(BaseCommand):def add_arguments(self, parser: CommandParser) -> None:parser.add_argument("--team-id", type=int, required=True)- parser.add_argument("--limit", type=int, help="Max prompts this run (default 50).")+ parser.add_argument("--limit", type=int, help="Max prompts this run (default AEO_MAX_PROMPTS_PER_RUN).")parser.add_argument("--engines",type=str,diff --git a/products/aeo/backend/runner.py b/products/aeo/backend/runner.pyindex f9cfeb74e416..c62462bc3d12 100644--- a/products/aeo/backend/runner.py+++ b/products/aeo/backend/runner.py@@ -15,6 +15,7 @@ gateway-key owner's project, not the checked team's, and carries thefrom __future__ import annotationsimport uuid+from concurrent.futures import ThreadPoolExecutor, as_completedfrom typing import Any, Optionalfrom django.conf import settings@@ -24,6 +25,7 @@ from django.utils import timezoneimport structlog+from posthog.dataclasses import frozenfrom posthog.models.scoping import team_scopefrom posthog.models.team import Team@@ -34,8 +36,6 @@ from products.aeo.backend.temporal.brand_sentiment import start_brand_sentiment_logger = structlog.get_logger(__name__)-DEFAULT_MAX_PROMPTS_PER_RUN = 50-def run_citation_checks(team: Team,@@ -55,7 +55,7 @@ def run_citation_checks(logger.warning("aeo_citation_run_no_engines", team_id=team.id)return _empty_summary(team, "no engines configured (AI_GATEWAY_URL/AI_GATEWAY_API_KEY, EXA_API_KEY)"), []- prompts = select_prompts(team, DEFAULT_MAX_PROMPTS_PER_RUN if limit is None else limit)+ prompts = select_prompts(team, settings.AEO_MAX_PROMPTS_PER_RUN if limit is None else limit)if not prompts:logger.warning("aeo_citation_run_no_prompts", team_id=team.id)return _empty_summary(team, "no active prompts — run `seed_aeo_prompts` first"), []@@ -69,59 +69,39 @@ def run_citation_checks(rows_written = 0write_failures = 0- for prompt in prompts:- prompt_rows: list[AEOCitationCheck] = []- for engine in engines:- trace_id = str(uuid.uuid4())- check = engine.run(- prompt.prompt,- trace_id=trace_id,- custom_properties={- "aeo_prompt_id": str(prompt.id),- "aeo_run_id": run_id,- "aeo_prompt_source": prompt.prompt_source,- # No $ai_ prefix, so the gateway keeps this on the $ai_generation- # event, which lets usage attribute spend back to the checked team.- "team_id": str(team.id),- },- )- if check.error is not None:- engine_failures += 1- logger.warning(- "aeo_citation_check_failed",- team_id=team.id,- engine=engine.name,- prompt_id=str(prompt.id),- error=check.error,- )- fields = {- **build_check_fields(- check=check,- run_id=run_id,- prompt_id=str(prompt.id),- prompt_text=prompt.prompt,- prompt_source=prompt.prompt_source,- prompt_hash=prompt.prompt_hash,- target_domains=target_domains,- ),- "prompt_topic": prompt.topic,- "prompt_version": prompt.current_version,- }- prompt_rows.append(AEOCitationCheck(team_id=canonical_team_id, **fields))- checks.append(fields)-- if record and prompt_rows:- # Write per prompt so a mid-run crash keeps the completed checks.+ # Engine calls are network-bound and run in worker threads. Database writes stay on+ # this thread, so the workers never open their own Django connections.+ pool = ThreadPoolExecutor(max_workers=max(1, settings.AEO_RUN_CONCURRENCY))+ futures = {pool.submit(_check_prompt, team, run_id, engines, target_domains, prompt): prompt for prompt in prompts}+ try:+ for done, future in enumerate(as_completed(futures), start=1):try:- # The fail-closed manager refuses writes without a team scope, and none of the callers- # (the Celery task, the management command) run inside one.- with team_scope(canonical_team_id):- AEOCitationCheck.objects.bulk_create(prompt_rows)- rows_written += len(prompt_rows)- AEOPrompt.objects.for_team(team.id).filter(id=prompt.id).update(last_checked_at=timezone.now())+ result = future.result()except Exception as e:- write_failures += len(prompt_rows)- logger.exception("aeo_citation_write_failed", team_id=team.id, run_id=run_id, error=str(e)[:300])+ # One prompt's unexpected engine error must not end a run that takes hours.+ engine_failures += len(engines)+ logger.exception(+ "aeo_citation_prompt_crashed",+ team_id=team.id,+ prompt_id=str(futures[future].id),+ error=str(e)[:300],+ )+ continue+ engine_failures += result.failures+ checks.extend(result.fields_per_engine)+ if record and result.fields_per_engine:+ if _record_prompt_checks(team, canonical_team_id, run_id, result.prompt, result.fields_per_engine):+ rows_written += len(result.fields_per_engine)+ else:+ write_failures += len(result.fields_per_engine)+ if done % 100 == 0:+ logger.info("aeo_citation_run_progress", team_id=team.id, run_id=run_id, done=done, total=len(prompts))+ except BaseException:+ # A Celery time limit or a worker shutdown lands here. Drop the queued prompts so they+ # make no more paid calls whose results nobody records.+ pool.shutdown(wait=False, cancel_futures=True)+ raise+ pool.shutdown(wait=True)summary = CitationRunSummary(team_id=team.id,@@ -150,6 +130,82 @@ def run_citation_checks(return summary, checks+@frozen+class PromptCheckResult:+ """One prompt's checks across every engine, before they are written."""++ prompt: AEOPrompt+ fields_per_engine: list[dict[str, Any]]+ failures: int+++def _check_prompt(+ team: Team, run_id: str, engines: list[CitationEngine], target_domains: list[str], prompt: AEOPrompt+) -> PromptCheckResult:+ """Run one prompt on every engine. Runs in a worker thread, so it must not touch the database."""+ fields_per_engine: list[dict[str, Any]] = []+ failures = 0+ for engine in engines:+ trace_id = str(uuid.uuid4())+ check = engine.run(+ prompt.prompt,+ trace_id=trace_id,+ custom_properties={+ "aeo_prompt_id": str(prompt.id),+ "aeo_run_id": run_id,+ "aeo_prompt_source": prompt.prompt_source,+ # No $ai_ prefix, so the gateway keeps this on the $ai_generation+ # event, which lets usage attribute spend back to the checked team.+ "team_id": str(team.id),+ },+ )+ if check.error is not None:+ failures += 1+ logger.warning(+ "aeo_citation_check_failed",+ team_id=team.id,+ engine=engine.name,+ prompt_id=str(prompt.id),+ error=check.error,+ )+ fields_per_engine.append(+ {+ **build_check_fields(+ check=check,+ run_id=run_id,+ prompt_id=str(prompt.id),+ prompt_text=prompt.prompt,+ prompt_source=prompt.prompt_source,+ prompt_hash=prompt.prompt_hash,+ target_domains=target_domains,+ ),+ "prompt_topic": prompt.topic,+ "prompt_version": prompt.current_version,+ }+ )+ return PromptCheckResult(prompt=prompt, fields_per_engine=fields_per_engine, failures=failures)+++def _record_prompt_checks(+ team: Team, canonical_team_id: int, run_id: str, prompt: AEOPrompt, fields_per_engine: list[dict[str, Any]]+) -> bool:+ """Write one prompt's checks and mark the prompt checked. Called per prompt so a+ mid-run crash keeps the completed checks; False when the write failed."""+ prompt_rows = [AEOCitationCheck(team_id=canonical_team_id, **fields) for fields in fields_per_engine]+ try:+ # The fail-closed manager refuses writes without a team scope, and none of the callers+ # (the Celery task, the management command) run inside one.+ with team_scope(canonical_team_id):+ AEOCitationCheck.objects.bulk_create(prompt_rows)+ # A prompt every engine failed on was not really checked, so leave it first in line for the next run.+ if any(not fields["check_failed"] for fields in fields_per_engine):+ AEOPrompt.objects.for_team(team.id).filter(id=prompt.id).update(last_checked_at=timezone.now())+ except Exception as e:+ logger.exception("aeo_citation_write_failed", team_id=team.id, run_id=run_id, error=str(e)[:300])+ return False+ return True++def select_prompts(team: Team, limit: int) -> list[AEOPrompt]:"""Pick up to `limit` active, unarchived prompts, alternating between topics.diff --git a/products/aeo/backend/tasks/tasks.py b/products/aeo/backend/tasks/tasks.pyindex 1f2a69198528..50348a8f5298 100644--- a/products/aeo/backend/tasks/tasks.py+++ b/products/aeo/backend/tasks/tasks.py@@ -25,10 +25,11 @@ from products.aeo.backend.runner import run_citation_checkslogger = structlog.get_logger(__name__)-# A full run is up to 50 prompts x 3 engines of sequential web-search calls-# (typically 10-60s each); four hours gives slow days headroom without letting-# a hung run hold a worker slot indefinitely.-RUN_SOFT_TIME_LIMIT_SECONDS = 4 * 60 * 60+# A full run covers the whole prompt set: each prompt makes one web-search call per+# engine (about a minute each), AEO_RUN_CONCURRENCY prompts at a time. Ten hours+# leaves headroom for a slow day and still ends a hung run well before the next+# daily dispatch.+RUN_SOFT_TIME_LIMIT_SECONDS = 10 * 60 * 60RUN_TIME_LIMIT_SECONDS = RUN_SOFT_TIME_LIMIT_SECONDS + 300diff --git a/products/aeo/backend/test/test_engines.py b/products/aeo/backend/test/test_engines.pyindex 8db3f963aa25..fc8bfed13e4e 100644--- a/products/aeo/backend/test/test_engines.py+++ b/products/aeo/backend/test/test_engines.py@@ -1,16 +1,23 @@+from collections.abc import Callable, Iterator+from contextlib import contextmanagerfrom types import SimpleNamespace-from unittest.mock import patch+import pytest+from unittest.mock import MagicMock, patch+import requestsfrom parameterized import parameterizedfrom products.aeo.backend.engines import (+ GATEWAY_MAX_ATTEMPTS,OPENAI_MAX_OUTPUT_TOKENS,CitationCheck,ClaudeWebSearchEngine,OpenAIWebSearchEngine,+ _SharedBackoff,anthropic_truncated_error,build_check_fields,+ gateway_post_json,is_target_url,openai_response_error,parse_anthropic_citations,@@ -350,3 +357,78 @@ def test_engine_derived_text_is_sanitized_before_it_reaches_the_event() -> None:# Sanitizing must not move the verdict: it runs on the recorded copy only.assert fields["cited"] is Trueassert fields["target_best_position"] == 1+++def _response(status: int, retry_after: str | None = None) -> requests.Response:+ response = requests.Response()+ response.status_code = status+ response._content = b'{"ok": true}'+ if retry_after is not None:+ response.headers["Retry-After"] = retry_after+ return response+++ [+ ("throttled_then_ok", [_response(429, retry_after="7"), _response(200)], None, 2, [7.0]),+ ("transient_until_attempts_run_out", [_response(503)] * GATEWAY_MAX_ATTEMPTS, requests.HTTPError, 3, [5, 10]),+ ("client_error_is_not_retried", [_response(400)], requests.HTTPError, 1, []),+ ]+)+def test_gateway_post_json_retries_throttling_and_transient_errors(+ _name: str, responses: list, raises: type[Exception] | None, posts: int, sleeps: list[float]+) -> None:+ session = MagicMock(spec=requests.Session)+ session.post.side_effect = responses++ with (+ patch("products.aeo.backend.engines.GATEWAY_BACKOFF", _SharedBackoff()),+ _fake_clock() as sleep,+ ):+ if raises is None:+ assert gateway_post_json(session, "https://gw.test", {}, {}) == {"ok": True}+ else:+ with pytest.raises(raises):+ gateway_post_json(session, "https://gw.test", {}, {})++ assert session.post.call_count == posts+ assert [call.args[0] for call in sleep.call_args_list] == sleeps+++ [+ ("shorter_pause_does_not_cut_a_longer_one", [30, 5], None, [30.0]),+ ("pause_extended_during_the_sleep", [30], 60, [30.0, 30.0]),+ ]+)+def test_shared_backoff_waits_for_the_latest_deadline(+ _name: str, pauses: list[float], extend_during_sleep: float | None, sleeps: list[float]+) -> None:+ backoff = _SharedBackoff()++ def other_worker_pauses() -> None:+ if extend_during_sleep is not None:+ backoff.pause(extend_during_sleep)++ with _fake_clock(on_first_sleep=other_worker_pauses) as sleep:+ for seconds in pauses:+ backoff.pause(seconds)+ backoff.wait()++ assert [call.args[0] for call in sleep.call_args_list] == sleeps+++@contextmanager+def _fake_clock(on_first_sleep: Callable[[], None] | None = None) -> Iterator[MagicMock]:+ now = [100.0]++ def sleep(seconds: float) -> None:+ if on_first_sleep is not None and sleep_mock.call_count == 1:+ on_first_sleep()+ now[0] += seconds++ with (+ patch("products.aeo.backend.engines.time.monotonic", side_effect=lambda: now[0]),+ patch("products.aeo.backend.engines.time.sleep", side_effect=sleep) as sleep_mock,+ ):+ yield sleep_mockdiff --git a/products/aeo/backend/test/test_runner.py b/products/aeo/backend/test/test_runner.pyindex 171554fb8fcc..0218de612a9f 100644--- a/products/aeo/backend/test/test_runner.py+++ b/products/aeo/backend/test/test_runner.py@@ -1,4 +1,6 @@import datetime as dt+import threading+from typing import NoReturnimport pytest@@ -7,6 +9,7 @@ from django.utils import timezonefrom posthog.models.organization import Organizationfrom posthog.models.team import Team+from products.aeo.backend import runnerfrom products.aeo.backend.engines import CitationCheckfrom products.aeo.backend.models import AEOCitationCheck, AEOPromptfrom products.aeo.backend.runner import run_citation_checks, select_prompts@@ -60,8 +63,12 @@ class _StubEngine:@pytest.mark.parametrize("via_child_environment", [False, True])-def test_run_records_checks_and_rotates_without_a_team_scope(team: Team, settings, via_child_environment: bool) -> None:[email protected]("concurrency", [1, 4])+def test_run_records_checks_and_rotates_without_a_team_scope(+ team: Team, settings, via_child_environment: bool, concurrency: int+) -> None:settings.AEO_TARGET_DOMAINS = ["posthog.com"]+ settings.AEO_RUN_CONCURRENCY = concurrencyrunner_team = (Team.objects.create(organization=team.organization, name="child env", parent_team=team)if via_child_environment@@ -82,3 +89,79 @@ def test_run_records_checks_and_rotates_without_a_team_scope(team: Team, settingassert {c.prompt_topic for c in checks} == {"t0", "t1"}assert len({c.prompt_id for c in checks}) == 4assert sorted((c.prompt_text, c.prompt_version) for c in checks) == [("q0?", 3), ("q1?", 1), ("q2?", 1), ("q3?", 1)]+++class _CrashingEngine(_StubEngine):+ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:+ if prompt == "q1?":+ raise RuntimeError("unexpected engine bug")+ return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties)+++def test_one_prompt_crashing_does_not_end_the_run(team: Team, settings) -> None:+ settings.AEO_RUN_CONCURRENCY = 2+ settings.AEO_MAX_PROMPTS_PER_RUN = 3+ upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(3)])++ summary, _ = run_citation_checks(team, engines=[_CrashingEngine()])++ assert (summary.prompts, summary.rows_written, summary.engine_failures) == (3, 2, 1)+ assert sorted(AEOCitationCheck.objects.for_team(team.id).values_list("prompt_text", flat=True)) == ["q0?", "q2?"]+++class _FailingEngine(_StubEngine):+ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:+ if prompt == "q1?":+ return CitationCheck(engine=self.name, model=self.model, error="HTTPError: status=429")+ return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties)+++def test_a_prompt_every_engine_failed_on_stays_unchecked(team: Team, settings) -> None:+ settings.AEO_RUN_CONCURRENCY = 2+ upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(2)])++ summary, _ = run_citation_checks(team, engines=[_FailingEngine()])++ assert (summary.rows_written, summary.engine_failures) == (2, 1)+ last_checked = dict(AEOPrompt.objects.for_team(team.id).values_list("prompt", "last_checked_at"))+ assert last_checked["q0?"] is not None+ assert last_checked["q1?"] is None+++class _BlockingEngine(_StubEngine):+ """Holds every check until the test releases it, so the test decides which prompts are in flight."""++ def __init__(self, workers: int) -> None:+ self.prompts: list[str] = []+ self.threads: set[threading.Thread] = set()+ self.all_workers_busy = threading.Barrier(workers + 1)+ self.release = threading.Event()++ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:+ self.prompts.append(prompt)+ self.threads.add(threading.current_thread())+ self.all_workers_busy.wait(timeout=5)+ self.release.wait(timeout=5)+ return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties)+++def test_an_interrupted_run_does_not_start_the_queued_prompts(team: Team, settings, monkeypatch) -> None:+ settings.AEO_RUN_CONCURRENCY = 2+ upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(5)])+ engine = _BlockingEngine(workers=2)++ def interrupt_once_every_worker_is_busy(futures: object) -> NoReturn:+ # Stands in for a Celery soft time limit landing while the run waits on its workers.+ engine.all_workers_busy.wait(timeout=5)+ raise KeyboardInterrupt++ monkeypatch.setattr(runner, "as_completed", interrupt_once_every_worker_is_busy)++ with pytest.raises(KeyboardInterrupt):+ run_citation_checks(team, engines=[engine])+ engine.release.set()+ for thread in engine.threads:+ thread.join(timeout=5)++ assert len(engine.prompts) == 2+ assert len(set(engine.prompts)) == 2