Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aeo): run the full prompt set daily with parallel checks (#122437)

Lucas Ricoy9b9a9869af91
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index fd90723bdcca..4f3f5e1f03d3 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -1471,6 +1471,9 @@ AEO_TARGET_DOMAINS = get_list(get_from_env("AEO_TARGET_DOMAINS", "posthog.com"))
AEO_BRAND_SENTIMENT_ENABLED = get_from_env("AEO_BRAND_SENTIMENT_ENABLED", False, type_cast=str_to_bool)
AEO_ANTHROPIC_MODEL = get_from_env("AEO_ANTHROPIC_MODEL", "claude-sonnet-5")
AEO_OPENAI_MODEL = get_from_env("AEO_OPENAI_MODEL", "gpt-5")
+# A full daily run covers the whole prompt set, so the cap only guards against a runaway import.
+AEO_MAX_PROMPTS_PER_RUN = get_from_env("AEO_MAX_PROMPTS_PER_RUN", 2000, type_cast=int)
+AEO_RUN_CONCURRENCY = get_from_env("AEO_RUN_CONCURRENCY", 16, type_cast=int)
EXA_API_KEY = get_from_env("EXA_API_KEY", "")
CONTENT_AUTOPILOT_MODEL = get_from_env("CONTENT_AUTOPILOT_MODEL", "claude-sonnet-5")
CONTENT_AUTOPILOT_SAFETY_MODEL = get_from_env("CONTENT_AUTOPILOT_SAFETY_MODEL", "claude-haiku-4-5")
diff --git a/products/aeo/README.md b/products/aeo/README.md
index 371976408e08..f8d8e72b9a70 100644
--- a/products/aeo/README.md
+++ b/products/aeo/README.md
@@ -67,7 +67,9 @@ A prompt keeps its id, so its checks, stats and detail view stay in one place. I
### Which prompts a run takes
-A run takes at most 50 prompts. It takes one prompt per topic in turn, so every topic is sampled on each run. Within a topic, prompts never checked come first, then the least recently checked, so a set larger than one run rotates.
+A run takes the whole prompt set, up to `AEO_MAX_PROMPTS_PER_RUN` (default 2,000), and checks `AEO_RUN_CONCURRENCY` prompts at a time (default 16).
+It takes one prompt per topic in turn, so a capped run still samples every topic.
+Within a topic, prompts never checked come first, then the least recently checked, so a set larger than one run rotates, and a run cut short by a worker restart resumes on the next dispatch.
## The table
@@ -115,7 +117,9 @@ Web analytics' Content autopilot (`/web/content-autopilot`, flag `web-analytics-
## Cost
-Roughly \$2–4/day at 50 prompts × 3 engines × 1 run/day: provider web-search fees + tokens (attributed as `$ai_web_search_cost_usd` and `$ai_total_cost_usd` on the `$ai_generation` event) plus Exa at \$5 per 1,000 requests (`cost_usd` on the check event).
+Measured in October 2026: about \$0.06 per OpenAI check and \$0.14 per Claude check, provider web-search fees and tokens included (`$ai_web_search_cost_usd` and `$ai_total_cost_usd` on the `$ai_generation` event).
+A daily run of 1,000 prompts on both engines is about \$200.
+Exa adds \$5 per 1,000 requests (`cost_usd` on the check event).
## Out of scope
diff --git a/products/aeo/backend/engines.py b/products/aeo/backend/engines.py
index db345841f8d2..30392e30886a 100644
--- a/products/aeo/backend/engines.py
+++ b/products/aeo/backend/engines.py
@@ -13,6 +13,8 @@ joined back to the prompt via `aeo_prompt_id` / `aeo_run_id`.
from __future__ import annotations
+import time
+import threading
from dataclasses import field
from typing import Any, Protocol
from urllib.parse import urlparse
@@ -97,12 +99,68 @@ class CitationEngine(Protocol):
def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: ...
-def _session() -> requests.Session:
+class _ThreadSessions:
+ """One requests.Session per thread. A run calls engines from a thread pool, and
+ requests does not document Session as safe to share across threads."""
+
+ def __init__(self, *, trust_env: bool) -> None:
+ self._local = threading.local()
+ self._trust_env = trust_env
+
+ def get(self) -> requests.Session:
+ session: requests.Session | None = getattr(self._local, "session", None)
+ if session is None:
+ session = requests.Session()
+ session.trust_env = self._trust_env
+ self._local.session = session
+ return session
+
+
+def _gateway_sessions() -> _ThreadSessions:
# trust_env=False keeps in-cluster gateway calls off any egress proxy, and
- # the session reuses connections across a run's many sequential calls.
- session = requests.Session()
- session.trust_env = False
- return session
+ # each thread's session reuses connections across its calls.
+ return _ThreadSessions(trust_env=False)
+
+
+# Throttling and transient upstream errors. A full run sends many calls in parallel, so
+# these get a short bounded retry instead of becoming a failed check straight away.
+RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})
+GATEWAY_MAX_ATTEMPTS = 3
+GATEWAY_RETRY_BASE_SECONDS = 5
+GATEWAY_RETRY_MAX_SECONDS = 60
+
+
+def _retry_delay_seconds(response: requests.Response, attempt: int) -> float:
+ retry_after = response.headers.get("Retry-After", "")
+ if retry_after.isdigit():
+ return min(float(retry_after), GATEWAY_RETRY_MAX_SECONDS)
+ return min(GATEWAY_RETRY_BASE_SECONDS * 2**attempt, GATEWAY_RETRY_MAX_SECONDS)
+
+
+class _SharedBackoff:
+ """A pause every worker thread honours before its next gateway request. One throttled
+ response pauses the whole run, so the other workers stop adding load while it backs off."""
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._resume_at = 0.0
+
+ def pause(self, seconds: float) -> None:
+ with self._lock:
+ # A shorter pause from another worker must not cut a longer one short.
+ self._resume_at = max(self._resume_at, time.monotonic() + seconds)
+
+ def wait(self) -> None:
+ # Another worker can extend the pause during the sleep, so read the deadline again after each one.
+ while True:
+ with self._lock:
+ remaining = self._resume_at - time.monotonic()
+ if remaining <= 0:
+ return
+ time.sleep(remaining)
+
+
+GATEWAY_BACKOFF = _SharedBackoff()
def gateway_post_json(
@@ -112,10 +170,17 @@ def gateway_post_json(
payload: dict[str, Any],
timeout: int = GATEWAY_TIMEOUT_SECONDS,
) -> dict[str, Any]:
- """POST and decode JSON; raises requests.RequestException on any failure."""
- response = session.post(url, headers=headers, json=payload, timeout=timeout)
- response.raise_for_status()
- return response.json()
+ """POST and decode JSON, retrying throttled and transient errors a bounded number of
+ times; raises requests.RequestException once the attempts run out or on any other failure."""
+ for attempt in range(GATEWAY_MAX_ATTEMPTS):
+ GATEWAY_BACKOFF.wait()
+ response = session.post(url, headers=headers, json=payload, timeout=timeout)
+ if response.status_code in RETRYABLE_STATUS_CODES and attempt < GATEWAY_MAX_ATTEMPTS - 1:
+ GATEWAY_BACKOFF.pause(_retry_delay_seconds(response, attempt))
+ continue
+ response.raise_for_status()
+ return response.json()
+ raise AssertionError("unreachable: the last attempt returns or raises")
def _dedupe_urls(urls: list[str]) -> list[str]:
@@ -349,7 +414,7 @@ class ClaudeWebSearchEngine:
if gateway is None:
raise ValueError("AI gateway is not configured (AI_GATEWAY_URL / AI_GATEWAY_API_KEY)")
self._gateway = gateway
- self._session = _session()
+ self._sessions = _gateway_sessions()
def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:
payload = {
@@ -375,7 +440,9 @@ class ClaudeWebSearchEngine:
**(ai_gateway_headers(trace_id=trace_id, properties=custom_properties) or {}),
}
try:
- body = gateway_post_json(self._session, self._gateway.url.rstrip("/") + "/messages", headers, payload)
+ body = gateway_post_json(
+ self._sessions.get(), self._gateway.url.rstrip("/") + "/messages", headers, payload
+ )
except requests.RequestException as e:
return CitationCheck(engine=self.name, model=self.model, trace_id=trace_id, error=_request_error(e))
parsed = parse_anthropic_citations(body)
@@ -403,7 +470,7 @@ class OpenAIWebSearchEngine:
if gateway is None:
raise ValueError("AI gateway is not configured (AI_GATEWAY_URL / AI_GATEWAY_API_KEY)")
self._gateway = gateway
- self._session = _session()
+ self._sessions = _gateway_sessions()
def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:
payload = {
@@ -417,7 +484,9 @@ class OpenAIWebSearchEngine:
**(ai_gateway_headers(trace_id=trace_id, properties=custom_properties) or {}),
}
try:
- body = gateway_post_json(self._session, self._gateway.url.rstrip("/") + "/responses", headers, payload)
+ body = gateway_post_json(
+ self._sessions.get(), self._gateway.url.rstrip("/") + "/responses", headers, payload
+ )
except requests.RequestException as e:
return CitationCheck(engine=self.name, model=self.model, trace_id=trace_id, error=_request_error(e))
if (error := openai_response_error(body)) is not None:
@@ -446,13 +515,13 @@ class ExaAnswerEngine:
model = "exa-answer"
def __init__(self) -> None:
- self._session = requests.Session()
+ self._sessions = _ThreadSessions(trust_env=True)
def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:
headers = {"x-api-key": settings.EXA_API_KEY}
try:
body = gateway_post_json(
- self._session,
+ self._sessions.get(),
EXA_ANSWER_URL,
headers,
{"query": prompt, "text": False},
diff --git a/products/aeo/backend/management/commands/run_aeo_citation_checks.py b/products/aeo/backend/management/commands/run_aeo_citation_checks.py
index 39b803d44a15..17b53e44245b 100644
--- a/products/aeo/backend/management/commands/run_aeo_citation_checks.py
+++ b/products/aeo/backend/management/commands/run_aeo_citation_checks.py
@@ -25,7 +25,7 @@ class Command(BaseCommand):
def add_arguments(self, parser: CommandParser) -> None:
parser.add_argument("--team-id", type=int, required=True)
- parser.add_argument("--limit", type=int, help="Max prompts this run (default 50).")
+ parser.add_argument("--limit", type=int, help="Max prompts this run (default AEO_MAX_PROMPTS_PER_RUN).")
parser.add_argument(
"--engines",
type=str,
diff --git a/products/aeo/backend/runner.py b/products/aeo/backend/runner.py
index f9cfeb74e416..c62462bc3d12 100644
--- a/products/aeo/backend/runner.py
+++ b/products/aeo/backend/runner.py
@@ -15,6 +15,7 @@ gateway-key owner's project, not the checked team's, and carries the
from __future__ import annotations
import uuid
+from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Optional
from django.conf import settings
@@ -24,6 +25,7 @@ from django.utils import timezone
import structlog
+from posthog.dataclasses import frozen
from posthog.models.scoping import team_scope
from posthog.models.team import Team
@@ -34,8 +36,6 @@ from products.aeo.backend.temporal.brand_sentiment import start_brand_sentiment_
logger = structlog.get_logger(__name__)
-DEFAULT_MAX_PROMPTS_PER_RUN = 50
-
def run_citation_checks(
team: Team,
@@ -55,7 +55,7 @@ def run_citation_checks(
logger.warning("aeo_citation_run_no_engines", team_id=team.id)
return _empty_summary(team, "no engines configured (AI_GATEWAY_URL/AI_GATEWAY_API_KEY, EXA_API_KEY)"), []
- prompts = select_prompts(team, DEFAULT_MAX_PROMPTS_PER_RUN if limit is None else limit)
+ prompts = select_prompts(team, settings.AEO_MAX_PROMPTS_PER_RUN if limit is None else limit)
if not prompts:
logger.warning("aeo_citation_run_no_prompts", team_id=team.id)
return _empty_summary(team, "no active prompts — run `seed_aeo_prompts` first"), []
@@ -69,59 +69,39 @@ def run_citation_checks(
rows_written = 0
write_failures = 0
- for prompt in prompts:
- prompt_rows: list[AEOCitationCheck] = []
- for engine in engines:
- trace_id = str(uuid.uuid4())
- check = engine.run(
- prompt.prompt,
- trace_id=trace_id,
- custom_properties={
- "aeo_prompt_id": str(prompt.id),
- "aeo_run_id": run_id,
- "aeo_prompt_source": prompt.prompt_source,
- # No $ai_ prefix, so the gateway keeps this on the $ai_generation
- # event, which lets usage attribute spend back to the checked team.
- "team_id": str(team.id),
- },
- )
- if check.error is not None:
- engine_failures += 1
- logger.warning(
- "aeo_citation_check_failed",
- team_id=team.id,
- engine=engine.name,
- prompt_id=str(prompt.id),
- error=check.error,
- )
- fields = {
- **build_check_fields(
- check=check,
- run_id=run_id,
- prompt_id=str(prompt.id),
- prompt_text=prompt.prompt,
- prompt_source=prompt.prompt_source,
- prompt_hash=prompt.prompt_hash,
- target_domains=target_domains,
- ),
- "prompt_topic": prompt.topic,
- "prompt_version": prompt.current_version,
- }
- prompt_rows.append(AEOCitationCheck(team_id=canonical_team_id, **fields))
- checks.append(fields)
-
- if record and prompt_rows:
- # Write per prompt so a mid-run crash keeps the completed checks.
+ # Engine calls are network-bound and run in worker threads. Database writes stay on
+ # this thread, so the workers never open their own Django connections.
+ pool = ThreadPoolExecutor(max_workers=max(1, settings.AEO_RUN_CONCURRENCY))
+ futures = {pool.submit(_check_prompt, team, run_id, engines, target_domains, prompt): prompt for prompt in prompts}
+ try:
+ for done, future in enumerate(as_completed(futures), start=1):
try:
- # The fail-closed manager refuses writes without a team scope, and none of the callers
- # (the Celery task, the management command) run inside one.
- with team_scope(canonical_team_id):
- AEOCitationCheck.objects.bulk_create(prompt_rows)
- rows_written += len(prompt_rows)
- AEOPrompt.objects.for_team(team.id).filter(id=prompt.id).update(last_checked_at=timezone.now())
+ result = future.result()
except Exception as e:
- write_failures += len(prompt_rows)
- logger.exception("aeo_citation_write_failed", team_id=team.id, run_id=run_id, error=str(e)[:300])
+ # One prompt's unexpected engine error must not end a run that takes hours.
+ engine_failures += len(engines)
+ logger.exception(
+ "aeo_citation_prompt_crashed",
+ team_id=team.id,
+ prompt_id=str(futures[future].id),
+ error=str(e)[:300],
+ )
+ continue
+ engine_failures += result.failures
+ checks.extend(result.fields_per_engine)
+ if record and result.fields_per_engine:
+ if _record_prompt_checks(team, canonical_team_id, run_id, result.prompt, result.fields_per_engine):
+ rows_written += len(result.fields_per_engine)
+ else:
+ write_failures += len(result.fields_per_engine)
+ if done % 100 == 0:
+ logger.info("aeo_citation_run_progress", team_id=team.id, run_id=run_id, done=done, total=len(prompts))
+ except BaseException:
+ # A Celery time limit or a worker shutdown lands here. Drop the queued prompts so they
+ # make no more paid calls whose results nobody records.
+ pool.shutdown(wait=False, cancel_futures=True)
+ raise
+ pool.shutdown(wait=True)
summary = CitationRunSummary(
team_id=team.id,
@@ -150,6 +130,82 @@ def run_citation_checks(
return summary, checks
+@frozen
+class PromptCheckResult:
+ """One prompt's checks across every engine, before they are written."""
+
+ prompt: AEOPrompt
+ fields_per_engine: list[dict[str, Any]]
+ failures: int
+
+
+def _check_prompt(
+ team: Team, run_id: str, engines: list[CitationEngine], target_domains: list[str], prompt: AEOPrompt
+) -> PromptCheckResult:
+ """Run one prompt on every engine. Runs in a worker thread, so it must not touch the database."""
+ fields_per_engine: list[dict[str, Any]] = []
+ failures = 0
+ for engine in engines:
+ trace_id = str(uuid.uuid4())
+ check = engine.run(
+ prompt.prompt,
+ trace_id=trace_id,
+ custom_properties={
+ "aeo_prompt_id": str(prompt.id),
+ "aeo_run_id": run_id,
+ "aeo_prompt_source": prompt.prompt_source,
+ # No $ai_ prefix, so the gateway keeps this on the $ai_generation
+ # event, which lets usage attribute spend back to the checked team.
+ "team_id": str(team.id),
+ },
+ )
+ if check.error is not None:
+ failures += 1
+ logger.warning(
+ "aeo_citation_check_failed",
+ team_id=team.id,
+ engine=engine.name,
+ prompt_id=str(prompt.id),
+ error=check.error,
+ )
+ fields_per_engine.append(
+ {
+ **build_check_fields(
+ check=check,
+ run_id=run_id,
+ prompt_id=str(prompt.id),
+ prompt_text=prompt.prompt,
+ prompt_source=prompt.prompt_source,
+ prompt_hash=prompt.prompt_hash,
+ target_domains=target_domains,
+ ),
+ "prompt_topic": prompt.topic,
+ "prompt_version": prompt.current_version,
+ }
+ )
+ return PromptCheckResult(prompt=prompt, fields_per_engine=fields_per_engine, failures=failures)
+
+
+def _record_prompt_checks(
+ team: Team, canonical_team_id: int, run_id: str, prompt: AEOPrompt, fields_per_engine: list[dict[str, Any]]
+) -> bool:
+ """Write one prompt's checks and mark the prompt checked. Called per prompt so a
+ mid-run crash keeps the completed checks; False when the write failed."""
+ prompt_rows = [AEOCitationCheck(team_id=canonical_team_id, **fields) for fields in fields_per_engine]
+ try:
+ # The fail-closed manager refuses writes without a team scope, and none of the callers
+ # (the Celery task, the management command) run inside one.
+ with team_scope(canonical_team_id):
+ AEOCitationCheck.objects.bulk_create(prompt_rows)
+ # A prompt every engine failed on was not really checked, so leave it first in line for the next run.
+ if any(not fields["check_failed"] for fields in fields_per_engine):
+ AEOPrompt.objects.for_team(team.id).filter(id=prompt.id).update(last_checked_at=timezone.now())
+ except Exception as e:
+ logger.exception("aeo_citation_write_failed", team_id=team.id, run_id=run_id, error=str(e)[:300])
+ return False
+ return True
+
+
def select_prompts(team: Team, limit: int) -> list[AEOPrompt]:
"""Pick up to `limit` active, unarchived prompts, alternating between topics.
diff --git a/products/aeo/backend/tasks/tasks.py b/products/aeo/backend/tasks/tasks.py
index 1f2a69198528..50348a8f5298 100644
--- a/products/aeo/backend/tasks/tasks.py
+++ b/products/aeo/backend/tasks/tasks.py
@@ -25,10 +25,11 @@ from products.aeo.backend.runner import run_citation_checks
logger = structlog.get_logger(__name__)
-# A full run is up to 50 prompts x 3 engines of sequential web-search calls
-# (typically 10-60s each); four hours gives slow days headroom without letting
-# a hung run hold a worker slot indefinitely.
-RUN_SOFT_TIME_LIMIT_SECONDS = 4 * 60 * 60
+# A full run covers the whole prompt set: each prompt makes one web-search call per
+# engine (about a minute each), AEO_RUN_CONCURRENCY prompts at a time. Ten hours
+# leaves headroom for a slow day and still ends a hung run well before the next
+# daily dispatch.
+RUN_SOFT_TIME_LIMIT_SECONDS = 10 * 60 * 60
RUN_TIME_LIMIT_SECONDS = RUN_SOFT_TIME_LIMIT_SECONDS + 300
diff --git a/products/aeo/backend/test/test_engines.py b/products/aeo/backend/test/test_engines.py
index 8db3f963aa25..fc8bfed13e4e 100644
--- a/products/aeo/backend/test/test_engines.py
+++ b/products/aeo/backend/test/test_engines.py
@@ -1,16 +1,23 @@
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
from types import SimpleNamespace
-from unittest.mock import patch
+import pytest
+from unittest.mock import MagicMock, patch
+import requests
from parameterized import parameterized
from products.aeo.backend.engines import (
+ GATEWAY_MAX_ATTEMPTS,
OPENAI_MAX_OUTPUT_TOKENS,
CitationCheck,
ClaudeWebSearchEngine,
OpenAIWebSearchEngine,
+ _SharedBackoff,
anthropic_truncated_error,
build_check_fields,
+ gateway_post_json,
is_target_url,
openai_response_error,
parse_anthropic_citations,
@@ -350,3 +357,78 @@ def test_engine_derived_text_is_sanitized_before_it_reaches_the_event() -> None:
# Sanitizing must not move the verdict: it runs on the recorded copy only.
assert fields["cited"] is True
assert fields["target_best_position"] == 1
+
+
+def _response(status: int, retry_after: str | None = None) -> requests.Response:
+ response = requests.Response()
+ response.status_code = status
+ response._content = b'{"ok": true}'
+ if retry_after is not None:
+ response.headers["Retry-After"] = retry_after
+ return response
+
+
+ [
+ ("throttled_then_ok", [_response(429, retry_after="7"), _response(200)], None, 2, [7.0]),
+ ("transient_until_attempts_run_out", [_response(503)] * GATEWAY_MAX_ATTEMPTS, requests.HTTPError, 3, [5, 10]),
+ ("client_error_is_not_retried", [_response(400)], requests.HTTPError, 1, []),
+ ]
+)
+def test_gateway_post_json_retries_throttling_and_transient_errors(
+ _name: str, responses: list, raises: type[Exception] | None, posts: int, sleeps: list[float]
+) -> None:
+ session = MagicMock(spec=requests.Session)
+ session.post.side_effect = responses
+
+ with (
+ patch("products.aeo.backend.engines.GATEWAY_BACKOFF", _SharedBackoff()),
+ _fake_clock() as sleep,
+ ):
+ if raises is None:
+ assert gateway_post_json(session, "https://gw.test", {}, {}) == {"ok": True}
+ else:
+ with pytest.raises(raises):
+ gateway_post_json(session, "https://gw.test", {}, {})
+
+ assert session.post.call_count == posts
+ assert [call.args[0] for call in sleep.call_args_list] == sleeps
+
+
+ [
+ ("shorter_pause_does_not_cut_a_longer_one", [30, 5], None, [30.0]),
+ ("pause_extended_during_the_sleep", [30], 60, [30.0, 30.0]),
+ ]
+)
+def test_shared_backoff_waits_for_the_latest_deadline(
+ _name: str, pauses: list[float], extend_during_sleep: float | None, sleeps: list[float]
+) -> None:
+ backoff = _SharedBackoff()
+
+ def other_worker_pauses() -> None:
+ if extend_during_sleep is not None:
+ backoff.pause(extend_during_sleep)
+
+ with _fake_clock(on_first_sleep=other_worker_pauses) as sleep:
+ for seconds in pauses:
+ backoff.pause(seconds)
+ backoff.wait()
+
+ assert [call.args[0] for call in sleep.call_args_list] == sleeps
+
+
+@contextmanager
+def _fake_clock(on_first_sleep: Callable[[], None] | None = None) -> Iterator[MagicMock]:
+ now = [100.0]
+
+ def sleep(seconds: float) -> None:
+ if on_first_sleep is not None and sleep_mock.call_count == 1:
+ on_first_sleep()
+ now[0] += seconds
+
+ with (
+ patch("products.aeo.backend.engines.time.monotonic", side_effect=lambda: now[0]),
+ patch("products.aeo.backend.engines.time.sleep", side_effect=sleep) as sleep_mock,
+ ):
+ yield sleep_mock
diff --git a/products/aeo/backend/test/test_runner.py b/products/aeo/backend/test/test_runner.py
index 171554fb8fcc..0218de612a9f 100644
--- a/products/aeo/backend/test/test_runner.py
+++ b/products/aeo/backend/test/test_runner.py
@@ -1,4 +1,6 @@
import datetime as dt
+import threading
+from typing import NoReturn
import pytest
@@ -7,6 +9,7 @@ from django.utils import timezone
from posthog.models.organization import Organization
from posthog.models.team import Team
+from products.aeo.backend import runner
from products.aeo.backend.engines import CitationCheck
from products.aeo.backend.models import AEOCitationCheck, AEOPrompt
from products.aeo.backend.runner import run_citation_checks, select_prompts
@@ -60,8 +63,12 @@ class _StubEngine:
@pytest.mark.parametrize("via_child_environment", [False, True])
-def test_run_records_checks_and_rotates_without_a_team_scope(team: Team, settings, via_child_environment: bool) -> None:
[email protected]("concurrency", [1, 4])
+def test_run_records_checks_and_rotates_without_a_team_scope(
+ team: Team, settings, via_child_environment: bool, concurrency: int
+) -> None:
settings.AEO_TARGET_DOMAINS = ["posthog.com"]
+ settings.AEO_RUN_CONCURRENCY = concurrency
runner_team = (
Team.objects.create(organization=team.organization, name="child env", parent_team=team)
if via_child_environment
@@ -82,3 +89,79 @@ def test_run_records_checks_and_rotates_without_a_team_scope(team: Team, setting
assert {c.prompt_topic for c in checks} == {"t0", "t1"}
assert len({c.prompt_id for c in checks}) == 4
assert sorted((c.prompt_text, c.prompt_version) for c in checks) == [("q0?", 3), ("q1?", 1), ("q2?", 1), ("q3?", 1)]
+
+
+class _CrashingEngine(_StubEngine):
+ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:
+ if prompt == "q1?":
+ raise RuntimeError("unexpected engine bug")
+ return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties)
+
+
+def test_one_prompt_crashing_does_not_end_the_run(team: Team, settings) -> None:
+ settings.AEO_RUN_CONCURRENCY = 2
+ settings.AEO_MAX_PROMPTS_PER_RUN = 3
+ upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(3)])
+
+ summary, _ = run_citation_checks(team, engines=[_CrashingEngine()])
+
+ assert (summary.prompts, summary.rows_written, summary.engine_failures) == (3, 2, 1)
+ assert sorted(AEOCitationCheck.objects.for_team(team.id).values_list("prompt_text", flat=True)) == ["q0?", "q2?"]
+
+
+class _FailingEngine(_StubEngine):
+ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:
+ if prompt == "q1?":
+ return CitationCheck(engine=self.name, model=self.model, error="HTTPError: status=429")
+ return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties)
+
+
+def test_a_prompt_every_engine_failed_on_stays_unchecked(team: Team, settings) -> None:
+ settings.AEO_RUN_CONCURRENCY = 2
+ upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(2)])
+
+ summary, _ = run_citation_checks(team, engines=[_FailingEngine()])
+
+ assert (summary.rows_written, summary.engine_failures) == (2, 1)
+ last_checked = dict(AEOPrompt.objects.for_team(team.id).values_list("prompt", "last_checked_at"))
+ assert last_checked["q0?"] is not None
+ assert last_checked["q1?"] is None
+
+
+class _BlockingEngine(_StubEngine):
+ """Holds every check until the test releases it, so the test decides which prompts are in flight."""
+
+ def __init__(self, workers: int) -> None:
+ self.prompts: list[str] = []
+ self.threads: set[threading.Thread] = set()
+ self.all_workers_busy = threading.Barrier(workers + 1)
+ self.release = threading.Event()
+
+ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:
+ self.prompts.append(prompt)
+ self.threads.add(threading.current_thread())
+ self.all_workers_busy.wait(timeout=5)
+ self.release.wait(timeout=5)
+ return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties)
+
+
+def test_an_interrupted_run_does_not_start_the_queued_prompts(team: Team, settings, monkeypatch) -> None:
+ settings.AEO_RUN_CONCURRENCY = 2
+ upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(5)])
+ engine = _BlockingEngine(workers=2)
+
+ def interrupt_once_every_worker_is_busy(futures: object) -> NoReturn:
+ # Stands in for a Celery soft time limit landing while the run waits on its workers.
+ engine.all_workers_busy.wait(timeout=5)
+ raise KeyboardInterrupt
+
+ monkeypatch.setattr(runner, "as_completed", interrupt_once_every_worker_is_busy)
+
+ with pytest.raises(KeyboardInterrupt):
+ run_citation_checks(team, engines=[engine])
+ engine.release.set()
+ for thread in engine.threads:
+ thread.join(timeout=5)
+
+ assert len(engine.prompts) == 2
+ assert len(set(engine.prompts)) == 2