Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

42.3 KB
Raw Blame
import json
import uuid
import datetime as dt

import time_machine
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch

from django.db import connection
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from django.utils import timezone

import requests
from parameterized import parameterized
from rest_framework import status

from posthog.models.personal_api_key import PersonalAPIKey, hash_key_value
from posthog.models.scoping import team_scope
from posthog.models.team import Team
from posthog.models.utils import generate_random_token_personal

from products.aeo.backend.engines import top_domains
from products.aeo.backend.facade.api import get_prompt_set_overview
from products.aeo.backend.models import AEOCitationCheck, AEOPrompt, AEOPromptVersion
from products.aeo.backend.test.factories import create_citation_check


class TestAEOPromptAPI(APIBaseTest):
    def setUp(self) -> None:
        super().setUp()
        flag_patcher = patch(
            "products.aeo.backend.presentation.views.posthog_feature_flag_enabled",
            return_value=True,
        )
        self.feature_enabled = flag_patcher.start()
        self.addCleanup(flag_patcher.stop)
        self.url = f"/api/projects/{self.team.id}/aeo_prompts/"

    def _record_check(
        self,
        prompt_id: str,
        engine: str,
        *,
        minutes_ago: int,
        cited: bool = False,
        check_failed: bool = False,
        answer_text: str | None = None,
        cited_urls: list[str] | None = None,
        search_queries: list[str] | None = None,
    ) -> str:
        with team_scope(self.team.pk):
            prompt = AEOPrompt.objects.get(id=prompt_id)
            check = AEOCitationCheck.objects.create(
                team=self.team,
                prompt=prompt,
                run_id=uuid.uuid4(),
                prompt_text=prompt.prompt,
                prompt_source=prompt.prompt_source,
                prompt_hash=prompt.prompt_hash,
                prompt_topic=prompt.topic,
                prompt_version=prompt.current_version,
                engine=engine,
                model=engine,
                cited=cited,
                check_failed=check_failed,
                answer_text=answer_text,
                cited_urls=cited_urls or [],
                top_cited_domains=top_domains(cited_urls or []),
                search_queries=search_queries or [],
            )
            AEOCitationCheck.objects.filter(id=check.id).update(
                created_at=timezone.now() - dt.timedelta(minutes=minutes_ago)
            )
        return str(check.id)

    def _import_one_prompt(self) -> str:
        self.client.post(f"{self.url}bulk_import/", {"prompts": [{"prompt": "tracked?"}]}, format="json")
        return self.client.get(self.url).json()["results"][0]["id"]

    def _import_prompts(self, prompts: list[dict]) -> dict[str, str]:
        self.client.post(f"{self.url}bulk_import/", {"prompts": prompts}, format="json")
        listed = self.client.get(f"{self.url}?limit=100").json()["results"]
        return {row["prompt"]: row["id"] for row in listed}

    def test_requires_the_rollout_flag(self) -> None:
        self.feature_enabled.return_value = False

        assert self.client.get(self.url).status_code == status.HTTP_403_FORBIDDEN
        response = self.client.post(f"{self.url}bulk_import/", {"prompts": [{"prompt": "q?"}]}, format="json")
        assert response.status_code == status.HTTP_403_FORBIDDEN
        assert not AEOPrompt.objects.for_team(self.team.id).exists()

    def test_bulk_import_then_deactivate_survives_a_reimport(self) -> None:
        payload = {
            "prompts": [
                {"prompt": "Best session replay tool?", "topic": "Session replay", "external_visibility": 42.5},
                {"prompt": "Best feature flag tool?", "topic": "Feature flags"},
                {"prompt": "   "},
            ]
        }
        response = self.client.post(f"{self.url}bulk_import/", payload, format="json")
        assert response.status_code == status.HTTP_200_OK, response.json()
        assert response.json() == {"created": 2, "updated": 0, "skipped": 1}

        listed = {row["prompt"]: row for row in self.client.get(self.url).json()["results"]}
        assert listed["Best session replay tool?"]["topic"] == "Session replay"
        assert listed["Best session replay tool?"]["external_visibility"] == 42.5
        assert listed["Best feature flag tool?"]["external_visibility"] is None

        replay_id = listed["Best session replay tool?"]["id"]
        response = self.client.patch(f"{self.url}{replay_id}/", {"active": False}, format="json")
        assert response.status_code == status.HTTP_200_OK
        assert response.json()["active"] is False

        response = self.client.post(f"{self.url}bulk_import/", payload, format="json")
        assert response.json() == {"created": 0, "updated": 2, "skipped": 1}
        assert AEOPrompt.objects.for_team(self.team.id).get(id=replay_id).active is False

        self.client.post(
            f"{self.url}bulk_import/",
            {"prompts": [{"prompt": "Best session replay tool?", "topic": "Session replay"}]},
            format="json",
        )
        assert self.client.get(f"{self.url}{replay_id}/").json()["external_visibility"] == 42.5

        flags_id = listed["Best feature flag tool?"]["id"]
        self.client.delete(f"{self.url}{flags_id}/")
        response = self.client.post(f"{self.url}bulk_import/", payload, format="json")
        assert response.json() == {"created": 0, "updated": 1, "skipped": 2}
        flags = AEOPrompt.objects.for_team(self.team.id).get(id=flags_id)
        assert (flags.archived_at is not None, flags.active) == (True, False)
        assert AEOPrompt.objects.for_team(self.team.id).count() == 2

    def test_text_edit_adds_a_version_and_current_version_rolls_it_back(self) -> None:
        prompt_id = self._import_one_prompt()
        self._record_check(prompt_id, "exa-answer", minutes_ago=30)

        response = self.client.patch(
            f"{self.url}{prompt_id}/",
            {"prompt": "  Still   tracked? ", "version_description": "Clearer wording", "topic": "New topic"},
            format="json",
        )

        assert response.status_code == status.HTTP_200_OK, response.json()
        edited = response.json()
        assert (edited["id"], edited["prompt"], edited["topic"]) == (prompt_id, "Still tracked?", "New topic")
        assert (edited["current_version"], edited["version_count"], len(edited["latest_checks"])) == (2, 2, 1)
        self._record_check(prompt_id, "exa-answer", minutes_ago=5)
        versions = self.client.get(f"{self.url}{prompt_id}/versions/").json()
        assert [(v["version"], v["text"], v["version_description"], v["is_current"]) for v in versions] == [
            (2, "Still tracked?", "Clearer wording", True),
            (1, "tracked?", "", False),
        ]
        assert versions[0]["created_by"]["email"] == self.user.email
        with team_scope(self.team.pk):
            AEOCitationCheck.objects.filter(prompt_id=prompt_id, prompt_version=1).update(prompt_version=None)
        checks = self.client.get(f"{self.url}{prompt_id}/checks/?version=1").json()["results"]
        assert [check["prompt_version"] for check in checks] == [None]

        response = self.client.patch(f"{self.url}{prompt_id}/", {"current_version": 1}, format="json")

        assert response.status_code == status.HTTP_200_OK, response.json()
        rolled_back = response.json()
        assert (rolled_back["prompt"], rolled_back["current_version"], rolled_back["version_count"]) == (
            "tracked?",
            1,
            2,
        )
        assert AEOPromptVersion.objects.for_team(self.team.id).count() == 2

    def test_import_matches_prompts_on_their_current_text(self) -> None:
        prompt_id = self._import_one_prompt()
        self.client.patch(f"{self.url}{prompt_id}/", {"prompt": "Still tracked?"}, format="json")

        response = self.client.post(
            f"{self.url}bulk_import/",
            {"prompts": [{"prompt": "still TRACKED?", "topic": "Kept"}, {"prompt": "tracked?"}]},
            format="json",
        )

        assert response.json() == {"created": 1, "updated": 1, "skipped": 0}
        edited = self.client.get(f"{self.url}{prompt_id}/").json()
        assert (edited["prompt"], edited["topic"], edited["current_version"]) == ("Still tracked?", "Kept", 2)
        assert {row["prompt"]: row["version_count"] for row in self.client.get(self.url).json()["results"]} == {
            "Still tracked?": 2,
            "tracked?": 1,
        }

    @parameterized.expand(
        [
            ("topic_only", {"topic": "Replay"}, "Replay"),
            ("clear_topic", {"topic": ""}, ""),
            ("case_and_whitespace_only", {"prompt": "  TRACKED?  ", "topic": "Replay"}, "Replay"),
            ("current_version_already_current", {"current_version": 1}, "Before"),
        ]
    )
    def test_edits_that_keep_the_text_add_no_version(self, _name: str, body: dict, expected_topic: str) -> None:
        prompt_id = self._import_one_prompt()
        self.client.patch(f"{self.url}{prompt_id}/", {"topic": "Before"}, format="json")

        response = self.client.patch(f"{self.url}{prompt_id}/", body, format="json")

        assert response.status_code == status.HTTP_200_OK, response.json()
        row = response.json()
        assert (row["prompt"], row["topic"], row["current_version"], row["version_count"]) == (
            "tracked?",
            expected_topic,
            1,
            1,
        )

    @parameterized.expand(
        [
            ("text_of_an_active_prompt", False, False),
            ("text_of_an_archived_prompt", True, False),
            ("rollback_onto_an_active_prompt", False, True),
        ]
    )
    def test_taking_another_prompts_text_conflicts(self, _name: str, archive_other: bool, rollback: bool) -> None:
        self.client.post(
            f"{self.url}bulk_import/", {"prompts": [{"prompt": "first?"}, {"prompt": "second?"}]}, format="json"
        )
        ids = {row["prompt"]: row["id"] for row in self.client.get(self.url).json()["results"]}
        if rollback:
            self.client.patch(f"{self.url}{ids['first?']}/", {"prompt": "third?"}, format="json")
            self.client.patch(f"{self.url}{ids['second?']}/", {"prompt": "first?"}, format="json")
            body: dict = {"current_version": 1}
        else:
            body = {"prompt": "SECOND?"}
        if archive_other:
            self.client.delete(f"{self.url}{ids['second?']}/")
        before = AEOPrompt.objects.for_team(self.team.id).get(id=ids["first?"])

        response = self.client.patch(f"{self.url}{ids['first?']}/", body, format="json")

        assert response.status_code == status.HTTP_409_CONFLICT, response.json()
        after = AEOPrompt.objects.for_team(self.team.id).get(id=ids["first?"])
        assert (after.prompt, after.current_version) == (before.prompt, before.current_version)
        assert AEOPromptVersion.objects.for_team(self.team.id).filter(prompt_id=after.id).count() == (
            2 if rollback else 1
        )

    @parameterized.expand(
        [
            ("patch", "patch", {"archived": True}, status.HTTP_200_OK),
            ("delete", "delete", None, status.HTTP_204_NO_CONTENT),
        ]
    )
    def test_archive_hides_the_prompt_from_the_default_list(
        self, _name: str, method: str, body: dict | None, expected_status: int
    ) -> None:
        prompt_id = self._import_one_prompt()
        self._record_check(prompt_id, "exa-answer", minutes_ago=5)

        response = getattr(self.client, method)(f"{self.url}{prompt_id}/", body, format="json")

        assert response.status_code == expected_status
        assert self.client.get(self.url).json()["count"] == 0
        assert self.client.get(f"{self.url}topics/").json() == []
        listed = self.client.get(f"{self.url}?include_archived=true").json()["results"]
        assert [row["id"] for row in listed] == [prompt_id]
        retrieved = self.client.get(f"{self.url}{prompt_id}/").json()
        assert (retrieved["active"], retrieved["archived_at"] is not None) == (False, True)
        assert AEOCitationCheck.objects.for_team(self.team.id).filter(prompt_id=prompt_id).count() == 1

    @parameterized.expand([("stays_off", {}, False), ("turned_on", {"active": True}, True)])
    def test_restore_leaves_the_prompt_off_unless_asked(self, _name: str, extra: dict, expected_active: bool) -> None:
        prompt_id = self._import_one_prompt()
        self.client.delete(f"{self.url}{prompt_id}/")

        response = self.client.patch(f"{self.url}{prompt_id}/", {"archived": False, **extra}, format="json")

        assert response.status_code == status.HTTP_200_OK, response.json()
        assert (response.json()["archived_at"], response.json()["active"]) == (None, expected_active)

    @parameterized.expand(
        [
            ("empty_body", False, {}),
            ("empty_text", False, {"prompt": "   "}),
            ("unknown_version", False, {"current_version": 5}),
            ("text_and_version", False, {"prompt": "new text?", "current_version": 1}),
            ("note_without_text", False, {"version_description": "why"}),
            ("archive_with_other_changes", False, {"archived": True, "topic": "x"}),
            ("toggle_archived", True, {"active": True}),
            ("retopic_archived", True, {"topic": "x"}),
            ("reword_archived", True, {"prompt": "new text?"}),
            ("restore_and_reword", True, {"archived": False, "prompt": "new text?"}),
            ("restore_and_roll_back", True, {"archived": False, "current_version": 1}),
        ]
    )
    def test_rejected_edits_change_nothing(self, _name: str, archive_first: bool, body: dict) -> None:
        prompt_id = self._import_one_prompt()
        if archive_first:
            self.client.delete(f"{self.url}{prompt_id}/")
        before = AEOPrompt.objects.for_team(self.team.id).get(id=prompt_id)

        response = self.client.patch(f"{self.url}{prompt_id}/", body, format="json")

        assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
        after = AEOPrompt.objects.for_team(self.team.id).get()
        assert (after.id, after.prompt, after.topic, after.active, after.archived_at, after.current_version) == (
            before.id,
            before.prompt,
            before.topic,
            before.active,
            before.archived_at,
            before.current_version,
        )
        assert AEOPromptVersion.objects.for_team(self.team.id).count() == 1

    @parameterized.expand(
        [
            ("no_filter", "", 6, None),
            ("one_topic", "&topic=same topic", 5, ("topic", "same topic")),
            ("no_topic", "&topic=", 1, ("topic", "")),
            ("active", "&active=true", 5, ("active", True)),
            ("inactive", "&active=false", 1, ("active", False)),
        ]
    )
    def test_list_pages_without_overlap(
        self, _name: str, filters: str, expected: int, field_value: tuple[str, object] | None
    ) -> None:
        prompts = [{"prompt": f"q{i}?", "topic": "same topic"} for i in range(5)] + [{"prompt": "untopiced?"}]
        self.client.post(f"{self.url}bulk_import/", {"prompts": prompts}, format="json")
        untopiced = next(row for row in self.client.get(self.url).json()["results"] if row["topic"] == "")
        self.client.patch(f"{self.url}{untopiced['id']}/", {"active": False}, format="json")

        first = self.client.get(f"{self.url}?limit=3{filters}").json()
        second = self.client.get(f"{self.url}?limit=3&offset=3{filters}").json()

        assert first["count"] == expected
        ids = [row["id"] for row in first["results"] + second["results"]]
        assert len(ids) == len(set(ids)) == expected
        if field_value is not None:
            field, value = field_value
            assert {row[field] for row in first["results"] + second["results"]} == {value}

    def test_topics_lists_each_topic_with_its_prompt_count(self) -> None:
        prompts = [{"prompt": "a?", "topic": "Replay"}, {"prompt": "b?", "topic": "Replay"}, {"prompt": "c?"}]
        self.client.post(f"{self.url}bulk_import/", {"prompts": prompts}, format="json")

        response = self.client.get(f"{self.url}topics/")

        assert response.status_code == status.HTTP_200_OK, response.json()
        assert response.json() == [{"topic": "", "prompts": 1}, {"topic": "Replay", "prompts": 2}]

    def test_list_returns_the_latest_check_per_engine(self) -> None:
        prompts = [{"prompt": "checked?", "topic": "a"}, {"prompt": "never checked?", "topic": "b"}]
        self.client.post(f"{self.url}bulk_import/", {"prompts": prompts}, format="json")
        ids = {row["prompt"]: row["id"] for row in self.client.get(self.url).json()["results"]}
        checked_id = ids["checked?"]
        self._record_check(checked_id, "exa-answer", minutes_ago=60, cited=True)
        self._record_check(checked_id, "exa-answer", minutes_ago=5, cited=False)
        self._record_check(checked_id, "claude-web-search", minutes_ago=90, check_failed=True)
        self._record_check(checked_id, "claude-web-search", minutes_ago=30, cited=True)

        listed = {row["prompt"]: row for row in self.client.get(self.url).json()["results"]}
        retrieved = self.client.get(f"{self.url}{checked_id}/").json()

        for row in (listed["checked?"], retrieved):
            latest = {check["engine"]: check for check in row["latest_checks"]}
            assert len(row["latest_checks"]) == 2
            assert (latest["exa-answer"]["cited"], latest["exa-answer"]["check_failed"]) == (False, False)
            assert (latest["claude-web-search"]["cited"], latest["claude-web-search"]["check_failed"]) == (True, False)
        assert listed["never checked?"]["latest_checks"] == []

    def test_list_needs_improvement_keeps_answered_prompts_without_a_recent_citation(self) -> None:
        texts = ["cited?", "never cited?", "only failed?", "cited long ago?", "never checked?"]
        self.client.post(f"{self.url}bulk_import/", {"prompts": [{"prompt": t} for t in texts]}, format="json")
        ids = {row["prompt"]: row["id"] for row in self.client.get(self.url).json()["results"]}
        self._record_check(ids["cited?"], "exa-answer", minutes_ago=10, cited=False)
        self._record_check(ids["cited?"], "claude-web-search", minutes_ago=10, cited=True)
        self._record_check(ids["never cited?"], "exa-answer", minutes_ago=10, cited=False)
        self._record_check(ids["only failed?"], "exa-answer", minutes_ago=10, check_failed=True)
        self._record_check(ids["cited long ago?"], "exa-answer", minutes_ago=60 * 24 * 40, cited=True)
        self._record_check(ids["cited long ago?"], "exa-answer", minutes_ago=10, cited=False)
        # A citation on the old wording must not hide a current wording that no engine cites.
        self.client.post(f"{self.url}bulk_import/", {"prompts": [{"prompt": "edited?"}]}, format="json")
        edited_id = next(row["id"] for row in self.client.get(self.url).json()["results"] if row["prompt"] == "edited?")
        self._record_check(edited_id, "exa-answer", minutes_ago=20, cited=True)
        self.client.patch(f"{self.url}{edited_id}/", {"prompt": "edited again?"}, format="json")
        self._record_check(edited_id, "exa-answer", minutes_ago=10, cited=False)

        response = self.client.get(self.url, {"needs_improvement": "true"}).json()

        assert sorted(row["prompt"] for row in response["results"]) == [
            "cited long ago?",
            "edited again?",
            "never cited?",
        ]
        assert response["count"] == 3

    def test_list_query_count_does_not_grow_with_page_size(self) -> None:
        prompts = [{"prompt": f"q{i}?"} for i in range(4)]
        self.client.post(f"{self.url}bulk_import/", {"prompts": prompts}, format="json")
        for row in self.client.get(self.url).json()["results"]:
            self._record_check(row["id"], "exa-answer", minutes_ago=1)
            self._record_check(row["id"], "claude-web-search", minutes_ago=1)

        with CaptureQueriesContext(connection) as one_prompt:
            self.client.get(f"{self.url}?limit=1")
        with CaptureQueriesContext(connection) as four_prompts:
            self.client.get(f"{self.url}?limit=4")

        assert len(four_prompts.captured_queries) == len(one_prompt.captured_queries)

    @parameterized.expand([("all_engines", "", ["c", "b", "a"]), ("one_engine", "&engine=exa-answer", ["c", "a"])])
    def test_checks_list_history_newest_first_in_pages(self, _name: str, filters: str, expected: list[str]) -> None:
        prompt_id = self._import_one_prompt()
        self._record_check(prompt_id, "exa-answer", minutes_ago=30, answer_text="a")
        self._record_check(prompt_id, "claude-web-search", minutes_ago=20, answer_text="b")
        self._record_check(prompt_id, "exa-answer", minutes_ago=10, answer_text="c\nwith a line break")

        first = self.client.get(f"{self.url}{prompt_id}/checks/?limit=2{filters}").json()
        second = self.client.get(f"{self.url}{prompt_id}/checks/?limit=2&offset=2{filters}").json()

        assert first["count"] == len(expected)
        answers = [row["answer_text"] for row in first["results"] + second["results"]]
        assert [answer.split("\n")[0] for answer in answers] == expected
        assert answers[0] == "c\nwith a line break"

    @parameterized.expand(
        [
            (
                "since_drops_older_checks",
                "since={since}",
                status.HTTP_200_OK,
                [("exa-answer", "c"), ("claude-web-search", "b")],
            ),
            (
                "without_answer_text",
                "include_answer_text=false",
                status.HTTP_200_OK,
                [("exa-answer", None), ("claude-web-search", None), ("exa-answer", None)],
            ),
            ("invalid_since", "since=yesterday", status.HTTP_400_BAD_REQUEST, None),
        ]
    )
    def test_checks_list_filters_by_date_and_drops_answer_text_on_request(
        self, _name: str, query: str, expected_status: int, expected: list[tuple[str, str | None]] | None
    ) -> None:
        prompt_id = self._import_one_prompt()
        self._record_check(prompt_id, "exa-answer", minutes_ago=30, answer_text="a")
        self._record_check(prompt_id, "claude-web-search", minutes_ago=20, answer_text="b")
        self._record_check(prompt_id, "exa-answer", minutes_ago=10, answer_text="c")
        since = (timezone.now() - dt.timedelta(minutes=25)).strftime("%Y-%m-%dT%H:%M:%SZ")

        response = self.client.get(f"{self.url}{prompt_id}/checks/?{query.format(since=since)}")

        assert response.status_code == expected_status, response.json()
        if expected is not None:
            assert response.json()["count"] == len(expected)
            assert [(row["engine"], row["answer_text"]) for row in response.json()["results"]] == expected

    @parameterized.expand([("default", "", 20), ("capped", "?limit=1000", 100)])
    def test_checks_page_size_is_bounded(self, _name: str, query: str, expected: int) -> None:
        prompt_id = self._import_one_prompt()
        with team_scope(self.team.pk):
            prompt = AEOPrompt.objects.get(id=prompt_id)
            AEOCitationCheck.objects.bulk_create(
                AEOCitationCheck(
                    team=self.team,
                    prompt=prompt,
                    run_id=uuid.uuid4(),
                    prompt_text=prompt.prompt,
                    prompt_source=prompt.prompt_source,
                    prompt_hash=prompt.prompt_hash,
                    prompt_version=prompt.current_version,
                    engine="exa-answer",
                    model="exa-answer",
                )
                for _ in range(101)
            )

        response = self.client.get(f"{self.url}{prompt_id}/checks/{query}").json()

        assert response["count"] == 101
        assert len(response["results"]) == expected

    @parameterized.expand(
        [
            ("checks", "get", "checks/", None),
            ("stats", "get", "stats/", None),
            ("versions", "get", "versions/", None),
            ("update", "patch", "", {"prompt": "taken over?"}),
            ("archive", "delete", "", None),
        ]
    )
    def test_another_teams_prompt_is_not_found(self, _name: str, method: str, suffix: str, body: dict | None) -> None:
        other_team = Team.objects.create(organization=self.organization, name="Other")
        other_check_id = create_citation_check(other_team, "other")
        with team_scope(other_team.pk):
            other_prompt_id = AEOCitationCheck.objects.get(id=other_check_id).prompt_id

        response = getattr(self.client, method)(f"{self.url}{other_prompt_id}/{suffix}", body, format="json")

        assert response.status_code == status.HTTP_404_NOT_FOUND
        with team_scope(other_team.pk):
            other_prompt = AEOPrompt.objects.get(id=other_prompt_id)
            assert (other_prompt.prompt, other_prompt.archived_at) == ("is other cited?", None)
            assert AEOPrompt.objects.count() == 1

    @time_machine.travel("2026-10-20T12:00:00Z", tick=False)
    def test_stats_count_checks_per_engine_and_per_day(self) -> None:
        prompt_id = self._import_one_prompt()
        day = 24 * 60
        self._record_check(prompt_id, "exa-answer", minutes_ago=10, cited=True)
        self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example Analytics is one option.")
        self._record_check(prompt_id, "exa-answer", minutes_ago=day, check_failed=True)
        self._record_check(prompt_id, "claude-web-search", minutes_ago=2 * day, cited=True)
        self._record_check(prompt_id, "claude-web-search", minutes_ago=40 * day)

        with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
            response = self.client.get(f"{self.url}{prompt_id}/stats/")

        assert response.status_code == status.HTTP_200_OK, response.json()
        stats = response.json()
        assert stats["total_checks"] == 5
        assert (stats["daily_from"], stats["daily_to"]) == ("2026-09-21", "2026-10-20")
        engines = {
            row["engine"]: (
                row["checks"],
                row["cited"],
                row["mentioned"],
                row["failed"],
                row["citation_rate"],
                row["mention_rate"],
            )
            for row in stats["engines"]
        }
        assert engines == {"claude-web-search": (2, 1, 1, 0, 0.5, 0.5), "exa-answer": (3, 1, 2, 1, 0.5, 1.0)}
        assert [
            (row["date"], row["engine"], row["checks"], row["cited"], row["mentioned"], row["failed"])
            for row in stats["daily"]
        ] == [
            ("2026-10-18", "claude-web-search", 1, 1, 1, 0),
            ("2026-10-19", "exa-answer", 1, 0, 0, 1),
            ("2026-10-20", "exa-answer", 2, 1, 2, 0),
        ]
        assert stats["brand_names"] == ["example"]
        assert (stats["consensus"], stats["top_words"]) == ([], [])
        assert [(row["engine"], row["unscored"]) for row in stats["sentiment"]] == [
            ("claude-web-search", 1),
            ("exa-answer", 2),
        ]

    @parameterized.expand(
        [
            ("brand_named_in_the_answer", {"answer_text": "Try Example Analytics for funnels."}, 1),
            ("brand_case_ignored", {"answer_text": "EXAMPLE ships session replay."}, 1),
            ("brand_inside_another_word", {"answer_text": "Counterexample: nothing fits."}, 0),
            ("cited_without_the_name_in_the_text", {"cited": True, "answer_text": "See [1]."}, 1),
            ("failed_check", {"check_failed": True}, 0),
        ]
    )
    def test_stats_count_a_mention_when_the_answer_names_the_brand(
        self, _name: str, check: dict, expected_mentioned: int
    ) -> None:
        prompt_id = self._import_one_prompt()
        self._record_check(prompt_id, "exa-answer", minutes_ago=10, **check)

        with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
            stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()

        assert [row["mentioned"] for row in stats["engines"]] == [expected_mentioned]
        assert [row["mentioned"] for row in stats["daily"]] == [expected_mentioned]

    def test_stats_count_brand_sentiment_per_engine_and_checks_expose_it(self) -> None:
        prompt_id = self._import_one_prompt()
        scored = self._record_check(prompt_id, "exa-answer", minutes_ago=10, answer_text="Example is great.")
        self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example is slow.")
        self._record_check(prompt_id, "exa-answer", minutes_ago=30, answer_text="Example exists.")
        self._record_check(prompt_id, "claude-web-search", minutes_ago=40, answer_text="Nothing about the brand.")
        with team_scope(self.team.pk):
            for text, label, score in (("great", "positive", 0.9), ("slow", "negative", 0.7)):
                AEOCitationCheck.objects.filter(answer_text__contains=text).update(
                    brand_sentiment=label, brand_sentiment_score=score
                )

        with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
            stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()
        checks = self.client.get(f"{self.url}{prompt_id}/checks/?engine=exa-answer").json()["results"]

        assert [
            (row["engine"], row["positive"], row["neutral"], row["negative"], row["unscored"])
            for row in stats["sentiment"]
        ] == [("claude-web-search", 0, 0, 0, 0), ("exa-answer", 1, 0, 1, 1)]
        assert {check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks} == {
            scored: ("positive", 0.9),
            **{check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks[1:]},
        }
        assert [check["brand_sentiment"] for check in checks] == ["positive", "negative", None]

    @time_machine.travel("2026-10-20T12:00:00Z", tick=False)
    def test_stats_rank_engine_consensus_and_search_words_over_the_window(self) -> None:
        prompt_id = self._import_one_prompt()
        day = 24 * 60
        ours = "https://docs.example.com/product-analytics"
        rival = "https://rival.example.org/best-tools"
        self._record_check(
            prompt_id,
            "claude-web-search",
            minutes_ago=10,
            cited=True,
            cited_urls=[rival, ours, "https://rival.example.org/pricing", rival],
            search_queries=["Best product analytics tools", "product analytics for startups"],
        )
        self._record_check(
            prompt_id,
            "claude-web-search",
            minutes_ago=day,
            cited_urls=[rival],
            search_queries=["best product analytics tools"],
        )
        self._record_check(
            prompt_id,
            "openai-web-search",
            minutes_ago=2 * day,
            cited=True,
            cited_urls=["https://community.example.net/thread", ours],
            search_queries=["open source analytics"],
        )
        self._record_check(prompt_id, "exa-answer", minutes_ago=3 * day, cited_urls=[])
        self._record_check(prompt_id, "openai-web-search", minutes_ago=4 * day, check_failed=True)
        self._record_check(prompt_id, "openai-web-search", minutes_ago=40 * day, cited_urls=[rival, rival, rival])

        with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
            stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()

        assert [(row["engine"], row["domain"], row["checks_citing"], row["checks"]) for row in stats["consensus"]] == [
            ("claude-web-search", "docs.example.com", 1, 2),
            ("claude-web-search", "rival.example.org", 2, 2),
            ("claude-web-search", "community.example.net", 0, 2),
            ("exa-answer", "docs.example.com", 0, 1),
            ("exa-answer", "rival.example.org", 0, 1),
            ("exa-answer", "community.example.net", 0, 1),
            ("openai-web-search", "docs.example.com", 1, 1),
            ("openai-web-search", "rival.example.org", 0, 1),
            ("openai-web-search", "community.example.net", 1, 1),
        ]
        assert [(row["word"], row["count"]) for row in stats["top_words"]] == [
            ("analytics", 4),
            ("product", 3),
            ("tools", 2),
            ("open", 1),
            ("source", 1),
            ("startups", 1),
        ]

    @time_machine.travel("2026-10-20T12:00:00Z", tick=False)
    def test_overview_counts_the_set_per_window_engine_and_topic(self) -> None:
        ids = self._import_prompts(
            [
                {"prompt": "funnels?", "topic": "Analytics"},
                {"prompt": "retention?", "topic": "Analytics"},
                {"prompt": "replay?", "topic": ""},
                {"prompt": "flags?", "topic": "Flags"},
                {"prompt": "surveys?", "topic": "Surveys"},
            ]
        )
        self.client.patch(f"{self.url}{ids['retention?']}/", {"active": False}, format="json")
        self.client.delete(f"{self.url}{ids['flags?']}/")
        day = 24 * 60
        ours = "https://docs.example.com/funnels"
        rival = "https://rival.example.org/funnels"
        self._record_check(ids["funnels?"], "claude-web-search", minutes_ago=10, cited=True, cited_urls=[ours, rival])
        self._record_check(ids["funnels?"], "openai-web-search", minutes_ago=10, check_failed=True)
        self._record_check(ids["retention?"], "exa-answer", minutes_ago=day, answer_text="Example has retention.")
        self._record_check(ids["replay?"], "claude-web-search", minutes_ago=2 * day, cited_urls=[rival, rival])
        self._record_check(ids["flags?"], "exa-answer", minutes_ago=3 * day, cited=True, cited_urls=[ours])
        self._record_check(ids["funnels?"], "claude-web-search", minutes_ago=35 * day)
        self._record_check(ids["replay?"], "exa-answer", minutes_ago=40 * day, cited=True)
        self._record_check(ids["funnels?"], "claude-web-search", minutes_ago=70 * day, cited=True)

        with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
            response = self.client.get(f"{self.url}overview/")

        assert response.status_code == status.HTTP_200_OK, response.json()
        overview = response.json()
        assert (overview["days"], overview["window_from"], overview["window_to"]) == (30, "2026-09-21", "2026-10-20")
        assert overview["prompts_tracked"] == 3
        assert (overview["target_domains"], overview["brand_names"]) == (["example.com"], ["example"])
        assert overview["current"] == {
            "checks": 5,
            "failed": 1,
            "cited": 2,
            "mentioned": 3,
            "prompts": 4,
            "prompts_cited": 2,
            "citation_rate": 0.5,
            "mention_rate": 0.75,
        }
        assert overview["previous"] == {
            "checks": 2,
            "failed": 0,
            "cited": 1,
            "mentioned": 1,
            "prompts": 2,
            "prompts_cited": 1,
            "citation_rate": 0.5,
            "mention_rate": 0.5,
        }
        assert [
            (row["engine"], row["checks"], row["failed"], row["cited"], row["mentioned"], row["prompts"])
            for row in overview["engines"]
        ] == [("claude-web-search", 2, 0, 1, 1, 2), ("exa-answer", 2, 0, 1, 2, 2), ("openai-web-search", 1, 1, 0, 0, 1)]
        assert [
            (
                row["topic"],
                row["active_prompts"],
                row["checks"],
                row["failed"],
                row["cited"],
                row["mentioned"],
                row["mention_rate"],
            )
            for row in overview["topics"]
        ] == [
            ("", 1, 1, 0, 0, 0, 0.0),
            ("Analytics", 1, 3, 1, 1, 2, 1.0),
            ("Flags", 0, 1, 0, 1, 1, 1.0),
            ("Surveys", 1, 0, 0, 0, 0, None),
        ]
        assert [
            (row["domain"], row["answers"], row["share_of_answers"], row["is_target"])
            for row in overview["top_domains"]
        ] == [
            ("docs.example.com", 2, 0.5, True),
            ("rival.example.org", 2, 0.5, False),
        ]
        child = Team.objects.create(organization=self.organization, name="child env", parent_team=self.team)
        via_child = get_prompt_set_overview(child.id, timezone_name="UTC", days=7)
        assert [domain.domain for domain in via_child.top_domains] == ["docs.example.com", "rival.example.org"]

    def test_overview_rejects_a_window_outside_the_bounds(self) -> None:
        assert self.client.get(f"{self.url}overview/?days=0").status_code == status.HTTP_400_BAD_REQUEST

    @parameterized.expand([("above_range", 140), ("not_a_number", "NaN"), ("infinite", "Infinity")])
    def test_bulk_import_rejects_invalid_visibility(self, _name: str, visibility: object) -> None:
        response = self.client.post(
            f"{self.url}bulk_import/",
            {"prompts": [{"prompt": "q?", "external_visibility": visibility}]},
            format="json",
        )

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert not AEOPrompt.objects.for_team(self.team.id).exists()

    @parameterized.expand(
        [
            ("write_scope", ["web_analytics:write"], status.HTTP_200_OK),
            ("read_scope_only", ["web_analytics:read"], status.HTTP_403_FORBIDDEN),
        ]
    )
    def test_bulk_import_with_a_personal_api_key(self, _name: str, scopes: list[str], expected: int) -> None:
        # Session auth skips scope checks, so only a scoped key catches a custom action left out of the write actions.
        value = generate_random_token_personal()
        PersonalAPIKey.objects.create(
            user=self.user, label="aeo import", secure_value=hash_key_value(value), scopes=scopes
        )
        self.client.logout()

        response = self.client.post(
            f"{self.url}bulk_import/",
            {"prompts": [{"prompt": "Best session replay tool?"}]},
            format="json",
            headers={"authorization": f"Bearer {value}"},
        )

        assert response.status_code == expected, response.json()

    @parameterized.expand(
        [
            ("answer", True, "completed", ["Best session replay tool?"], 1),
            ("paused", True, "completed", ["Best session replay tool?"], 2),
            ("cut_off", True, "completed", ["Best session replay tool?"], 1),
            ("timeout", True, "failed", [], 1),
            ("answer", False, None, [], 0),
        ]
    )
    def test_suggest_runs_as_a_job_and_never_saves_prompts(
        self, outcome: str, ai_approved: bool, final_status: str | None, expected_prompts: list[str], calls: int
    ) -> None:
        self.organization.is_ai_data_processing_approved = ai_approved
        self.organization.save(update_fields=["is_ai_data_processing_approved"])
        self.client.post(
            f"{self.url}bulk_import/",
            {"prompts": [{"prompt": "Tracked already?"}, {"prompt": "Old wording?"}]},
            format="json",
        )
        old_id = next(r["id"] for r in self.client.get(self.url).json()["results"] if r["prompt"] == "Old wording?")
        self.client.patch(f"{self.url}{old_id}/", {"prompt": "New wording?"}, format="json")
        items = [
            {"prompt": p, "topic": "Replay", "circle": "battleground", "funnel_stage": "bottom", "branded": False}
            for p in ["Best session replay tool?", "Tracked already?", "Old wording?"]
        ]
        paused = MagicMock(status_code=200)
        paused.json.return_value = {"stop_reason": "pause_turn", "content": [{"type": "text", "text": "Reading."}]}
        answered = MagicMock(status_code=200)
        answered.json.return_value = {
            "stop_reason": "end_turn",
            "content": [{"type": "text", "text": json.dumps(items)}],
        }
        # The token limit cut the array mid-way through its second object.
        cut_off = MagicMock(status_code=200)
        cut_off.json.return_value = {
            "stop_reason": "max_tokens",
            "content": [{"type": "text", "text": json.dumps(items[:1])[:-1] + ', {"prompt": "Half'}],
        }
        post = MagicMock(
            side_effect={
                "answer": [answered],
                "paused": [paused, answered],
                "cut_off": [cut_off],
                "timeout": requests.Timeout(),
            }[outcome]
        )
        gateway = MagicMock(url="http://gateway", api_key="test-key")

        with (
            patch("products.aeo.backend.suggestions.resolve_ai_gateway_config", return_value=gateway),
            patch("products.aeo.backend.suggestions.requests.Session.post", post),
        ):
            started = self.client.post(f"{self.url}suggest/", {"domain": "example.com"}, format="json")

        assert post.call_count == calls
        assert AEOPrompt.objects.for_team(self.team.id).count() == 2
        if final_status is None:
            assert started.status_code == status.HTTP_403_FORBIDDEN
            return
        assert started.status_code == status.HTTP_202_ACCEPTED, started.json()
        assert started.json()["status"] == "pending"
        job = self.client.get(f"{self.url}suggest/{started.json()['id']}/").json()
        assert job["status"] == final_status
        assert [s["prompt"] for s in job["suggestions"]] == expected_prompts
        if outcome == "paused":
            resumed = post.call_args_list[1].kwargs["json"]["messages"]
            assert resumed[-1] == {"role": "assistant", "content": [{"type": "text", "text": "Reading."}]}

    def test_a_pending_job_past_its_deadline_reads_as_failed(self) -> None:
        with patch("products.aeo.backend.facade.api.suggest_aeo_prompts_task.delay"):
            started = self.client.post(f"{self.url}suggest/", {"domain": "example.com"}, format="json").json()

        with time_machine.travel(timezone.now() + dt.timedelta(minutes=9), tick=False):
            job = self.client.get(f"{self.url}suggest/{started['id']}/").json()

        assert job["status"] == "failed"
        assert job["error"] == "Prompt suggestions stopped before they finished. Try again."

    def test_import_records_that_a_prompt_came_from_a_suggestion(self) -> None:
        self.client.post(
            f"{self.url}bulk_import/",
            {"prompts": [{"prompt": "Suggested?", "origin": "suggestion"}, {"prompt": "Written?"}]},
            format="json",
        )

        evidence = {p.prompt: p.evidence for p in AEOPrompt.objects.for_team(self.team.id)}
        assert evidence["Suggested?"]["via"] == "suggestion"
        assert evidence["Written?"]["via"] == "api"

    def test_suggestion_get_returns_404_for_an_unknown_job(self) -> None:
        response = self.client.get(f"{self.url}suggest/{uuid.uuid4()}/")
        assert response.status_code == status.HTTP_404_NOT_FOUND