Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.9 KB
Raw Blame
import uuid
import datetime as dt

import time_machine
from posthog.test.base import BaseTest

from django.test import override_settings
from django.utils import timezone

from parameterized import parameterized

from products.aeo.backend.facade.api import import_prompts, list_citation_gaps, update_prompt
from products.aeo.backend.facade.contracts import PromptImport, PromptUpdate
from products.aeo.backend.models import AEOCitationCheck, AEOPrompt


class TestListCitationGaps(BaseTest):
    def _prompt(self, text: str, *, active: bool = True) -> AEOPrompt:
        return AEOPrompt.objects.for_team(self.team.id).create(
            team=self.team, prompt=text, prompt_hash=text, prompt_source=AEOPrompt.Source.MANUAL, active=active
        )

    def _check(
        self,
        prompt: AEOPrompt,
        engine: str,
        *,
        cited_urls: list[str],
        failed: bool = False,
        answer_text: str | None = None,
        queries: list[str] | None = None,
        version: int | None = None,
    ) -> None:
        target_urls = [url for url in cited_urls if "posthog.com" in url]
        AEOCitationCheck.objects.for_team(self.team.id).create(
            team=self.team,
            prompt=prompt,
            run_id=uuid.uuid4(),
            prompt_text=prompt.prompt,
            prompt_source=prompt.prompt_source,
            prompt_hash=prompt.prompt_hash,
            prompt_version=version,
            engine=engine,
            model=engine,
            check_failed=failed,
            cited=bool(target_urls) and not failed,
            cited_urls=cited_urls,
            target_urls=target_urls,
            search_queries=queries or [],
            answer_text=answer_text,
        )

    def test_returns_only_uncited_active_prompts_with_competitor_evidence(self) -> None:
        gap = self._prompt("best session replay tool")
        self._check(
            gap,
            "claude-web-search",
            cited_urls=["https://rival.example/replay", "https://other.example/a"],
            answer_text="Rival is the best, though PostHog also records sessions.",
            queries=["session replay tools"],
        )
        self._check(
            gap,
            "openai-web-search",
            cited_urls=["https://rival.example/replay", "https://posthog.com/session-replay"],
            queries=["session replay tools", "open source replay"],
        )
        self._check(gap, "exa-answer", cited_urls=[], failed=True)

        always_cited = self._prompt("posthog pricing")
        self._check(always_cited, "claude-web-search", cited_urls=["https://posthog.com/pricing"])

        only_failed = self._prompt("engine outage")
        self._check(only_failed, "claude-web-search", cited_urls=[], failed=True)

        inactive = self._prompt("retired prompt", active=False)
        self._check(inactive, "claude-web-search", cited_urls=["https://rival.example/x"])

        gaps = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))

        assert [g.prompt_text for g in gaps] == ["best session replay tool"]
        result = gaps[0]
        assert result.checks == 2
        assert result.cited_checks == 1
        assert result.mentioned_checks == 2
        assert result.engines == ("claude-web-search", "openai-web-search")
        assert result.engines_not_citing == ("claude-web-search",)
        assert result.competitor_urls[0] == "https://rival.example/replay"
        assert "https://posthog.com/session-replay" not in result.competitor_urls
        assert result.competitor_domains[0] == "rival.example"
        assert result.our_cited_urls == ("https://posthog.com/session-replay",)
        assert result.engine_search_queries[0] == "session replay tools"
        assert [a.answer_text for a in result.latest_answers] == [
            "Rival is the best, though PostHog also records sessions."
        ]

    @parameterized.expand(
        [
            ("registrable_name_under_a_multi_part_suffix", "Example Analytics records every session.", 1),
            ("public_suffix_label_is_not_the_brand", "Rival Co records every session.", 0),
        ]
    )
    def test_counts_brand_mentions_by_the_registrable_domain_name(
        self, _name: str, answer_text: str, mentioned_checks: int
    ) -> None:
        prompt = self._prompt("best session replay tool")
        self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/replay"], answer_text=answer_text)

        with override_settings(AEO_TARGET_DOMAINS=["example.co.uk"]):
            gaps = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))

        assert gaps[0].mentioned_checks == mentioned_checks

    def test_ignores_checks_before_the_window(self) -> None:
        prompt = self._prompt("old gap")
        with time_machine.travel(timezone.now() - dt.timedelta(days=30), tick=False):
            self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/x"])

        assert list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14)) == []

    def test_text_edit_keeps_the_gap_key_and_counts_only_the_current_version(self) -> None:
        import_prompts(self.team.id, [PromptImport(prompt="Best replay tool?")])
        prompt = AEOPrompt.objects.for_team(self.team.id).get()
        first_key = prompt.prompt_hash
        self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/old"])

        update_prompt(self.team.id, str(prompt.id), PromptUpdate(prompt="Best session replay tool?"), user_id=None)
        prompt.refresh_from_db()
        self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/new"], version=2)

        [gap] = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))

        assert (gap.prompt_hash, gap.prompt_text, gap.checks) == (first_key, "Best session replay tool?", 1)
        assert gap.competitor_urls == ("https://rival.example/new",)

    def test_reimporting_an_edited_prompts_old_text_keeps_gap_keys_unique(self) -> None:
        import_prompts(self.team.id, [PromptImport(prompt="Best replay tool?")])
        edited = AEOPrompt.objects.for_team(self.team.id).get()
        update_prompt(self.team.id, str(edited.id), PromptUpdate(prompt="Best session replay tool?"), user_id=None)
        import_prompts(self.team.id, [PromptImport(prompt="Best replay tool?")])
        edited.refresh_from_db()
        reimported = AEOPrompt.objects.for_team(self.team.id).exclude(id=edited.id).get()
        self._check(edited, "claude-web-search", cited_urls=["https://rival.example/a"], version=2)
        self._check(reimported, "claude-web-search", cited_urls=["https://rival.example/b"])

        gaps = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))

        assert {gap.prompt_id: gap.prompt_hash for gap in gaps} == {
            str(edited.id): edited.prompt_hash,
            str(reimported.id): reimported.prompt_hash,
        }