:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import uuid
import datetime as dt
import time_machine
from posthog.test.base import BaseTest
from django.test import override_settings
from django.utils import timezone
from parameterized import parameterized
from products.aeo.backend.facade.api import import_prompts, list_citation_gaps, update_prompt
from products.aeo.backend.facade.contracts import PromptImport, PromptUpdate
from products.aeo.backend.models import AEOCitationCheck, AEOPrompt
class TestListCitationGaps(BaseTest):
def _prompt(self, text: str, *, active: bool = True) -> AEOPrompt:
return AEOPrompt.objects.for_team(self.team.id).create(
team=self.team, prompt=text, prompt_hash=text, prompt_source=AEOPrompt.Source.MANUAL, active=active
)
def _check(
self,
prompt: AEOPrompt,
engine: str,
*,
cited_urls: list[str],
failed: bool = False,
answer_text: str | None = None,
queries: list[str] | None = None,
version: int | None = None,
) -> None:
target_urls = [url for url in cited_urls if "posthog.com" in url]
AEOCitationCheck.objects.for_team(self.team.id).create(
team=self.team,
prompt=prompt,
run_id=uuid.uuid4(),
prompt_text=prompt.prompt,
prompt_source=prompt.prompt_source,
prompt_hash=prompt.prompt_hash,
prompt_version=version,
engine=engine,
model=engine,
check_failed=failed,
cited=bool(target_urls) and not failed,
cited_urls=cited_urls,
target_urls=target_urls,
search_queries=queries or [],
answer_text=answer_text,
)
def test_returns_only_uncited_active_prompts_with_competitor_evidence(self) -> None:
gap = self._prompt("best session replay tool")
self._check(
gap,
"claude-web-search",
cited_urls=["https://rival.example/replay", "https://other.example/a"],
answer_text="Rival is the best, though PostHog also records sessions.",
queries=["session replay tools"],
)
self._check(
gap,
"openai-web-search",
cited_urls=["https://rival.example/replay", "https://posthog.com/session-replay"],
queries=["session replay tools", "open source replay"],
)
self._check(gap, "exa-answer", cited_urls=[], failed=True)
always_cited = self._prompt("posthog pricing")
self._check(always_cited, "claude-web-search", cited_urls=["https://posthog.com/pricing"])
only_failed = self._prompt("engine outage")
self._check(only_failed, "claude-web-search", cited_urls=[], failed=True)
inactive = self._prompt("retired prompt", active=False)
self._check(inactive, "claude-web-search", cited_urls=["https://rival.example/x"])
gaps = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))
assert [g.prompt_text for g in gaps] == ["best session replay tool"]
result = gaps[0]
assert result.checks == 2
assert result.cited_checks == 1
assert result.mentioned_checks == 2
assert result.engines == ("claude-web-search", "openai-web-search")
assert result.engines_not_citing == ("claude-web-search",)
assert result.competitor_urls[0] == "https://rival.example/replay"
assert "https://posthog.com/session-replay" not in result.competitor_urls
assert result.competitor_domains[0] == "rival.example"
assert result.our_cited_urls == ("https://posthog.com/session-replay",)
assert result.engine_search_queries[0] == "session replay tools"
assert [a.answer_text for a in result.latest_answers] == [
"Rival is the best, though PostHog also records sessions."
]
@parameterized.expand(
[
("registrable_name_under_a_multi_part_suffix", "Example Analytics records every session.", 1),
("public_suffix_label_is_not_the_brand", "Rival Co records every session.", 0),
]
)
def test_counts_brand_mentions_by_the_registrable_domain_name(
self, _name: str, answer_text: str, mentioned_checks: int
) -> None:
prompt = self._prompt("best session replay tool")
self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/replay"], answer_text=answer_text)
with override_settings(AEO_TARGET_DOMAINS=["example.co.uk"]):
gaps = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))
assert gaps[0].mentioned_checks == mentioned_checks
def test_ignores_checks_before_the_window(self) -> None:
prompt = self._prompt("old gap")
with time_machine.travel(timezone.now() - dt.timedelta(days=30), tick=False):
self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/x"])
assert list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14)) == []
def test_text_edit_keeps_the_gap_key_and_counts_only_the_current_version(self) -> None:
import_prompts(self.team.id, [PromptImport(prompt="Best replay tool?")])
prompt = AEOPrompt.objects.for_team(self.team.id).get()
first_key = prompt.prompt_hash
self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/old"])
update_prompt(self.team.id, str(prompt.id), PromptUpdate(prompt="Best session replay tool?"), user_id=None)
prompt.refresh_from_db()
self._check(prompt, "claude-web-search", cited_urls=["https://rival.example/new"], version=2)
[gap] = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))
assert (gap.prompt_hash, gap.prompt_text, gap.checks) == (first_key, "Best session replay tool?", 1)
assert gap.competitor_urls == ("https://rival.example/new",)
def test_reimporting_an_edited_prompts_old_text_keeps_gap_keys_unique(self) -> None:
import_prompts(self.team.id, [PromptImport(prompt="Best replay tool?")])
edited = AEOPrompt.objects.for_team(self.team.id).get()
update_prompt(self.team.id, str(edited.id), PromptUpdate(prompt="Best session replay tool?"), user_id=None)
import_prompts(self.team.id, [PromptImport(prompt="Best replay tool?")])
edited.refresh_from_db()
reimported = AEOPrompt.objects.for_team(self.team.id).exclude(id=edited.id).get()
self._check(edited, "claude-web-search", cited_urls=["https://rival.example/a"], version=2)
self._check(reimported, "claude-web-search", cited_urls=["https://rival.example/b"])
gaps = list_citation_gaps(self.team.id, since=timezone.now() - dt.timedelta(days=14))
assert {gap.prompt_id: gap.prompt_hash for gap in gaps} == {
str(edited.id): edited.prompt_hash,
str(reimported.id): reimported.prompt_hash,
}