:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aeo): report sources, engines and sentiment on the prompt detail (#122978)
Lucas Ricoy36e8e6f648f0
diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambrindex 3077b9a516f5..0908cc96bd17 100644--- a/posthog/hogql/database/test/__snapshots__/test_database.ambr+++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr@@ -2889,6 +2889,26 @@"table": null,"type": "json"},+ "brand_sentiment": {+ "chain": null,+ "fields": null,+ "hogql_value": "brand_sentiment",+ "id": null,+ "name": "brand_sentiment",+ "schema_valid": true,+ "table": null,+ "type": "string"+ },+ "brand_sentiment_score": {+ "chain": null,+ "fields": null,+ "hogql_value": "brand_sentiment_score",+ "id": null,+ "name": "brand_sentiment_score",+ "schema_valid": true,+ "table": null,+ "type": "float"+ },"cost_usd": {"chain": null,"fields": null,@@ -17537,6 +17557,26 @@"table": null,"type": "json"},+ "brand_sentiment": {+ "chain": null,+ "fields": null,+ "hogql_value": "brand_sentiment",+ "id": null,+ "name": "brand_sentiment",+ "schema_valid": true,+ "table": null,+ "type": "string"+ },+ "brand_sentiment_score": {+ "chain": null,+ "fields": null,+ "hogql_value": "brand_sentiment_score",+ "id": null,+ "name": "brand_sentiment_score",+ "schema_valid": true,+ "table": null,+ "type": "float"+ },"cost_usd": {"chain": null,"fields": null,diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.pyindex 0b5c6eebd33d..5328e5c6cd84 100644--- a/posthog/management/commands/start_temporal_worker.py+++ b/posthog/management/commands/start_temporal_worker.py@@ -156,6 +156,10 @@ from posthog.temporal.weekly_digest import (WORKFLOWS as WEEKLY_DIGEST_WORKFLOWS,)+from products.aeo.backend.facade.temporal import (+ ACTIVITIES as AEO_BRAND_SENTIMENT_ACTIVITIES,+ WORKFLOWS as AEO_BRAND_SENTIMENT_WORKFLOWS,+)from products.alerts.backend.facade.temporal import (PLATFORM_AI_QUEUE_ACTIVITIES as INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES,PLATFORM_EVALUATION_ACTIVITIES as INSIGHT_PLATFORM_EVALUATION_ACTIVITIES,@@ -580,9 +584,10 @@ _task_queue_specs = [WEEKLY_DIGEST_ACTIVITIES + WA_DIGEST_ACTIVITIES + DATA_CATALOG_DIGEST_ACTIVITIES,),(+ # AEO brand sentiment rides this queue because its image is the one that carries the ONNX sentiment model.settings.LLMA_EVALS_TASK_QUEUE,- LLM_ANALYTICS_EVAL_WORKFLOWS + LLM_ANALYTICS_TAGGER_WORKFLOWS,- LLM_ANALYTICS_EVAL_ACTIVITIES + LLM_ANALYTICS_TAGGER_ACTIVITIES,+ LLM_ANALYTICS_EVAL_WORKFLOWS + LLM_ANALYTICS_TAGGER_WORKFLOWS + AEO_BRAND_SENTIMENT_WORKFLOWS,+ LLM_ANALYTICS_EVAL_ACTIVITIES + LLM_ANALYTICS_TAGGER_ACTIVITIES + AEO_BRAND_SENTIMENT_ACTIVITIES,),(settings.LLMA_TASK_QUEUE,diff --git a/posthog/settings/web.py b/posthog/settings/web.pyindex ff605a97f505..fd90723bdcca 100644--- a/posthog/settings/web.py+++ b/posthog/settings/web.py@@ -1467,6 +1467,8 @@ MCP_STORE_SLACK_DEV_ALLOWED_TEAM_IDS = get_list(get_from_env("MCP_STORE_SLACK_DE# teams in this allowlist AND with the `aeo-citation-tracking` flag enabled.AEO_CITATION_TEAM_IDS = get_list(get_from_env("AEO_CITATION_TEAM_IDS", ""))AEO_TARGET_DOMAINS = get_list(get_from_env("AEO_TARGET_DOMAINS", "posthog.com"))+# Scores brand sentiment on the evals Temporal queue after each run. Off until that worker carries the model.+AEO_BRAND_SENTIMENT_ENABLED = get_from_env("AEO_BRAND_SENTIMENT_ENABLED", False, type_cast=str_to_bool)AEO_ANTHROPIC_MODEL = get_from_env("AEO_ANTHROPIC_MODEL", "claude-sonnet-5")AEO_OPENAI_MODEL = get_from_env("AEO_OPENAI_MODEL", "gpt-5")EXA_API_KEY = get_from_env("EXA_API_KEY", "")diff --git a/products/aeo/backend/brand.py b/products/aeo/backend/brand.pynew file mode 100644index 000000000000..87f339740e39--- /dev/null+++ b/products/aeo/backend/brand.py@@ -0,0 +1,47 @@+"""How an answer is judged to name the brand.++The brand names are the registrable names of the target domains (`example` for+`docs.example.co.uk`), so the same list drives the SQL mention filter, the brand+regex the sentiment scorer uses, and the names the frontend builds its HogQL+mention filter from.+"""++from __future__ import annotations++import re++from django.db.models import Q++import tldextract++_DOMAIN_PARTS = tldextract.TLDExtract(suffix_list_urls=())+++def brand_names(target_domains: list[str]) -> tuple[str, ...]:+ """Lower-cased registrable names of the target domains, sorted, without duplicates."""+ return tuple(sorted({(_DOMAIN_PARTS(domain).domain or domain).lower() for domain in target_domains if domain}))+++def brand_regex(target_domains: list[str]) -> str | None:+ """Postgres regex (word-bounded, for `iregex`) that matches any brand name. None without a brand."""+ names = brand_names(target_domains)+ if not names:+ return None+ return r"\m(" + "|".join(re.escape(name) for name in names) + r")\M"+++def brand_pattern(target_domains: list[str]) -> re.Pattern[str] | None:+ """Python equivalent of `brand_regex`, for text already loaded from the database."""+ names = brand_names(target_domains)+ if not names:+ return None+ return re.compile(r"\b(" + "|".join(re.escape(name) for name in names) + r")\b", re.IGNORECASE)+++def mention_filter(target_domains: list[str]) -> Q:+ """Checks whose answer names the brand. A cited answer counts as a mention even when the+ text only carries the URL, so the mention rate is never below the citation rate."""+ mention = Q(cited=True)+ if regex := brand_regex(target_domains):+ mention |= Q(answer_text__iregex=regex)+ return mentiondiff --git a/products/aeo/backend/engines.py b/products/aeo/backend/engines.pyindex e8706d4eeadb..db345841f8d2 100644--- a/products/aeo/backend/engines.py+++ b/products/aeo/backend/engines.py@@ -245,13 +245,19 @@ def parse_exa_citations(body: dict[str, Any]) -> ParsedAnswer:)+def is_target_host(host: str, target_domains: list[str]) -> bool:+ """True when the host is one of the target domains or a subdomain of it."""+ host = host.lower()+ return any(host == domain or host.endswith(f".{domain}") for domain in (d.lower() for d in target_domains))++def is_target_url(url: str, target_domains: list[str]) -> bool:"""True when the URL's host is one of the target domains or a subdomain of it."""try:- host = (urlparse(url).hostname or "").lower()+ host = urlparse(url).hostname or ""except ValueError:return False- return any(host == domain or host.endswith(f".{domain}") for domain in (d.lower() for d in target_domains))+ return is_target_host(host, target_domains)def target_position(cited_urls: list[str], target_domains: list[str]) -> int | None:diff --git a/products/aeo/backend/facade/api.py b/products/aeo/backend/facade/api.pyindex af9ffa63ff4f..3b759994f85d 100644--- a/products/aeo/backend/facade/api.py+++ b/products/aeo/backend/facade/api.py@@ -6,22 +6,25 @@ capabilities. Celery wiring is re-exported from facade/tasks.py.from __future__ import annotations+import reimport uuidimport datetime as dt-from collections import defaultdict+from collections import Counter, defaultdictfrom typing import Anyfrom zoneinfo import ZoneInfo+from django.conf import settingsfrom django.core.exceptions import ValidationErrorfrom django.db import IntegrityError, transaction-from django.db.models import Count, Max, Min, Q+from django.db.models import Count, Max, Min, Q, QuerySetfrom django.db.models.functions import TruncDatefrom django.utils import timezonefrom posthog.models.team import Teamfrom products.aeo.backend import gaps-from products.aeo.backend.engines import MAX_PROMPT_LENGTH+from products.aeo.backend.brand import brand_names, mention_filter+from products.aeo.backend.engines import MAX_PROMPT_LENGTH, top_domainsfrom products.aeo.backend.facade.contracts import (PROMPT_TOPIC_MAX_LENGTH,CitationGap,@@ -31,10 +34,13 @@ from products.aeo.backend.facade.contracts import (PromptCheck,PromptDailyEngineStats,PromptEngineCheck,+ PromptEngineDomainShare,+ PromptEngineSentiment,PromptEngineStats,PromptImport,PromptImportResult,PromptNotFoundError,+ PromptSearchWord,PromptStats,PromptTextConflictError,PromptTopic,@@ -182,12 +188,17 @@ def list_prompt_checks(def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days: int) -> PromptStats:- """Totals per engine over the prompt's whole history, plus per-day counts for the last `days` days+ """Totals per engine over the prompt's whole history, plus per-day counts, engine consensus on+ the cited domains, brand sentiment and the words of the web searches over the last `days` daysin the team's timezone."""prompt = _get_prompt_row(team_id, prompt_id)+ target_domains: list[str] = settings.AEO_TARGET_DOMAINSchecks = AEOCitationCheck.objects.for_team(team_id).filter(prompt_id=prompt.id)+ # `mentioned` filters on the `cited` column, and an annotation named `cited` would shadow that+ # column for every later annotation in the same call, so it must come first.counts = {"checks": Count("id"),+ "mentioned": Count("id", filter=Q(check_failed=False) & mention_filter(target_domains)),"cited": Count("id", filter=Q(cited=True)),"failed": Count("id", filter=Q(check_failed=True)),}@@ -197,6 +208,7 @@ def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days:engine=row["engine"],checks=row["checks"],cited=row["cited"],+ mentioned=row["mentioned"],failed=row["failed"],first_checked_at=row["first_checked_at"],last_checked_at=row["last_checked_at"],@@ -210,26 +222,117 @@ def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days:daily_to = timezone.now().astimezone(tz).date()daily_from = daily_to - dt.timedelta(days=days - 1)window_start = dt.datetime.combine(daily_from, dt.time.min, tzinfo=tz)+ in_window = checks.filter(created_at__gte=window_start)daily = tuple(PromptDailyEngineStats(- date=row["date"], engine=row["engine"], checks=row["checks"], cited=row["cited"], failed=row["failed"]+ date=row["date"],+ engine=row["engine"],+ checks=row["checks"],+ cited=row["cited"],+ mentioned=row["mentioned"],+ failed=row["failed"],)- for row in checks.filter(created_at__gte=window_start)- .annotate(date=TruncDate("created_at", tzinfo=tz))+ for row in in_window.annotate(date=TruncDate("created_at", tzinfo=tz)).values("date", "engine").annotate(**counts).order_by("date", "engine"))+ answered = in_window.filter(check_failed=False)+ sentiment = tuple(+ PromptEngineSentiment(+ engine=row["engine"],+ positive=row["positive"],+ neutral=row["neutral"],+ negative=row["negative"],+ unscored=row["unscored"],+ )+ for row in answered.values("engine")+ .annotate(+ positive=Count("id", filter=Q(brand_sentiment="positive")),+ neutral=Count("id", filter=Q(brand_sentiment="neutral")),+ negative=Count("id", filter=Q(brand_sentiment="negative")),+ unscored=Count("id", filter=Q(brand_sentiment__isnull=True) & mention_filter(target_domains)),+ )+ .order_by("engine")+ )+ report = _PromptWindowReport.from_checks(answered)return PromptStats(total_checks=sum(engine.checks for engine in engines),+ brand_names=brand_names(target_domains),engines=engines,daily_from=daily_from,daily_to=daily_to,daily=daily,+ consensus=report.consensus(),+ sentiment=sentiment,+ top_words=report.top_words(),)+MAX_CONSENSUS_DOMAINS = 10+MAX_STATS_WORDS = 20+MIN_SEARCH_WORD_LENGTH = 3+_WORD = re.compile(r"[^\W_]+", re.UNICODE)+SEARCH_STOP_WORDS = frozenset(+ """a an and are as at be best by can do does for from how i in is it its my of on or that the this to top vs+ what which who with without your you""".split()+)+++class _PromptWindowReport:+ """Cited domains and web searches across one prompt's answered checks in the window.++ Counts are per check: a domain cited twice in one answer counts once, so every number reads as+ "how many answers", the same unit as the citation rate."""++ def __init__(self) -> None:+ self._domain_checks: Counter[str] = Counter()+ self._engine_checks: Counter[str] = Counter()+ self._engine_domain_checks: Counter[tuple[str, str]] = Counter()+ self._queries: Counter[str] = Counter()++ @classmethod+ def from_checks(cls, answered: QuerySet[AEOCitationCheck]) -> _PromptWindowReport:+ report = cls()+ for row in answered.values("engine", "cited_urls", "search_queries"):+ report.add(row["engine"], _strings(row["cited_urls"]), _strings(row["search_queries"]))+ return report++ def add(self, engine: str, cited_urls: tuple[str, ...], search_queries: tuple[str, ...]) -> None:+ self._engine_checks[engine] += 1+ domains = top_domains(list(cited_urls))+ self._domain_checks.update(domains)+ self._engine_domain_checks.update((engine, domain) for domain in domains)+ self._queries.update(key for key in (" ".join(query.casefold().split()) for query in search_queries) if key)++ def consensus(self) -> tuple[PromptEngineDomainShare, ...]:+ domains = [domain for domain, _ in self._ranked(self._domain_checks, MAX_CONSENSUS_DOMAINS)]+ return tuple(+ PromptEngineDomainShare(+ engine=engine,+ domain=domain,+ checks_citing=self._engine_domain_checks[(engine, domain)],+ checks=checks,+ )+ for engine, checks in sorted(self._engine_checks.items())+ for domain in domains+ )++ def top_words(self) -> tuple[PromptSearchWord, ...]:+ words: Counter[str] = Counter()+ for key, count in self._queries.items():+ for word in set(_WORD.findall(key)):+ if len(word) >= MIN_SEARCH_WORD_LENGTH and word not in SEARCH_STOP_WORDS:+ words[word] += count+ return tuple(PromptSearchWord(word=word, count=count) for word, count in self._ranked(words, MAX_STATS_WORDS))++ @staticmethod+ def _ranked(counter: Counter[str], limit: int) -> list[tuple[str, int]]:+ # Ties break on the value, so the order is stable between two requests.+ return sorted(counter.items(), key=lambda item: (-item[1], item[0]))[:limit]++def import_prompts(team_id: int, prompts: list[PromptImport]) -> PromptImportResult:"""Add prompts to the team's set, or update the topic and evidence ofprompts that already exist. Existing prompts keep their active state."""@@ -373,6 +476,8 @@ _CHECK_FIELDS = ("search_queries","answer_text","prompt_version",+ "brand_sentiment",+ "brand_sentiment_score",)@@ -397,6 +502,8 @@ def _to_prompt_check(row: dict[str, Any]) -> PromptCheck:search_queries=_strings(row["search_queries"]),answer_text=row["answer_text"],prompt_version=row["prompt_version"],+ brand_sentiment=row["brand_sentiment"],+ brand_sentiment_score=row["brand_sentiment_score"],)diff --git a/products/aeo/backend/facade/contracts.py b/products/aeo/backend/facade/contracts.pyindex 21dff934d5de..7bf4131837ef 100644--- a/products/aeo/backend/facade/contracts.py+++ b/products/aeo/backend/facade/contracts.py@@ -114,6 +114,9 @@ class PromptCheck:search_queries: tuple[str, ...]answer_text: str | Noneprompt_version: int | None+ # Sentiment of the sentences that name the brand. None until scored, and when the brand is not named.+ brand_sentiment: str | None = None+ brand_sentiment_score: float | None = None@frozen@@ -121,6 +124,8 @@ class PromptEngineStats:engine: strchecks: intcited: int+ # Answers that name the brand, cited ones included: the same definition the citation gaps use.+ mentioned: intfailed: intfirst_checked_at: dt.datetimelast_checked_at: dt.datetime@@ -130,6 +135,11 @@ class PromptEngineStats:answered = self.checks - self.failedreturn self.cited / answered if answered else None+ @property+ def mention_rate(self) -> float | None:+ answered = self.checks - self.failed+ return self.mentioned / answered if answered else None+@frozenclass PromptDailyEngineStats:@@ -137,16 +147,50 @@ class PromptDailyEngineStats:engine: strchecks: intcited: int+ mentioned: intfailed: int+@frozen+class PromptEngineDomainShare:+ """How many of one engine's answered checks in the window cited a domain."""++ engine: str+ domain: str+ checks_citing: int+ checks: int+++@frozen+class PromptEngineSentiment:+ """Brand sentiment of one engine's answered checks in the window. `unscored` answers name the+ brand but have no sentiment yet."""++ engine: str+ positive: int+ neutral: int+ negative: int+ unscored: int+++@frozen+class PromptSearchWord:+ word: str+ count: int++@frozenclass PromptStats:total_checks: int+ # The names an answer must contain to count as a mention, so a client can filter answers the same way.+ brand_names: tuple[str, ...]engines: tuple[PromptEngineStats, ...]daily_from: dt.datedaily_to: dt.datedaily: tuple[PromptDailyEngineStats, ...]+ consensus: tuple[PromptEngineDomainShare, ...]+ sentiment: tuple[PromptEngineSentiment, ...]+ top_words: tuple[PromptSearchWord, ...]@frozendiff --git a/products/aeo/backend/facade/hogql.py b/products/aeo/backend/facade/hogql.pyindex d5e1ad64fa75..6d2c1a705eb1 100644--- a/products/aeo/backend/facade/hogql.py+++ b/products/aeo/backend/facade/hogql.py@@ -88,6 +88,16 @@ aeo_citation_checks: PostgresTable = PostgresTable("top_cited_domains": StringJSONDatabaseField(name="top_cited_domains", description="JSON array of distinct hosts across the cited URLs."),+ "brand_sentiment": StringDatabaseField(+ name="brand_sentiment",+ nullable=True,+ description="Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand.",+ ),+ "brand_sentiment_score": FloatDatabaseField(+ name="brand_sentiment_score",+ nullable=True,+ description="Classifier confidence (0-1) in brand_sentiment.",+ ),"cost_usd": FloatDatabaseField(name="cost_usd", nullable=True, description="Engine-reported cost, where the engine reports one."),diff --git a/products/aeo/backend/facade/temporal.py b/products/aeo/backend/facade/temporal.pynew file mode 100644index 000000000000..efff02307d98--- /dev/null+++ b/products/aeo/backend/facade/temporal.py@@ -0,0 +1,6 @@+"""Temporal workflows and activities this product registers on the evals queue, which is the+only worker image that carries the sentiment model."""++from products.aeo.backend.temporal.brand_sentiment import ACTIVITIES, WORKFLOWS++__all__ = ["ACTIVITIES", "WORKFLOWS"]diff --git a/products/aeo/backend/gaps.py b/products/aeo/backend/gaps.pyindex 5487b6260d57..0c51e5c12b35 100644--- a/products/aeo/backend/gaps.py+++ b/products/aeo/backend/gaps.py@@ -1,6 +1,5 @@from __future__ import annotations-import reimport datetime as dtfrom collections import Counter, defaultdictfrom typing import Any@@ -8,8 +7,7 @@ from typing import Anyfrom django.conf import settingsfrom django.db.models import BooleanField, Case, F, Q, Value, When-import tldextract-+from products.aeo.backend.brand import mention_filterfrom products.aeo.backend.engines import is_target_url, top_domainsfrom products.aeo.backend.facade.contracts import CitationGap, EngineAnswerfrom products.aeo.backend.models import AEOCitationCheck, AEOPromptVersion@@ -17,20 +15,12 @@ from products.aeo.backend.seeding import prompt_hashMAX_GAP_URLS = 10MAX_GAP_QUERIES = 10-_DOMAIN_PARTS = tldextract.TLDExtract(suffix_list_urls=())def _ranked(counter: Counter[str], limit: int) -> tuple[str, ...]:return tuple(value for value, _ in counter.most_common(limit))-def _brand_regex(target_domains: list[str]) -> str | None:- names = {(_DOMAIN_PARTS(domain).domain or domain).lower() for domain in target_domains if domain}- if not names:- return None- return r"\m(" + "|".join(re.escape(name) for name in sorted(names)) + r")\M"--def _gap_keys(first_hashes: dict[str, str], current_hashes: dict[str, str]) -> dict[str, str]:"""Key each prompt's gap by its first version's hash, so a text edit keeps the key.@@ -60,12 +50,13 @@ def list_citation_gaps(team_id: int, *, since: dt.datetime) -> list[CitationGap].filter(created_at__gte=since, check_failed=False, prompt__active=True).filter(current_version))- mention = Q(cited=True)- if brand := _brand_regex(target_domains):- mention |= Q(answer_text__iregex=brand)rows = (checks_in_window.annotate(- mentioned=Case(When(mention, then=Value(True)), default=Value(False), output_field=BooleanField())+ mentioned=Case(+ When(mention_filter(target_domains), then=Value(True)),+ default=Value(False),+ output_field=BooleanField(),+ )).order_by("-created_at").values(diff --git a/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py b/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.pynew file mode 100644index 000000000000..3df58b8408f8--- /dev/null+++ b/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py@@ -0,0 +1,31 @@+# Generated by Django 5.2.18 on 2026-10-23 23:35++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("aeo", "0006_backfill_prompt_versions"),+ ]++ operations = [+ migrations.AddField(+ model_name="aeocitationcheck",+ name="brand_sentiment",+ field=models.CharField(+ blank=True,+ help_text="Sentiment of the sentences that name the brand: positive, neutral or negative. Null until scored, and when the answer never names the brand.",+ max_length=16,+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="aeocitationcheck",+ name="brand_sentiment_score",+ field=models.FloatField(+ blank=True,+ help_text="Classifier confidence (0-1) in `brand_sentiment`, averaged over the brand sentences.",+ null=True,+ ),+ ),+ ]diff --git a/products/aeo/backend/migrations/max_migration.txt b/products/aeo/backend/migrations/max_migration.txtindex a1727f32d961..637c4c4a811e 100644--- a/products/aeo/backend/migrations/max_migration.txt+++ b/products/aeo/backend/migrations/max_migration.txt@@ -1 +1 @@-0006_backfill_prompt_versions+0007_citation_check_brand_sentimentdiff --git a/products/aeo/backend/models.py b/products/aeo/backend/models.pyindex 2e1c729af35d..ead33a5ce13e 100644--- a/products/aeo/backend/models.py+++ b/products/aeo/backend/models.py@@ -188,6 +188,18 @@ class AEOCitationCheck(TeamScopedRootMixin, CreatedMetaFields, UUIDTModel):target_urls = models.JSONField(default=list, blank=True, help_text="Cited URLs on a target domain.")top_cited_domains = models.JSONField(default=list, blank=True, help_text="Distinct hosts across the cited URLs.")+ brand_sentiment = models.CharField(+ max_length=16,+ null=True,+ blank=True,+ help_text="Sentiment of the sentences that name the brand: positive, neutral or negative. Null until scored, and when the answer never names the brand.",+ )+ brand_sentiment_score = models.FloatField(+ null=True,+ blank=True,+ help_text="Classifier confidence (0-1) in `brand_sentiment`, averaged over the brand sentences.",+ )+cost_usd = models.FloatField(null=True, blank=True, help_text="Engine-reported cost, where the engine reports one.")gateway_trace_id = models.CharField(max_length=64,diff --git a/products/aeo/backend/presentation/views.py b/products/aeo/backend/presentation/views.pyindex b11ecf0e2371..5b88e8956031 100644--- a/products/aeo/backend/presentation/views.py+++ b/products/aeo/backend/presentation/views.py@@ -16,6 +16,8 @@ from posthog.exceptions import Conflictfrom posthog.permissions import posthog_feature_flag_enabledfrom products.aeo.backend.facade.api import (+ MAX_CONSENSUS_DOMAINS,+ MAX_STATS_WORDS,archive_prompt,get_prompt,get_prompt_stats,@@ -35,10 +37,13 @@ from products.aeo.backend.facade.contracts import (PromptCheck,PromptDailyEngineStats,PromptEngineCheck,+ PromptEngineDomainShare,+ PromptEngineSentiment,PromptEngineStats,PromptImport,PromptImportResult,PromptNotFoundError,+ PromptSearchWord,PromptStats,PromptTextConflictError,PromptTopic,@@ -171,6 +176,12 @@ class AEOPromptCheckSerializer(DataclassSerializer):"prompt_version": {"help_text": "Version of the prompt that ran. Null for checks recorded before prompts had versions."},+ "brand_sentiment": {+ "help_text": "Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand."+ },+ "brand_sentiment_score": {+ "help_text": "Classifier confidence (0-1) in 'brand_sentiment'. Null when 'brand_sentiment' is null."+ },}@@ -187,12 +198,23 @@ class AEOPromptChecksQuerySerializer(serializers.Serializer):)+MENTIONED_HELP = (+ "Checks whose answer names the brand (the registrable name of a target domain, case ignored). "+ "A cited check always counts as mentioned."+)++class AEOPromptEngineStatsSerializer(DataclassSerializer):citation_rate = serializers.FloatField(read_only=True,allow_null=True,help_text="Share (0-1) of answered checks that cited a target domain. Failed checks are left out. Null when no check was answered.",)+ mention_rate = serializers.FloatField(+ read_only=True,+ allow_null=True,+ help_text="Share (0-1) of answered checks whose answer names the brand. Failed checks are left out. Null when no check was answered.",+ )class Meta:dataclass = PromptEngineStats@@ -201,6 +223,7 @@ class AEOPromptEngineStatsSerializer(DataclassSerializer):"engine": {"help_text": ENGINE_HELP},"checks": {"help_text": "Checks of the prompt on this engine, failed ones included."},"cited": {"help_text": "Checks whose answer cited a target domain."},+ "mentioned": {"help_text": MENTIONED_HELP},"failed": {"help_text": "Checks where the engine did not answer."},"first_checked_at": {"help_text": "When the engine first ran the prompt."},"last_checked_at": {"help_text": "When the engine last ran the prompt."},@@ -216,10 +239,50 @@ class AEOPromptDailyEngineStatsSerializer(DataclassSerializer):"engine": {"help_text": ENGINE_HELP},"checks": {"help_text": "Checks on this engine that day, failed ones included."},"cited": {"help_text": "Checks that day whose answer cited a target domain."},+ "mentioned": {"help_text": "Checks that day whose answer names the brand. " + MENTIONED_HELP},"failed": {"help_text": "Checks that day where the engine did not answer."},}+class AEOPromptEngineDomainShareSerializer(DataclassSerializer):+ class Meta:+ dataclass = PromptEngineDomainShare+ ref_name = "AEOPromptEngineDomainShare"+ extra_kwargs = {+ "engine": {"help_text": ENGINE_HELP},+ "domain": {"help_text": "One of the most cited domains across all engines in the window."},+ "checks_citing": {+ "help_text": f"This engine's answered checks in the last {STATS_DAYS} days that cited the domain."+ },+ "checks": {"help_text": f"This engine's answered checks in the last {STATS_DAYS} days."},+ }+++class AEOPromptEngineSentimentSerializer(DataclassSerializer):+ class Meta:+ dataclass = PromptEngineSentiment+ ref_name = "AEOPromptEngineSentiment"+ extra_kwargs = {+ "engine": {"help_text": ENGINE_HELP},+ "positive": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read positive."},+ "neutral": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read neutral."},+ "negative": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read negative."},+ "unscored": {+ "help_text": f"Answers in the last {STATS_DAYS} days that name the brand but have no sentiment yet."+ },+ }+++class AEOPromptSearchWordSerializer(DataclassSerializer):+ class Meta:+ dataclass = PromptSearchWord+ ref_name = "AEOPromptSearchWord"+ extra_kwargs = {+ "word": {"help_text": "A word from the web searches, lower-cased, common English words left out."},+ "count": {"help_text": "How many searches used the word, weighted by how often each search ran."},+ }++class AEOPromptStatsSerializer(DataclassSerializer):engines = AEOPromptEngineStatsSerializer(many=True, help_text="Totals per answer engine over the prompt's whole history."@@ -228,12 +291,27 @@ class AEOPromptStatsSerializer(DataclassSerializer):many=True,help_text=f"Counts per day and engine from 'daily_from' to 'daily_to' ({STATS_DAYS} days). Days and engines without a check are left out.",)+ consensus = AEOPromptEngineDomainShareSerializer(+ many=True,+ help_text=f"One row per engine and per domain for the {MAX_CONSENSUS_DOMAINS} most cited domains of the last {STATS_DAYS} days: how many of the engine's answers cited the domain. Shows which sources the engines agree on.",+ )+ sentiment = AEOPromptEngineSentimentSerializer(+ many=True,+ help_text=f"Brand sentiment of the answers in the last {STATS_DAYS} days, one row per engine. Only the sentences that name the brand are scored.",+ )+ top_words = AEOPromptSearchWordSerializer(+ many=True,+ help_text=f"Most used words across the web searches the engines ran in the last {STATS_DAYS} days, at most {MAX_STATS_WORDS}. Only engines that report their searches contribute.",+ )class Meta:dataclass = PromptStatsref_name = "AEOPromptStats"extra_kwargs = {"total_checks": {"help_text": "Checks of the prompt across all engines, failed ones included."},+ "brand_names": {+ "help_text": "Lower-cased brand names, derived from the project's target domains. An answer that contains one of them as a word counts as a mention."+ },"daily_from": {"help_text": "First day of the daily series, in the project's timezone."},"daily_to": {"help_text": "Last day of the daily series (today), in the project's timezone."},}@@ -512,7 +590,7 @@ class AEOPromptViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):@validated_request(operation_id="aeo_prompts_stats",summary="Get an AEO prompt's citation stats",- description=f"Gets citation totals per answer engine for one prompt, plus per-day counts for the last {STATS_DAYS} days in the project's timezone.",+ description=f"Gets citation and brand-mention totals per answer engine for one prompt, plus, for the last {STATS_DAYS} days in the project's timezone, per-day counts, which engines cite which domains, brand sentiment per engine, and the words of the web searches the engines ran.",responses={200: OpenApiResponse(response=AEOPromptStatsSerializer)},tags=["web_analytics"],)diff --git a/products/aeo/backend/runner.py b/products/aeo/backend/runner.pyindex 414116d6fc89..f9cfeb74e416 100644--- a/products/aeo/backend/runner.py+++ b/products/aeo/backend/runner.py@@ -30,6 +30,7 @@ from posthog.models.team import Teamfrom products.aeo.backend.engines import CitationEngine, available_engines, build_check_fieldsfrom products.aeo.backend.facade.contracts import CitationRunSummaryfrom products.aeo.backend.models import AEOCitationCheck, AEOPrompt+from products.aeo.backend.temporal.brand_sentiment import start_brand_sentiment_workflowlogger = structlog.get_logger(__name__)@@ -144,6 +145,8 @@ def run_citation_checks(rows_written=summary.rows_written,write_failures=summary.write_failures,)+ if record and rows_written and settings.AEO_BRAND_SENTIMENT_ENABLED:+ start_brand_sentiment_workflow(canonical_team_id, run_id)return summary, checksdiff --git a/products/aeo/backend/sentiment.py b/products/aeo/backend/sentiment.pynew file mode 100644index 000000000000..641b25825b39--- /dev/null+++ b/products/aeo/backend/sentiment.py@@ -0,0 +1,56 @@+"""How an answer engine's reply speaks about the brand.++Only the sentences that name the brand are scored, so praise of a competitor in+the same answer does not count as praise of the brand. The classifier itself is+the AI observability ONNX model; it lives on the evals Temporal queue, and this+module only prepares its input and folds its output back into one label per answer.+"""++from __future__ import annotations++import re+from collections.abc import Mapping, Sequence++from posthog.dataclasses import frozen+from posthog.hogql_queries.ai.sentiment_labeling import select_sentiment_label++from products.aeo.backend.brand import brand_pattern++SENTIMENT_LABELS = ("negative", "neutral", "positive")+# Bounds the classifier input per answer: an answer that names the brand in more sentences than+# this is scored on the first ones, and the model truncates a sentence longer than this anyway.+MAX_BRAND_SENTENCES = 20+MAX_SENTENCE_CHARS = 1000+_SENTENCE_BREAK = re.compile(r"(?<=[.!?])\s+|\n+")+++@frozen+class AnswerSentiment:+ label: str+ score: float+++def brand_sentences(answer_text: str | None, target_domains: list[str]) -> list[str]:+ """The sentences of an answer that name the brand, in order, trimmed and bounded."""+ pattern = brand_pattern(target_domains)+ if not answer_text or pattern is None:+ return []+ sentences = (sentence.strip() for sentence in _SENTENCE_BREAK.split(answer_text))+ matching = (sentence[:MAX_SENTENCE_CHARS] for sentence in sentences if sentence and pattern.search(sentence))+ return [sentence for _, sentence in zip(range(MAX_BRAND_SENTENCES), matching)]+++def aggregate_sentiment(sentence_scores: Sequence[Mapping[str, float]]) -> AnswerSentiment | None:+ """One label for the answer from its brand sentences' class scores.++ The class scores are averaged over the sentences before the label is picked, so one strongly+ negative sentence among neutral ones pulls the answer toward negative without a majority vote+ flipping it, and the same neutral margin as AI observability keeps near-ties neutral."""+ if not sentence_scores:+ return None+ mean_scores = {+ label: round(sum(scores.get(label, 0.0) for scores in sentence_scores) / len(sentence_scores), 4)+ for label in SENTIMENT_LABELS+ }+ label = select_sentiment_label(mean_scores)+ return AnswerSentiment(label=label, score=mean_scores[label])diff --git a/products/aeo/backend/temporal/__init__.py b/products/aeo/backend/temporal/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/aeo/backend/temporal/brand_sentiment.py b/products/aeo/backend/temporal/brand_sentiment.pynew file mode 100644index 000000000000..43528c65a239--- /dev/null+++ b/products/aeo/backend/temporal/brand_sentiment.py@@ -0,0 +1,175 @@+"""Scores brand sentiment for one citation run.++The ONNX sentiment model and its dependencies are only in the image that serves+the evals Temporal queue, so the runner cannot classify inline: it starts this+workflow, and the activity reads the run's answers back from Postgres, scores the+sentences that name the brand, and writes the label and score to the check rows.+"""++from __future__ import annotations++import asyncio+import datetime as dt+import dataclasses+from collections.abc import Sequence+from typing import Any++from django.conf import settings+from django.utils import timezone++import structlog+from temporalio import activity, workflow+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy+from temporalio.exceptions import WorkflowAlreadyStartedError++from posthog.exceptions_capture import capture_exception+from posthog.sync import database_sync_to_async+from posthog.temporal.common.base import PostHogWorkflow+from posthog.temporal.common.client import sync_connect+from posthog.temporal.common.utils import close_db_connections++from products.aeo.backend.brand import brand_regex+from products.aeo.backend.models import AEOCitationCheck+from products.aeo.backend.sentiment import aggregate_sentiment, brand_sentences++logger = structlog.get_logger(__name__)++WORKFLOW_NAME = "aeo-brand-sentiment"+# One run is at most ~150 answers x 20 sentences on a CPU model, well inside this.+CLASSIFY_TIMEOUT = dt.timedelta(minutes=15)+CLASSIFY_BATCH = 32+# Answers scored before the setting was turned on, or missed by a failed run, are picked up from+# this far back, which matches the window the prompt report shows.+BACKLOG_WINDOW = dt.timedelta(days=30)+MAX_ROWS_PER_RUN = 2000++[email protected](frozen=True)+class AEOBrandSentimentInputs:+ team_id: int+ run_id: str++[email protected](frozen=True)+class _PendingAnswer:+ check_id: str+ sentences: list[str]+++def _load_pending(inputs: AEOBrandSentimentInputs) -> list[_PendingAnswer]:+ """The run's unscored answers, then the team's other unscored answers from the backlog window,+ oldest first, capped so one activity never scans an unbounded history.++ Only answers that name the brand count toward the cap: an answer without a mention stays NULL+ forever, so letting it fill the slice would starve newer answers that can be scored.++ Two concurrent runs can both pick up the same backlog row. The writes are idempotent (the same+ text scores the same), so that only costs a duplicate classification, and no transaction is+ held across the model call that a row lock could ride on."""+ target_domains: list[str] = settings.AEO_TARGET_DOMAINS+ regex = brand_regex(target_domains)+ if regex is None:+ return []+ unscored = AEOCitationCheck.objects.for_team(inputs.team_id).filter(+ check_failed=False, brand_sentiment__isnull=True, answer_text__iregex=regex+ )+ own_rows = list(unscored.filter(run_id=inputs.run_id).order_by("created_at").values_list("id", "answer_text"))+ backlog_rows = list(+ unscored.exclude(run_id=inputs.run_id)+ .filter(created_at__gte=timezone.now() - BACKLOG_WINDOW)+ .order_by("created_at")+ .values_list("id", "answer_text")[: max(0, MAX_ROWS_PER_RUN - len(own_rows))]+ )+ pending = [+ _PendingAnswer(check_id=str(check_id), sentences=brand_sentences(text, target_domains))+ for check_id, text in [*own_rows, *backlog_rows]+ ]+ return [answer for answer in pending if answer.sentences]+++def _write_scores(team_id: int, scored: Sequence[tuple[str, str, float]]) -> None:+ rows = [+ AEOCitationCheck(id=check_id, brand_sentiment=label, brand_sentiment_score=score)+ for check_id, label, score in scored+ ]+ AEOCitationCheck.objects.for_team(team_id).bulk_update(rows, ["brand_sentiment", "brand_sentiment_score"])+++@close_db_connections+async def classify_aeo_brand_sentiment(inputs: AEOBrandSentimentInputs) -> dict[str, Any]:+ """Score every answer of the run that names the brand and has no sentiment yet.++ A missing model is a deployment problem, not a data problem, so the activity reports it and+ leaves the rows unscored instead of retrying: the next run picks nothing up either, and the+ rows read as "not scored yet" until the worker image carries the model."""+ pending = await database_sync_to_async(_load_pending, thread_sensitive=False)(inputs)+ if not pending:+ return {"scored": 0, "answers": 0}++ texts = [sentence for answer in pending for sentence in answer.sentences]+ try:+ import posthog.temporal.ai_observability.sentiment.model as sentiment_model # noqa: PLC0415 -- ONNX deps++ results = []+ for start in range(0, len(texts), CLASSIFY_BATCH):+ results.extend(await asyncio.to_thread(sentiment_model.classify, texts[start : start + CLASSIFY_BATCH]))+ except (ImportError, FileNotFoundError) as error:+ logger.warning(+ "aeo_brand_sentiment_model_unavailable", team_id=inputs.team_id, run_id=inputs.run_id, error=str(error)+ )+ return {"scored": 0, "answers": len(pending), "error": "model_unavailable"}++ scored: list[tuple[str, str, float]] = []+ position = 0+ for answer in pending:+ sentence_scores = [result.scores for result in results[position : position + len(answer.sentences)]]+ position += len(answer.sentences)+ sentiment = aggregate_sentiment(sentence_scores)+ if sentiment is not None:+ scored.append((answer.check_id, sentiment.label, sentiment.score))+ await database_sync_to_async(_write_scores, thread_sensitive=False)(inputs.team_id, scored)+ logger.info("aeo_brand_sentiment_scored", team_id=inputs.team_id, run_id=inputs.run_id, scored=len(scored))+ return {"scored": len(scored), "answers": len(pending)}++[email protected](name=WORKFLOW_NAME)+class AEOBrandSentimentWorkflow(PostHogWorkflow):+ inputs_cls = AEOBrandSentimentInputs++ @workflow.run+ async def run(self, inputs: AEOBrandSentimentInputs) -> dict[str, Any]:+ return await workflow.execute_activity(+ classify_aeo_brand_sentiment,+ inputs,+ start_to_close_timeout=CLASSIFY_TIMEOUT,+ retry_policy=RetryPolicy(maximum_attempts=3, initial_interval=dt.timedelta(seconds=30)),+ )+++def start_brand_sentiment_workflow(team_id: int, run_id: str) -> None:+ """Fire-and-forget: the citation run is already recorded, so a Temporal outage only means+ this run's answers stay unscored."""+ inputs = AEOBrandSentimentInputs(team_id=team_id, run_id=run_id)+ try:+ client = sync_connect()+ asyncio.run(+ client.start_workflow(+ WORKFLOW_NAME,+ inputs,+ id=f"{WORKFLOW_NAME}-{team_id}-{run_id}",+ task_queue=settings.LLMA_EVALS_TASK_QUEUE,+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,+ )+ )+ except WorkflowAlreadyStartedError:+ logger.info("aeo_brand_sentiment_already_started", team_id=team_id, run_id=run_id)+ except Exception as error:+ logger.exception("aeo_brand_sentiment_start_failed", team_id=team_id, run_id=run_id, error=str(error)[:300])+ capture_exception(error)+ else:+ logger.info("aeo_brand_sentiment_started", team_id=team_id, run_id=run_id)+++WORKFLOWS = [AEOBrandSentimentWorkflow]+ACTIVITIES = [classify_aeo_brand_sentiment]diff --git a/products/aeo/backend/test/test_api.py b/products/aeo/backend/test/test_api.pyindex 16c56c35ddca..9d4e57581b02 100644--- a/products/aeo/backend/test/test_api.py+++ b/products/aeo/backend/test/test_api.py@@ -6,6 +6,7 @@ from posthog.test.base import APIBaseTestfrom unittest.mock import patchfrom django.db import connection+from django.test import override_settingsfrom django.test.utils import CaptureQueriesContextfrom django.utils import timezone@@ -41,6 +42,8 @@ class TestAEOPromptAPI(APIBaseTest):cited: bool = False,check_failed: bool = False,answer_text: str | None = None,+ cited_urls: list[str] | None = None,+ search_queries: list[str] | None = None,) -> str:with team_scope(self.team.pk):prompt = AEOPrompt.objects.get(id=prompt_id)@@ -57,6 +60,8 @@ class TestAEOPromptAPI(APIBaseTest):cited=cited,check_failed=check_failed,answer_text=answer_text,+ cited_urls=cited_urls or [],+ search_queries=search_queries or [],)AEOCitationCheck.objects.filter(id=check.id).update(created_at=timezone.now() - dt.timedelta(minutes=minutes_ago)@@ -424,26 +429,146 @@ class TestAEOPromptAPI(APIBaseTest):prompt_id = self._import_one_prompt()day = 24 * 60self._record_check(prompt_id, "exa-answer", minutes_ago=10, cited=True)- self._record_check(prompt_id, "exa-answer", minutes_ago=20)+ self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example Analytics is one option.")self._record_check(prompt_id, "exa-answer", minutes_ago=day, check_failed=True)self._record_check(prompt_id, "claude-web-search", minutes_ago=2 * day, cited=True)self._record_check(prompt_id, "claude-web-search", minutes_ago=40 * day)- response = self.client.get(f"{self.url}{prompt_id}/stats/")+ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):+ response = self.client.get(f"{self.url}{prompt_id}/stats/")assert response.status_code == status.HTTP_200_OK, response.json()stats = response.json()assert stats["total_checks"] == 5assert (stats["daily_from"], stats["daily_to"]) == ("2026-09-21", "2026-10-20")engines = {- row["engine"]: (row["checks"], row["cited"], row["failed"], row["citation_rate"])+ row["engine"]: (+ row["checks"],+ row["cited"],+ row["mentioned"],+ row["failed"],+ row["citation_rate"],+ row["mention_rate"],+ )for row in stats["engines"]}- assert engines == {"claude-web-search": (2, 1, 0, 0.5), "exa-answer": (3, 1, 1, 0.5)}- assert [(row["date"], row["engine"], row["checks"], row["cited"], row["failed"]) for row in stats["daily"]] == [- ("2026-10-18", "claude-web-search", 1, 1, 0),- ("2026-10-19", "exa-answer", 1, 0, 1),- ("2026-10-20", "exa-answer", 2, 1, 0),+ assert engines == {"claude-web-search": (2, 1, 1, 0, 0.5, 0.5), "exa-answer": (3, 1, 2, 1, 0.5, 1.0)}+ assert [+ (row["date"], row["engine"], row["checks"], row["cited"], row["mentioned"], row["failed"])+ for row in stats["daily"]+ ] == [+ ("2026-10-18", "claude-web-search", 1, 1, 1, 0),+ ("2026-10-19", "exa-answer", 1, 0, 0, 1),+ ("2026-10-20", "exa-answer", 2, 1, 2, 0),+ ]+ assert stats["brand_names"] == ["example"]+ assert (stats["consensus"], stats["top_words"]) == ([], [])+ assert [(row["engine"], row["unscored"]) for row in stats["sentiment"]] == [+ ("claude-web-search", 1),+ ("exa-answer", 2),+ ]++ @parameterized.expand(+ [+ ("brand_named_in_the_answer", {"answer_text": "Try Example Analytics for funnels."}, 1),+ ("brand_case_ignored", {"answer_text": "EXAMPLE ships session replay."}, 1),+ ("brand_inside_another_word", {"answer_text": "Counterexample: nothing fits."}, 0),+ ("cited_without_the_name_in_the_text", {"cited": True, "answer_text": "See [1]."}, 1),+ ("failed_check", {"check_failed": True}, 0),+ ]+ )+ def test_stats_count_a_mention_when_the_answer_names_the_brand(+ self, _name: str, check: dict, expected_mentioned: int+ ) -> None:+ prompt_id = self._import_one_prompt()+ self._record_check(prompt_id, "exa-answer", minutes_ago=10, **check)++ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):+ stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()++ assert [row["mentioned"] for row in stats["engines"]] == [expected_mentioned]+ assert [row["mentioned"] for row in stats["daily"]] == [expected_mentioned]++ def test_stats_count_brand_sentiment_per_engine_and_checks_expose_it(self) -> None:+ prompt_id = self._import_one_prompt()+ scored = self._record_check(prompt_id, "exa-answer", minutes_ago=10, answer_text="Example is great.")+ self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example is slow.")+ self._record_check(prompt_id, "exa-answer", minutes_ago=30, answer_text="Example exists.")+ self._record_check(prompt_id, "claude-web-search", minutes_ago=40, answer_text="Nothing about the brand.")+ with team_scope(self.team.pk):+ for text, label, score in (("great", "positive", 0.9), ("slow", "negative", 0.7)):+ AEOCitationCheck.objects.filter(answer_text__contains=text).update(+ brand_sentiment=label, brand_sentiment_score=score+ )++ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):+ stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()+ checks = self.client.get(f"{self.url}{prompt_id}/checks/?engine=exa-answer").json()["results"]++ assert [+ (row["engine"], row["positive"], row["neutral"], row["negative"], row["unscored"])+ for row in stats["sentiment"]+ ] == [("claude-web-search", 0, 0, 0, 0), ("exa-answer", 1, 0, 1, 1)]+ assert {check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks} == {+ scored: ("positive", 0.9),+ **{check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks[1:]},+ }+ assert [check["brand_sentiment"] for check in checks] == ["positive", "negative", None]++ @time_machine.travel("2026-10-20T12:00:00Z", tick=False)+ def test_stats_rank_engine_consensus_and_search_words_over_the_window(self) -> None:+ prompt_id = self._import_one_prompt()+ day = 24 * 60+ ours = "https://docs.example.com/product-analytics"+ rival = "https://rival.example.org/best-tools"+ self._record_check(+ prompt_id,+ "claude-web-search",+ minutes_ago=10,+ cited=True,+ cited_urls=[rival, ours, "https://rival.example.org/pricing", rival],+ search_queries=["Best product analytics tools", "product analytics for startups"],+ )+ self._record_check(+ prompt_id,+ "claude-web-search",+ minutes_ago=day,+ cited_urls=[rival],+ search_queries=["best product analytics tools"],+ )+ self._record_check(+ prompt_id,+ "openai-web-search",+ minutes_ago=2 * day,+ cited=True,+ cited_urls=["https://community.example.net/thread", ours],+ search_queries=["open source analytics"],+ )+ self._record_check(prompt_id, "exa-answer", minutes_ago=3 * day, cited_urls=[])+ self._record_check(prompt_id, "openai-web-search", minutes_ago=4 * day, check_failed=True)+ self._record_check(prompt_id, "openai-web-search", minutes_ago=40 * day, cited_urls=[rival, rival, rival])++ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):+ stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()++ assert [(row["engine"], row["domain"], row["checks_citing"], row["checks"]) for row in stats["consensus"]] == [+ ("claude-web-search", "docs.example.com", 1, 2),+ ("claude-web-search", "rival.example.org", 2, 2),+ ("claude-web-search", "community.example.net", 0, 2),+ ("exa-answer", "docs.example.com", 0, 1),+ ("exa-answer", "rival.example.org", 0, 1),+ ("exa-answer", "community.example.net", 0, 1),+ ("openai-web-search", "docs.example.com", 1, 1),+ ("openai-web-search", "rival.example.org", 0, 1),+ ("openai-web-search", "community.example.net", 1, 1),+ ]+ assert [(row["word"], row["count"]) for row in stats["top_words"]] == [+ ("analytics", 4),+ ("product", 3),+ ("tools", 2),+ ("open", 1),+ ("source", 1),+ ("startups", 1),]@parameterized.expand([("above_range", 140), ("not_a_number", "NaN"), ("infinite", "Infinity")])diff --git a/products/aeo/backend/test/test_brand_sentiment.py b/products/aeo/backend/test/test_brand_sentiment.pynew file mode 100644index 000000000000..d94352334b40--- /dev/null+++ b/products/aeo/backend/test/test_brand_sentiment.py@@ -0,0 +1,178 @@+import uuid+import datetime as dt++import pytest+from unittest.mock import MagicMock, patch++from django.utils import timezone++from asgiref.sync import sync_to_async+from temporalio.testing import ActivityEnvironment++from posthog.models.organization import Organization+from posthog.models.scoping import team_scope+from posthog.models.team import Team+from posthog.temporal.ai_observability.sentiment.schema import SentimentResult++from products.aeo.backend.engines import CitationCheck+from products.aeo.backend.models import AEOCitationCheck, AEOPrompt+from products.aeo.backend.runner import run_citation_checks+from products.aeo.backend.seeding import PromptCandidate, upsert_prompts+from products.aeo.backend.temporal.brand_sentiment import AEOBrandSentimentInputs, classify_aeo_brand_sentiment++_MODULE = "products.aeo.backend.temporal.brand_sentiment"+++def _team() -> Team:+ organization = Organization.objects.create(name="aeo sentiment org")+ return Team.objects.create(organization=organization, name="aeo sentiment team")+++def _check(team: Team, prompt: AEOPrompt, run_id: uuid.UUID, answer_text: str | None, **fields) -> AEOCitationCheck:+ with team_scope(team.pk):+ return AEOCitationCheck.objects.create(+ team=team,+ prompt=prompt,+ run_id=run_id,+ prompt_text=prompt.prompt,+ prompt_source=prompt.prompt_source,+ prompt_hash=prompt.prompt_hash,+ engine="exa-answer",+ model="exa-answer",+ answer_text=answer_text,+ **fields,+ )+++def _seed_run(team: Team) -> tuple[uuid.UUID, dict[str, AEOCitationCheck]]:+ with team_scope(team.pk):+ prompt = AEOPrompt.objects.create(+ team=team, prompt="best tool?", prompt_hash="h", prompt_source=AEOPrompt.Source.MANUAL+ )+ run_id = uuid.uuid4()+ return run_id, {+ "praise": _check(team, prompt, run_id, "Example is fast. Rival is slow."),+ "two_sentences": _check(team, prompt, run_id, "Example is slow. Still, Example is cheap."),+ "no_brand": _check(team, prompt, run_id, "Rival is the only option."),+ "failed": _check(team, prompt, run_id, None, check_failed=True),+ "scored": _check(team, prompt, run_id, "Example again.", brand_sentiment="neutral", brand_sentiment_score=0.5),+ "other_run": _check(team, prompt, uuid.uuid4(), "Example elsewhere."),+ "no_brand_backlog": _check(team, prompt, uuid.uuid4(), "Rival elsewhere."),+ "too_old": _check(team, prompt, uuid.uuid4(), "Example long ago."),+ }+++def _age(check: AEOCitationCheck, days: int) -> None:+ with team_scope(check.team_id):+ AEOCitationCheck.objects.filter(id=check.id).update(created_at=timezone.now() - dt.timedelta(days=days))+++def _result(**scores: float) -> SentimentResult:+ label = max(scores, key=scores.__getitem__)+ return SentimentResult(label=label, score=scores[label], scores=scores)++[email protected]_db(transaction=True)+async def test_activity_scores_the_runs_answers_then_the_recent_unscored_backlog(settings) -> None:+ settings.AEO_TARGET_DOMAINS = ["example.com"]+ team = await sync_to_async(_team)()+ run_id, checks = await sync_to_async(_seed_run)(team)+ await sync_to_async(_age)(checks["other_run"], 10)+ await sync_to_async(_age)(checks["no_brand_backlog"], 20)+ await sync_to_async(_age)(checks["too_old"], 31)+ classify = MagicMock(+ return_value=[+ _result(positive=0.9, neutral=0.08, negative=0.02),+ _result(positive=0.05, neutral=0.15, negative=0.8),+ _result(positive=0.75, neutral=0.2, negative=0.05),+ _result(positive=0.1, neutral=0.85, negative=0.05),+ ]+ )++ # Cap of 4 leaves one backlog slot after the run's own rows. The older brand-less backlog row+ # must not take it from the newer answer that names the brand.+ with (+ patch("posthog.temporal.ai_observability.sentiment.model.classify", classify),+ patch(f"{_MODULE}.MAX_ROWS_PER_RUN", 4),+ ):+ result = await ActivityEnvironment().run(+ classify_aeo_brand_sentiment, AEOBrandSentimentInputs(team_id=team.id, run_id=str(run_id))+ )++ assert result == {"scored": 3, "answers": 3}+ classify.assert_called_once_with(+ ["Example is fast.", "Example is slow.", "Still, Example is cheap.", "Example elsewhere."]+ )++ def sentiments() -> dict[str, tuple[str | None, float | None]]:+ with team_scope(team.pk):+ rows = AEOCitationCheck.objects.in_bulk([check.id for check in checks.values()])+ return {+ name: (rows[check.id].brand_sentiment, rows[check.id].brand_sentiment_score)+ for name, check in checks.items()+ }++ assert await sync_to_async(sentiments)() == {+ "praise": ("positive", 0.9),+ "two_sentences": ("negative", 0.425),+ "no_brand": (None, None),+ "failed": (None, None),+ "scored": ("neutral", 0.5),+ "other_run": ("neutral", 0.85),+ "no_brand_backlog": (None, None),+ "too_old": (None, None),+ }++[email protected]_db(transaction=True)[email protected]("error", [ImportError("No module named 'onnxruntime'"), FileNotFoundError("model.onnx")])+async def test_activity_leaves_rows_unscored_when_the_model_is_unavailable(settings, error: Exception) -> None:+ settings.AEO_TARGET_DOMAINS = ["example.com"]+ team = await sync_to_async(_team)()+ run_id, checks = await sync_to_async(_seed_run)(team)++ with patch("posthog.temporal.ai_observability.sentiment.model.classify", MagicMock(side_effect=error)):+ result = await ActivityEnvironment().run(+ classify_aeo_brand_sentiment, AEOBrandSentimentInputs(team_id=team.id, run_id=str(run_id))+ )++ assert result == {"scored": 0, "answers": 4, "error": "model_unavailable"}+ unscored = await sync_to_async(+ lambda: AEOCitationCheck.objects.for_team(team.id).filter(brand_sentiment__isnull=True).count()+ )()+ assert unscored == len(checks) - 1+++class _StubEngine:+ name = "stub-engine"+ model = "stub"++ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:+ return CitationCheck(engine=self.name, model=self.model, answer_text="Example is fine.")+++ "enabled, record, expected_starts",+ [(True, True, 1), (False, True, 0), (True, False, 0)],+ ids=["on", "off", "dry_run"],+)+def test_runner_starts_the_sentiment_workflow_only_when_enabled_and_recording(+ settings, enabled: bool, record: bool, expected_starts: int+) -> None:+ settings.AEO_BRAND_SENTIMENT_ENABLED = enabled+ team = _team()+ upsert_prompts(team, [PromptCandidate(text="q?", source="imported")])++ with patch(f"{_MODULE}.sync_connect") as connect, patch(f"{_MODULE}.asyncio.run") as run:+ summary, _ = run_citation_checks(team, engines=[_StubEngine()], record=record)++ assert summary.rows_written == (1 if record else 0)+ assert run.call_count == expected_starts+ if expected_starts:+ start = connect.return_value.start_workflow+ start.assert_called_once()+ assert start.call_args.kwargs["id"] == f"aeo-brand-sentiment-{team.id}-{summary.run_id}"+ assert start.call_args.kwargs["task_queue"] == settings.LLMA_EVALS_TASK_QUEUEdiff --git a/products/aeo/backend/test/test_sentiment.py b/products/aeo/backend/test/test_sentiment.pynew file mode 100644index 000000000000..3adef29e30b2--- /dev/null+++ b/products/aeo/backend/test/test_sentiment.py@@ -0,0 +1,74 @@+import pytest++from products.aeo.backend.brand import brand_names, brand_regex+from products.aeo.backend.sentiment import (+ MAX_BRAND_SENTENCES,+ MAX_SENTENCE_CHARS,+ AnswerSentiment,+ aggregate_sentiment,+ brand_sentences,+)++TARGET = ["docs.example.co.uk", "Example.com"]+++ "domains, names, regex",+ [+ (TARGET, ("example",), r"\m(example)\M"),+ (["posthog.com", "rival.io"], ("posthog", "rival"), r"\m(posthog|rival)\M"),+ ([], (), None),+ (["", "example.com"], ("example",), r"\m(example)\M"),+ ],+)+def test_brand_names_come_from_the_registrable_domain(domains: list[str], names: tuple[str, ...], regex: str | None):+ assert brand_names(domains) == names+ assert brand_regex(domains) == regex+++ "answer, expected",+ [+ (+ "Example is fast. Rival is cheaper! Pick Example if you self-host?",+ ["Example is fast.", "Pick Example if you self-host?"],+ ),+ ("EXAMPLE works\nRival too\nexample's docs help", ["EXAMPLE works", "example's docs help"]),+ ("A counterexample: nothing here. Examples abound.", []),+ ("", []),+ (None, []),+ ],+)+def test_brand_sentences_keep_only_sentences_that_name_the_brand(answer: str | None, expected: list[str]):+ assert brand_sentences(answer, TARGET) == expected+++def test_brand_sentences_are_bounded():+ answer = " ".join(f"Example point {index}." for index in range(MAX_BRAND_SENTENCES + 5)) + " Example " + "x" * 2000+ sentences = brand_sentences(answer, TARGET)+ assert len(sentences) == MAX_BRAND_SENTENCES+ assert all(len(sentence) <= MAX_SENTENCE_CHARS for sentence in sentences)+ assert brand_sentences("Example is fine.", []) == []+++ "scores, expected",+ [+ ([], None),+ ([{"positive": 0.9, "neutral": 0.08, "negative": 0.02}], AnswerSentiment(label="positive", score=0.9)),+ # The mean decides: one strongly negative sentence beats two mildly neutral ones.+ (+ [+ {"positive": 0.1, "neutral": 0.5, "negative": 0.4},+ {"positive": 0.05, "neutral": 0.45, "negative": 0.5},+ {"positive": 0.0, "neutral": 0.05, "negative": 0.95},+ ],+ AnswerSentiment(label="negative", score=0.6167),+ ),+ # A polar label that only just beats neutral stays neutral, as in AI observability.+ ([{"positive": 0.5, "neutral": 0.4, "negative": 0.1}], AnswerSentiment(label="neutral", score=0.4)),+ ([{"positive": 1.0}], AnswerSentiment(label="positive", score=1.0)),+ ],+)+def test_aggregate_sentiment_averages_sentence_scores_before_labeling(scores: list[dict[str, float]], expected):+ assert aggregate_sentiment(scores) == expecteddiff --git a/products/aeo/mcp/tools.yaml b/products/aeo/mcp/tools.yamlindex 14b2e1593dc3..87d29dd32963 100644--- a/products/aeo/mcp/tools.yaml+++ b/products/aeo/mcp/tools.yaml@@ -91,10 +91,16 @@ tools:title: Get a tracked AI prompt's citation statsdescription: >Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the- prompt's whole history: `checks`, `cited`, `failed`, `citation_rate` (cited share of answered checks, 0-1,- failed checks left out), and the first and last check times. `daily` has counts per day and engine for the- last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Use- it to compare engines or to see whether a prompt's citations are trending up or down.+ prompt's whole history: `checks`, `cited`, `mentioned` (answers that name the brand), `failed`,+ `citation_rate` and `mention_rate` (shares of answered checks, 0-1, failed checks left out), and the first+ and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to+ `daily_to`, in the project's timezone); days without a check are left out. Over the same 30 days,+ `consensus` says how many of each engine's answers cited each of the ten most cited domains, `sentiment`+ counts positive, neutral and negative answers toward the brand per engine (`unscored` answers name the brand+ but are not scored yet), and `top_words` lists the most used words of the web searches the engines ran.+ `brand_names` are the words an answer must contain to count as a mention; use them to filter `answer_text`+ in a HogQL query over `system.aeo_citation_checks`. Use it to compare engines, to see whether a prompt's+ citations are trending up or down, and to find which sources the engines agree on.param_overrides:id:description: The prompt id (a UUID), from `aeo-prompts-list`.diff --git a/products/aeo/turbo.json b/products/aeo/turbo.jsonindex a38de87df10f..2abbe99ac41c 100644--- a/products/aeo/turbo.json+++ b/products/aeo/turbo.json@@ -5,6 +5,7 @@"inputs": ["backend/facade/**","backend/tasks/**",+ "backend/temporal/**","backend/models.py","backend/migrations/**","backend/routes.py",diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptConsensus.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptConsensus.tsxnew file mode 100644index 000000000000..453be6c54e06--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptConsensus.tsx@@ -0,0 +1,67 @@+import clsx from 'clsx'++import { LemonTable, LemonTableColumn, Tooltip } from '@posthog/lemon-ui'++import { percentage } from 'lib/utils/numbers'+import { pluralize } from 'lib/utils/strings'++import type { PromptConsensusCell, PromptConsensusMatrix } from './promptReport'+import { engineLabel } from './utils'++type ConsensusRow = PromptConsensusMatrix['rows'][number]++const heatClass = (share: number | null): string =>+ share === null || share === 0+ ? ''+ : share < 0.25+ ? 'bg-success/15'+ : share < 0.5+ ? 'bg-success/30'+ : share < 0.75+ ? 'bg-success/50'+ : 'bg-success/70'++const HeatCell = ({ cell, checks }: { cell: PromptConsensusCell; checks: number }): JSX.Element => (+ <Tooltip title={`${cell.checksCiting} of ${pluralize(checks, 'answer')} link to ${cell.domain}`}>+ <span className={clsx('block rounded px-2 py-1 text-right tabular-nums', heatClass(cell.share))}>+ {cell.share === null ? '-' : percentage(cell.share, 0)}+ </span>+ </Tooltip>+)++export const ContentAutopilotPromptConsensus = ({ matrix }: { matrix: PromptConsensusMatrix }): JSX.Element => {+ const columns: LemonTableColumn<ConsensusRow, keyof ConsensusRow | undefined>[] = [+ {+ title: 'Engine',+ key: 'engine',+ render: (_, row) => (+ <span className="whitespace-nowrap">+ {engineLabel(row.engine)} <span className="text-muted">({pluralize(row.checks, 'answer')})</span>+ </span>+ ),+ },+ ...matrix.domains.map(+ (domain, index): LemonTableColumn<ConsensusRow, keyof ConsensusRow | undefined> => ({+ title: (+ <Tooltip title={domain}>+ <span className="block max-w-32 truncate">{domain}</span>+ </Tooltip>+ ),+ key: domain,+ align: 'right',+ render: (_, row) => <HeatCell cell={row.cells[index]} checks={row.checks} />,+ })+ ),+ ]+ return (+ <div className="overflow-x-auto">+ <LemonTable+ size="small"+ dataSource={matrix.rows}+ rowKey="engine"+ columns={columns}+ emptyState="No answer in the last 30 days linked to a site."+ />+ </div>+ )+}diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsxindex f6c5f549dbb3..5db8e1329afb 100644--- a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsx+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsx@@ -1,6 +1,6 @@import { useActions, useValues } from 'kea'-import { LemonBanner, LemonButton, LemonModal, LemonSkeleton, LemonTag } from '@posthog/lemon-ui'+import { LemonBanner, LemonButton, LemonModal, LemonSegmentedButton, LemonSkeleton, LemonTag } from '@posthog/lemon-ui'import { TZLabel } from 'lib/components/TZLabel'import { pluralize } from 'lib/utils/strings'@@ -9,16 +9,53 @@ import { ContentAutopilotPromptActiveSwitch } from './ContentAutopilotPromptActiimport { ContentAutopilotPromptAnswer } from './ContentAutopilotPromptAnswer'import { ContentAutopilotPromptArchiveBanner } from './ContentAutopilotPromptArchiveBanner'import { ContentAutopilotPromptChecks } from './ContentAutopilotPromptChecks'+import { ContentAutopilotPromptConsensus } from './ContentAutopilotPromptConsensus'import { contentAutopilotPromptDetailLogic } from './contentAutopilotPromptDetailLogic'-import { ContentAutopilotPromptEngineSummary } from './ContentAutopilotPromptEngineSummary'-import { ContentAutopilotPromptHistoryGrid } from './ContentAutopilotPromptHistoryGrid'+import { ContentAutopilotPromptKpiCard } from './ContentAutopilotPromptKpiCard'+import { ContentAutopilotPromptQueryTile } from './ContentAutopilotPromptQueryTile'+import { ContentAutopilotPromptSentiment } from './ContentAutopilotPromptSentiment'import { ContentAutopilotPromptVersions } from './ContentAutopilotPromptVersions'+import { promptCitationsContext, promptSearchesContext, promptSourcesContext } from './promptReportColumns'++const Section = ({+ title,+ description,+ children,+}: {+ title: string+ description?: string+ children: React.ReactNode+}): JSX.Element => (+ <section className="flex flex-col gap-3">+ <div>+ <h4 className="m-0">{title}</h4>+ {description ? <p className="m-0 text-xs text-muted">{description}</p> : null}+ </div>+ {children}+ </section>+)export const ContentAutopilotPromptDetail = (): JSX.Element | null => {- const { promptId, prompt, stats, historyGrid, lastCheckedAt, detailError, archiveConfirming } = useValues(+ const {+ promptId,+ prompt,+ stats,+ lastCheckedAt,+ detailError,+ archiveConfirming,+ reportKpis,+ consensusMatrix,+ sentimentSummary,+ visibilityMetric,+ visibilityQuery,+ sourcesQuery,+ citationsQuery,+ searchesQuery,+ sentimentQuery,+ } = useValues(contentAutopilotPromptDetailLogic)+ const { closePromptDetail, openPromptDetail, setArchiveConfirming, setVisibilityMetric } = useActions(contentAutopilotPromptDetailLogic)- const { closePromptDetail, openPromptDetail, setArchiveConfirming } = useActions(contentAutopilotPromptDetailLogic)if (!promptId) {return null@@ -34,20 +71,32 @@ export const ContentAutopilotPromptDetail = (): JSX.Element | null => {)}+ const reportReady =+ prompt &&+ stats &&+ reportKpis &&+ consensusMatrix &&+ sentimentSummary &&+ visibilityQuery &&+ sourcesQuery &&+ citationsQuery &&+ searchesQuery &&+ sentimentQuery+return (<LemonModal isOpen onClose={closePromptDetail} title={prompt?.prompt ?? 'Loading prompt'} width={960}>{detailError === 'failed' ? (<LemonBanner type="error" action={{ children: 'Try again', onClick: () => openPromptDetail(promptId) }}>Couldn't load this prompt. Try again, and if it keeps happening contact support.</LemonBanner>- ) : !prompt || !stats || !historyGrid ? (+ ) : !reportReady ? (<div className="flex flex-col gap-3"><LemonSkeleton className="h-8 w-1/2" /><LemonSkeleton className="h-24 w-full" /><LemonSkeleton className="h-40 w-full" /></div>) : (- <div className="flex flex-col gap-6">+ <div className="@container/prompt-report flex flex-col gap-6"><div className="flex flex-wrap items-center gap-x-4 gap-y-2 text-sm">{prompt.topic ? (<LemonTag>{prompt.topic}</LemonTag>@@ -84,10 +133,9 @@ export const ContentAutopilotPromptDetail = (): JSX.Element | null => {</div><ContentAutopilotPromptArchiveBanner />{prompt.version_count > 1 ? (- <section className="flex flex-col gap-2">- <h4 className="m-0">Versions</h4>+ <Section title="Versions"><ContentAutopilotPromptVersions promptId={prompt.id} />- </section>+ </Section>) : null}{stats.engines.length === 0 ? (@@ -99,22 +147,123 @@ export const ContentAutopilotPromptDetail = (): JSX.Element | null => {</p>) : (<>- <section className="flex flex-col gap-2">- <h4 className="m-0">Citation rate by engine</h4>- <ContentAutopilotPromptEngineSummary engines={stats.engines} />- </section>- <section className="flex flex-col gap-2">- <h4 className="m-0">Last 30 days</h4>- <ContentAutopilotPromptHistoryGrid grid={historyGrid} />- </section>- <section className="flex flex-col gap-2">- <h4 className="m-0">Answers</h4>+ <Section+ title="Last 30 days, all engines"+ description={`${pluralize(reportKpis.answers, 'answer')} across ${pluralize(reportKpis.days, 'day')}.`}+ >+ <div className="grid grid-cols-1 @2xl/prompt-report:grid-cols-2 gap-3">+ <ContentAutopilotPromptKpiCard+ label="Mentioned in answers"+ description="Share of answers that name you. An answer that cites your site always counts."+ kpi={reportKpis.visibility}+ data-attr="content-autopilot-prompt-kpi-mentioned"+ />+ <ContentAutopilotPromptKpiCard+ label="Cited as a source"+ description="Share of answers that link to a page on your site."+ kpi={reportKpis.citation}+ data-attr="content-autopilot-prompt-kpi-cited"+ />+ </div>+ </Section>++ <Section+ title="How often engines mention or cite you"+ description="One line per engine, one point per day it ran."+ >+ <ContentAutopilotPromptQueryTile+ title={+ visibilityMetric === 'cited'+ ? 'Share of answers that cite you'+ : 'Share of answers that mention you'+ }+ query={visibilityQuery}+ uniqueKey={`aeo-prompt-visibility-${promptId}-${visibilityMetric}`}+ bodyClassName="h-72 p-2"+ actions={+ <LemonSegmentedButton+ size="xsmall"+ value={visibilityMetric}+ onChange={setVisibilityMetric}+ options={[+ { value: 'cited', label: 'Cited' },+ { value: 'mentioned', label: 'Mentioned' },+ ]}+ data-attr="content-autopilot-prompt-visibility-metric"+ />+ }+ data-attr="content-autopilot-prompt-visibility-tile"+ />+ </Section>++ <Section title="Answers" description="The latest answer from each engine."><ContentAutopilotPromptAnswer />- </section>- <section className="flex flex-col gap-2">- <h4 className="m-0">All checks</h4>+ </Section>++ <Section+ title="Where the answers come from"+ description="The sites and pages the engines link to when they answer this prompt."+ >+ <div className="grid grid-cols-1 @3xl/prompt-report:grid-cols-2 gap-4">+ <ContentAutopilotPromptQueryTile+ title="Sites the engines cite for this prompt"+ query={sourcesQuery}+ uniqueKey={`aeo-prompt-sources-${promptId}`}+ context={promptSourcesContext}+ data-attr="content-autopilot-prompt-sources-tile"+ />+ <ContentAutopilotPromptQueryTile+ title="Pages the engines cite most"+ query={citationsQuery}+ uniqueKey={`aeo-prompt-citations-${promptId}`}+ context={promptCitationsContext}+ data-attr="content-autopilot-prompt-citations-tile"+ />+ </div>+ </Section>++ <Section+ title="Which sites each engine relies on"+ description="The share of each engine's answers that link to a site. A darker cell means the engine links to that site more often."+ >+ <ContentAutopilotPromptConsensus matrix={consensusMatrix} />+ </Section>++ <Section+ title="How answers talk about you"+ description="Scored on the sentences that name you, so praise of another tool does not count."+ >+ <ContentAutopilotPromptSentiment summary={sentimentSummary} query={sentimentQuery} />+ </Section>++ <Section+ title="What the engines searched for"+ description="The searches an engine ran before it answered. Engines that do not report their searches are left out."+ >+ <ContentAutopilotPromptQueryTile+ title="Searches"+ query={searchesQuery}+ uniqueKey={`aeo-prompt-searches-${promptId}`}+ context={promptSearchesContext}+ footer={+ stats.top_words.length ? (+ <span className="flex flex-wrap items-center gap-1">+ <span className="mr-1">Most common words in those searches:</span>+ {stats.top_words.map((word) => (+ <LemonTag key={word.word} size="small">+ {word.word} <span className="text-muted">{word.count}</span>+ </LemonTag>+ ))}+ </span>+ ) : undefined+ }+ data-attr="content-autopilot-prompt-searches-tile"+ />+ </Section>++ <Section title="All checks"><ContentAutopilotPromptChecks />- </section>+ </Section></>)}</div>diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptEngineSummary.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptEngineSummary.tsxdeleted file mode 100644index e8670a639083..000000000000--- a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptEngineSummary.tsx+++ /dev/null@@ -1,30 +0,0 @@-import { LemonCard } from '@posthog/lemon-ui'--import { pluralize } from 'lib/utils/strings'--import type { AEOPromptEngineStatsApi } from 'products/web_analytics/frontend/generated/api.schemas'--import { engineLabel } from './utils'--export const ContentAutopilotPromptEngineSummary = ({- engines,-}: {- engines: AEOPromptEngineStatsApi[]-}): JSX.Element => (- <div className="grid grid-cols-[repeat(auto-fill,minmax(12rem,1fr))] gap-2">- {engines.map((engine) => (- <LemonCard key={engine.engine} hoverEffect={false} className="p-3 flex flex-col gap-1">- <span className="text-muted text-xs font-semibold uppercase">{engineLabel(engine.engine)}</span>- <span className="text-2xl font-semibold">- {engine.citation_rate === null ? 'No answers' : `${Math.round(engine.citation_rate * 100)}%`}- </span>- <span className="text-muted text-xs">- Cited in {engine.cited} of {pluralize(engine.checks - engine.failed, 'answer')}- </span>- <span className="text-muted text-xs">- {pluralize(engine.checks, 'check')}, {engine.failed} failed- </span>- </LemonCard>- ))}- </div>-)diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptHistoryGrid.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptHistoryGrid.tsxdeleted file mode 100644index c6085003d196..000000000000--- a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptHistoryGrid.tsx+++ /dev/null@@ -1,70 +0,0 @@-import clsx from 'clsx'--import { Tooltip } from '@posthog/lemon-ui'--import { dayjs } from 'lib/dayjs'--import { PromptHistoryCell, PromptHistoryCellStatus, PromptHistoryGrid, engineLabel } from './utils'--const STATUS_LABELS: Record<PromptHistoryCellStatus, string> = {- cited: 'Cited',- not_cited: 'Not cited',- failed: 'Failed',- no_run: 'No run',-}--const STATUS_CLASSES: Record<PromptHistoryCellStatus, string> = {- cited: 'bg-success',- not_cited: 'bg-warning',- failed: 'bg-danger',- no_run: 'bg-border',-}--const cellTooltip = ({ date, status, checks, cited, failed }: PromptHistoryCell): string => {- const day = dayjs(date).format('MMM D, YYYY')- if (status === 'no_run') {- return `${day}: no run`- }- const answered = checks - failed- const failures = failed ? `, ${failed} failed` : ''- return `${day}: ${STATUS_LABELS[status].toLowerCase()} (cited in ${cited} of ${answered} answers${failures})`-}--export const ContentAutopilotPromptHistoryGrid = ({ grid }: { grid: PromptHistoryGrid }): JSX.Element => (- <div className="flex flex-col gap-2">- <div className="overflow-x-auto">- <div className="flex flex-col gap-1 w-max">- {grid.rows.map(({ engine, cells }) => (- <div key={engine} className="flex items-center gap-2">- <span className="w-20 shrink-0 text-xs text-muted truncate">{engineLabel(engine)}</span>- <div className="flex gap-0.5">- {cells.map((cell) => (- <Tooltip key={cell.date} title={cellTooltip(cell)}>- <span- role="img"- aria-label={cellTooltip(cell)}- className={clsx('block size-4 rounded-sm', STATUS_CLASSES[cell.status])}- />- </Tooltip>- ))}- </div>- </div>- ))}- {grid.dates.length > 0 ? (- <div className="flex justify-between pl-22 text-xs text-muted">- <span>{dayjs(grid.dates[0]).format('MMM D')}</span>- <span>{dayjs(grid.dates[grid.dates.length - 1]).format('MMM D')}</span>- </div>- ) : null}- </div>- </div>- <div className="flex flex-wrap gap-3 text-xs text-muted">- {(Object.keys(STATUS_LABELS) as PromptHistoryCellStatus[]).map((status) => (- <span key={status} className="flex items-center gap-1">- <span className={clsx('block size-3 rounded-sm', STATUS_CLASSES[status])} />- {STATUS_LABELS[status]}- </span>- ))}- </div>- </div>-)diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptKpiCard.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptKpiCard.tsxnew file mode 100644index 000000000000..c7371334f6da--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptKpiCard.tsx@@ -0,0 +1,54 @@+import { IconTrending } from '@posthog/icons'+import { LemonCard, LemonTag, Tooltip } from '@posthog/lemon-ui'++import { dayjs } from 'lib/dayjs'+import { IconTrendingDown, IconTrendingFlat } from 'lib/lemon-ui/icons'+import { percentage } from 'lib/utils/numbers'++import type { PromptRateKpi } from './promptReport'++const DeltaTag = ({ delta }: { delta: number }): JSX.Element => {+ const points = Math.round(delta * 100)+ const Icon = points === 0 ? IconTrendingFlat : points > 0 ? IconTrending : IconTrendingDown+ return (+ <Tooltip title="The latest day against the 30-day average, in percentage points">+ <LemonTag+ type={points === 0 ? 'muted' : points > 0 ? 'success' : 'danger'}+ size="small"+ icon={<Icon />}+ className="tabular-nums"+ >+ {points === 0 ? 'No change' : `${points > 0 ? '+' : ''}${points} pts`}+ </LemonTag>+ </Tooltip>+ )+}++export const ContentAutopilotPromptKpiCard = ({+ label,+ description,+ kpi,+ 'data-attr': dataAttr,+}: {+ label: string+ description: string+ kpi: PromptRateKpi+ 'data-attr'?: string+}): JSX.Element => (+ <LemonCard hoverEffect={false} className="p-3 flex flex-col gap-1 min-w-0" data-attr={dataAttr}>+ <div className="flex items-center justify-between gap-2">+ <Tooltip title={description}>+ <span className="text-muted text-xs font-semibold uppercase truncate">{label}</span>+ </Tooltip>+ {kpi.delta !== null ? <DeltaTag delta={kpi.delta} /> : null}+ </div>+ <span className="text-2xl font-semibold tabular-nums">+ {kpi.average === null ? 'No answers' : percentage(kpi.average, 0)}+ </span>+ <span className="text-muted text-xs">+ {kpi.latest === null || !kpi.latestDate+ ? 'Average over the last 30 days'+ : `Average over the last 30 days. On ${dayjs(kpi.latestDate).format('MMM D')}: ${percentage(kpi.latest, 0)}`}+ </span>+ </LemonCard>+)diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptQueryTile.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptQueryTile.tsxnew file mode 100644index 000000000000..448badc4c714--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptQueryTile.tsx@@ -0,0 +1,61 @@+import clsx from 'clsx'+import { ReactNode } from 'react'++import { LemonButton } from '@posthog/lemon-ui'++import { IconOpenInNew } from 'lib/lemon-ui/icons'+import { urls } from 'scenes/urls'+import { PagePerformanceCard } from 'scenes/web-analytics/PagePerformanceCard'+import { PagePerformanceCardHeader } from 'scenes/web-analytics/PagePerformanceCardHeader'++import { Query } from '~/queries/Query/Query'+import { DataTableNode, DataVisualizationNode } from '~/queries/schema/schema-general'+import { QueryContext } from '~/queries/types'++/** One report tile: a card with the web analytics tile header, a read-only query inside, and a way to open+ * the same query as an insight so it can be saved or added to a dashboard. */+export const ContentAutopilotPromptQueryTile = ({+ title,+ query,+ uniqueKey,+ context,+ actions,+ footer,+ className,+ bodyClassName,+ 'data-attr': dataAttr,+}: {+ title: string+ query: DataTableNode | DataVisualizationNode+ uniqueKey: string+ context?: QueryContext+ actions?: ReactNode+ footer?: ReactNode+ className?: string+ bodyClassName?: string+ 'data-attr': string+}): JSX.Element => (+ <PagePerformanceCard className={clsx('min-w-0', className)} footer={footer}>+ <PagePerformanceCardHeader+ title={title}+ actions={+ <div className="flex flex-wrap items-center gap-2">+ {actions}+ <LemonButton+ size="small"+ icon={<IconOpenInNew />}+ to={urls.insightNew({ query })}+ targetBlank+ tooltip="Open as insight"+ aria-label="Open as insight"+ data-attr={`${dataAttr}-open-insight`}+ />+ </div>+ }+ />+ {/* No flex-1 here: in the card's column layout it would override a fixed chart height from `bodyClassName`. */}+ <div className={clsx('flex flex-col min-h-0 min-w-0', bodyClassName)} data-attr={dataAttr}>+ <Query uniqueKey={uniqueKey} query={query} readOnly embedded context={context} />+ </div>+ </PagePerformanceCard>+)diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptSentiment.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptSentiment.tsxnew file mode 100644index 000000000000..b0b27a0af4a5--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptSentiment.tsx@@ -0,0 +1,53 @@+import { percentage } from 'lib/utils/numbers'+import { pluralize } from 'lib/utils/strings'++import type { DataVisualizationNode } from '~/queries/schema/schema-general'++import { ContentAutopilotPromptQueryTile } from './ContentAutopilotPromptQueryTile'+import type { PromptSentimentCounts, PromptSentimentSummary } from './promptReport'++const LABELS: { key: keyof PromptSentimentCounts; label: string }[] = [+ { key: 'positive', label: 'positive' },+ { key: 'neutral', label: 'neutral' },+ { key: 'negative', label: 'negative' },+]++export const ContentAutopilotPromptSentiment = ({+ summary,+ query,+}: {+ summary: PromptSentimentSummary+ query: DataVisualizationNode+}): JSX.Element => {+ if (summary.scored === 0) {+ return (+ <p className="m-0 text-muted">+ {summary.unscored > 0+ ? `${pluralize(summary.unscored, 'answer')} mention you but have not been scored yet.`+ : 'No sentences mention you yet, so there is nothing to score.'}+ </p>+ )+ }+ return (+ <div className="flex flex-col gap-3">+ <p className="m-0 text-sm">+ {pluralize(summary.scored, 'scored answer')} in the last 30 days:{' '}+ {LABELS.map(({ key, label }, index) => (+ <span key={key}>+ {index > 0 ? ', ' : ''}+ <span className="font-semibold tabular-nums">{summary[key]}</span> {label} (+ {percentage(summary[key] / summary.scored, 0)})+ </span>+ ))}+ {summary.unscored > 0 ? `. ${pluralize(summary.unscored, 'answer')} not scored yet.` : '.'}+ </p>+ <ContentAutopilotPromptQueryTile+ title="Scored answers per engine"+ query={query}+ uniqueKey={`aeo-prompt-sentiment-${query.source.values?.prompt_id ?? ''}`}+ bodyClassName="h-64 p-2"+ data-attr="content-autopilot-prompt-sentiment-tile"+ />+ </div>+ )+}diff --git a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.ts b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.tsindex 36ee312931f4..a33167888bba 100644--- a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.ts+++ b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.ts@@ -5,6 +5,7 @@ import { expectLogic } from 'kea-test-utils'import { ApiError } from 'lib/api-error'import { resumeKeaLoadersErrors, silenceKeaLoadersErrors } from '~/initKea'+import type { HogQLQuery } from '~/queries/schema/schema-general'import { initKeaTests } from '~/test/init'import {@@ -77,22 +78,38 @@ describe('contentAutopilotPromptDetailLogic', () => {expect(mockChecks).toHaveBeenCalledWith(TEAM_ID, PROMPT.id, { engine, limit: 1 })}expect(logic.values.prompt?.id).toBe(PROMPT.id)- expect(logic.values.historyGrid?.dates).toHaveLength(30)- expect(logic.values.historyGrid?.rows.map((row) => row.engine)).toEqual([- 'claude-web-search',- 'openai-web-search',- 'exa-answer',- ])- expect(logic.values.historyGrid?.rows[0].cells.slice(0, 4).map((cell) => cell.status)).toEqual([- 'cited',- 'cited',- 'no_run',- 'not_cited',- ])+ expect(logic.values.engines).toEqual(['claude-web-search', 'openai-web-search', 'exa-answer'])expect(logic.values.activeEngine).toBe('claude-web-search')expect(logic.values.shownCheck?.id).toBe(EXAMPLE_PROMPT_CHECKS[0].id)expect(mockVersions).toHaveBeenCalledWith(TEAM_ID, PROMPT.id)expect(logic.values.versions).toEqual(EXAMPLE_PROMPT_VERSIONS)+ expect(logic.values.reportKpis?.days).toBe(30)+ expect(logic.values.consensusMatrix?.rows.map((row) => row.engine)).toEqual([+ 'claude-web-search',+ 'openai-web-search',+ 'exa-answer',+ ])+ })++ it('scopes the report queries to the prompt and the stats window, and switches the visibility metric', async () => {+ logic.actions.openPromptDetail(PROMPT.id)+ await expectLogic(logic).toFinishAllListeners()++ const citedQuery = logic.values.visibilityQuery!+ expect(citedQuery.source.values).toEqual({ prompt_id: PROMPT.id, window_start: '2026-09-01 00:00:00' })+ expect(citedQuery.source.query).toContain('countIf(cited)')+ expect((logic.values.sourcesQuery!.source as HogQLQuery).values).toEqual(citedQuery.source.values)+ expect(logic.values.sentimentQuery!.source.values).toEqual(citedQuery.source.values)+ expect(logic.values.sentimentQuery!.source.query).toContain('brand_sentiment IS NOT NULL')++ logic.actions.setVisibilityMetric('mentioned')+ const mentionedQuery = logic.values.visibilityQuery!+ expect(mentionedQuery.source.query).toContain('{brand_pattern}')+ expect(mentionedQuery.source.values?.brand_pattern).toBe('\\b(example)\\b')++ // A new prompt starts over on the citation metric.+ logic.actions.openPromptDetail(EXAMPLE_PROMPTS[1].id)+ expect(logic.values.visibilityMetric).toBe('cited')})it('does not load versions for a prompt with one version', async () => {diff --git a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.ts b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.tsindex 36929e1573c1..7c636837692b 100644--- a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.ts+++ b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.ts@@ -6,6 +6,8 @@ import { ApiError } from 'lib/api-error'import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'import { teamLogic } from 'scenes/teamLogic'+import type { DataTableNode, DataVisualizationNode } from '~/queries/schema/schema-general'+import {aeoPromptsChecks,aeoPromptsPartialUpdate,@@ -22,7 +24,24 @@ import type {} from 'products/web_analytics/frontend/generated/api.schemas'import { contentAutopilotPromptsLogic } from './contentAutopilotPromptsLogic'-import { PromptHistoryGrid, promptHistoryGrid, sortEngines } from './utils'+import {+ PromptConsensusMatrix,+ PromptReportKpis,+ PromptSentimentSummary,+ PromptVisibilityMetric,+ promptConsensusMatrix,+ promptReportKpis,+ promptSentimentSummary,+} from './promptReport'+import {+ PromptReportWindow,+ promptCitationsQuery,+ promptSearchesQuery,+ promptSentimentQuery,+ promptSourcesQuery,+ promptVisibilityQuery,+} from './promptReportQueries'+import { sortEngines } from './utils'export const PROMPT_CHECKS_PAGE_SIZE = 10@@ -37,9 +56,10 @@ export interface contentAutopilotPromptDetailLogicValues {checksFailed: booleanchecksLoading: booleanchecksPage: number+ citationsQuery: DataTableNode | null+ consensusMatrix: PromptConsensusMatrix | nulldetailError: ContentAutopilotPromptDetailError | nullengines: string[]- historyGrid: PromptHistoryGrid | nulllastCheckedAt: string | nulllatestChecks: Record<string, AEOPromptCheckApi>latestChecksFailed: boolean@@ -48,14 +68,22 @@ export interface contentAutopilotPromptDetailLogicValues {promptArchiving: booleanpromptId: string | nullpromptLoading: boolean+ reportKpis: PromptReportKpis | null+ reportWindow: PromptReportWindow | null+ searchesQuery: DataTableNode | nullselectedCheck: AEOPromptCheckApi | nullselectedEngine: string | null+ sentimentQuery: DataVisualizationNode | null+ sentimentSummary: PromptSentimentSummary | nullshownCheck: AEOPromptCheckApi | null+ sourcesQuery: DataTableNode | nullstats: AEOPromptStatsApi | nullstatsLoading: booleanversions: AEOPromptVersionApi[] | nullversionsFailed: booleanversionsLoading: boolean+ visibilityMetric: PromptVisibilityMetric+ visibilityQuery: DataVisualizationNode | null}// Generated by kea-typegen. Update if you're an agent, ignore if you're human.@@ -196,6 +224,9 @@ export interface contentAutopilotPromptDetailLogicActions {setPromptArchivedSuccess: (prompt: AEOPromptApi) => {prompt: AEOPromptApi}+ setVisibilityMetric: (metric: PromptVisibilityMetric) => {+ metric: PromptVisibilityMetric+ }}// Generated by kea-typegen. Update if you're an agent, ignore if you're human.@@ -208,8 +239,19 @@ export interface contentAutopilotPromptDetailLogicMeta {activeEngine: string | null,latestChecks: Record<string, AEOPromptCheckApi>) => AEOPromptCheckApi | null- historyGrid: (stats: AEOPromptStatsApi | null, engines: string[]) => PromptHistoryGrid | nulllastCheckedAt: (stats: AEOPromptStatsApi | null) => string | null+ reportKpis: (stats: AEOPromptStatsApi | null) => PromptReportKpis | null+ consensusMatrix: (stats: AEOPromptStatsApi | null, engines: string[]) => PromptConsensusMatrix | null+ sentimentSummary: (stats: AEOPromptStatsApi | null, engines: string[]) => PromptSentimentSummary | null+ reportWindow: (promptId: string | null, stats: AEOPromptStatsApi | null) => PromptReportWindow | null+ visibilityQuery: (+ reportWindow: PromptReportWindow | null,+ visibilityMetric: PromptVisibilityMetric+ ) => DataVisualizationNode | null+ sourcesQuery: (reportWindow: PromptReportWindow | null) => DataTableNode | null+ citationsQuery: (reportWindow: PromptReportWindow | null) => DataTableNode | null+ searchesQuery: (reportWindow: PromptReportWindow | null) => DataTableNode | null+ sentimentQuery: (reportWindow: PromptReportWindow | null) => DataVisualizationNode | null}}@@ -245,6 +287,7 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetaiselectEngine: (engine: string) => ({ engine }),selectCheck: (check: AEOPromptCheckApi) => ({ check }),setChecksPage: (page: number) => ({ page }),+ setVisibilityMetric: (metric: PromptVisibilityMetric) => ({ metric }),}),loaders(({ values }) => ({prompt: [@@ -379,6 +422,13 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetaiselectCheck: (_, { check }) => check,},],+ visibilityMetric: [+ 'cited' as PromptVisibilityMetric,+ {+ openPromptDetail: () => 'cited',+ setVisibilityMetric: (_, { metric }) => metric,+ },+ ],detailError: [null as ContentAutopilotPromptDetailError | null,{@@ -421,11 +471,6 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetai): AEOPromptCheckApi | null =>selectedCheck ?? (activeEngine ? (latestChecks[activeEngine] ?? null) : null),],- historyGrid: [- (s) => [s.stats, s.engines],- (stats: AEOPromptStatsApi | null, engines: string[]): PromptHistoryGrid | null =>- stats ? promptHistoryGrid(stats, engines) : null,- ],lastCheckedAt: [(s) => [s.stats],(stats: AEOPromptStatsApi | null): string | null =>@@ -435,6 +480,50 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetainull),],+ reportKpis: [+ (s) => [s.stats],+ (stats: AEOPromptStatsApi | null): PromptReportKpis | null => (stats ? promptReportKpis(stats) : null),+ ],+ consensusMatrix: [+ (s) => [s.stats, s.engines],+ (stats: AEOPromptStatsApi | null, engines: string[]): PromptConsensusMatrix | null =>+ stats ? promptConsensusMatrix(stats, engines) : null,+ ],+ sentimentSummary: [+ (s) => [s.stats, s.engines],+ (stats: AEOPromptStatsApi | null, engines: string[]): PromptSentimentSummary | null =>+ stats ? promptSentimentSummary(stats, engines) : null,+ ],+ reportWindow: [+ (s) => [s.promptId, s.stats],+ (promptId: string | null, stats: AEOPromptStatsApi | null): PromptReportWindow | null =>+ promptId && stats ? { promptId, from: stats.daily_from, brandNames: stats.brand_names } : null,+ ],+ visibilityQuery: [+ (s) => [s.reportWindow, s.visibilityMetric],+ (reportWindow: PromptReportWindow | null, visibilityMetric: PromptVisibilityMetric) =>+ reportWindow ? promptVisibilityQuery(reportWindow, visibilityMetric) : null,+ ],+ sourcesQuery: [+ (s) => [s.reportWindow],+ (reportWindow: PromptReportWindow | null): DataTableNode | null =>+ reportWindow ? promptSourcesQuery(reportWindow) : null,+ ],+ citationsQuery: [+ (s) => [s.reportWindow],+ (reportWindow: PromptReportWindow | null): DataTableNode | null =>+ reportWindow ? promptCitationsQuery(reportWindow) : null,+ ],+ searchesQuery: [+ (s) => [s.reportWindow],+ (reportWindow: PromptReportWindow | null): DataTableNode | null =>+ reportWindow ? promptSearchesQuery(reportWindow) : null,+ ],+ sentimentQuery: [+ (s) => [s.reportWindow],+ (reportWindow: PromptReportWindow | null): DataVisualizationNode | null =>+ reportWindow ? promptSentimentQuery(reportWindow) : null,+ ],}),listeners(({ actions, values }) => ({openPromptDetail: ({ promptId }) => {diff --git a/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.ts b/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.tsindex 88007a0eb620..373787c9dac4 100644--- a/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.ts+++ b/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.ts@@ -348,6 +348,8 @@ const EXAMPLE_CHECK: AEOPromptCheckApi = {top_cited_domains: ['analytics-review.example.com', 'example.com', 'community.example.org'],search_queries: ['best open source product analytics', 'self-hosted product analytics for startups'],prompt_version: 2,+ brand_sentiment: 'positive',+ brand_sentiment_score: 0.86,answer_text:'For a small startup, a few open source product analytics tools stand out.\n\n' +'1. Example Analytics: event tracking, funnels and retention, with a generous free tier.\n' +@@ -367,6 +369,8 @@ export const EXAMPLE_PROMPT_CHECKS: AEOPromptCheckApi[] = [target_best_position: null,top_cited_domains: ['analytics-review.example.com'],search_queries: ['open source product analytics startup'],+ brand_sentiment: null,+ brand_sentiment_score: null,answer_text: 'Most startups begin with a hosted tool.\nCompare pricing tiers before you commit.',},{@@ -383,23 +387,27 @@ export const EXAMPLE_PROMPT_CHECKS: AEOPromptCheckApi[] = [target_best_position: null,top_cited_domains: [],search_queries: [],+ brand_sentiment: null,+ brand_sentiment_score: null,answer_text: null,},]-const EXAMPLE_ENGINE_DAYS: Record<string, ('cited' | 'not_cited' | 'failed' | null)[]> = {- 'claude-web-search': ['cited', 'cited', null, 'not_cited', 'cited', 'cited', 'cited', 'failed', 'cited', 'cited'],+type ExampleDay = 'cited' | 'mentioned' | 'not_cited' | 'failed' | null++const EXAMPLE_ENGINE_DAYS: Record<string, ExampleDay[]> = {+ 'claude-web-search': ['cited', 'cited', null, 'mentioned', 'cited', 'cited', 'cited', 'failed', 'cited', 'cited'],'openai-web-search': ['not_cited',- 'not_cited',+ 'mentioned',null,'cited','not_cited',- 'not_cited',+ 'mentioned',null,'not_cited','cited',- 'not_cited',+ 'mentioned',],'exa-answer': ['cited', 'failed', null, 'cited', 'cited', 'not_cited', 'cited', 'cited', 'failed', 'cited'],}@@ -413,29 +421,62 @@ const EXAMPLE_PROMPT_DAILY: AEOPromptStatsApi['daily'] = Object.entries(EXAMPLE_engine,checks: 1,cited: status === 'cited' ? 1 : 0,+ mentioned: status === 'cited' || status === 'mentioned' ? 1 : 0,failed: status === 'failed' ? 1 : 0,})))const exampleEngineStats = (engine: string): AEOPromptStatsApi['engines'][number] => {const days = EXAMPLE_PROMPT_DAILY.filter((row) => row.engine === engine)- const sum = (key: 'checks' | 'cited' | 'failed'): number => days.reduce((total, row) => total + row[key], 0)+ const sum = (key: 'checks' | 'cited' | 'mentioned' | 'failed'): number =>+ days.reduce((total, row) => total + row[key], 0)const answered = sum('checks') - sum('failed')return {engine,checks: sum('checks'),cited: sum('cited'),+ mentioned: sum('mentioned'),failed: sum('failed'),citation_rate: answered ? sum('cited') / answered : null,+ mention_rate: answered ? sum('mentioned') / answered : null,first_checked_at: `${days[0].date}T09:00:00Z`,last_checked_at: `${days[days.length - 1].date}T09:00:00Z`,}}+const EXAMPLE_CONSENSUS_DOMAINS = [+ 'analytics-review.example.com',+ 'example.com',+ 'community.example.org',+ 'rival.example.net',+ 'docs.rival.example.net',+]+export const EXAMPLE_PROMPT_STATS: AEOPromptStatsApi = {total_checks: EXAMPLE_PROMPT_DAILY.length,+ brand_names: ['example'],daily_from: '2026-09-01',daily_to: '2026-09-30',engines: Object.keys(EXAMPLE_ENGINE_DAYS).map(exampleEngineStats),daily: EXAMPLE_PROMPT_DAILY,+ consensus: Object.keys(EXAMPLE_ENGINE_DAYS).flatMap((engine, engineIndex) =>+ EXAMPLE_CONSENSUS_DOMAINS.map((domain, domainIndex) => ({+ engine,+ domain,+ checks: 24,+ checks_citing: Math.max(0, 22 - domainIndex * 5 - engineIndex * 3),+ }))+ ),+ sentiment: [+ { engine: 'claude-web-search', positive: 14, neutral: 6, negative: 1, unscored: 2 },+ { engine: 'openai-web-search', positive: 5, neutral: 9, negative: 3, unscored: 0 },+ { engine: 'exa-answer', positive: 0, neutral: 0, negative: 0, unscored: 18 },+ ],+ top_words: [+ { word: 'analytics', count: 41 },+ { word: 'product', count: 38 },+ { word: 'open', count: 22 },+ { word: 'source', count: 22 },+ { word: 'startups', count: 11 },+ ],}diff --git a/products/web_analytics/frontend/contentAutopilot/promptReport.test.ts b/products/web_analytics/frontend/contentAutopilot/promptReport.test.tsnew file mode 100644index 000000000000..ebb2e543727b--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/promptReport.test.ts@@ -0,0 +1,116 @@+import type { AEOPromptStatsApi } from '../generated/api.schemas'+import { EXAMPLE_PROMPT_STATS } from './contentAutopilotStoryFixtures'+import { promptConsensusMatrix, promptReportKpis, promptSentimentSummary } from './promptReport'++const day = (+ date: string,+ engine: string,+ counts: Partial<Pick<AEOPromptStatsApi['daily'][number], 'checks' | 'cited' | 'mentioned' | 'failed'>>+): AEOPromptStatsApi['daily'][number] => ({ date, engine, checks: 1, cited: 0, mentioned: 0, failed: 0, ...counts })++const stats = (daily: AEOPromptStatsApi['daily'], overrides: Partial<AEOPromptStatsApi> = {}): AEOPromptStatsApi => ({+ ...EXAMPLE_PROMPT_STATS,+ daily_from: '2026-09-01',+ daily_to: '2026-09-10',+ daily,+ ...overrides,+})++describe('promptReport', () => {+ describe('promptReportKpis', () => {+ it('pools the window and compares the latest answered day against it', () => {+ const kpis = promptReportKpis(+ stats([+ day('2026-09-01', 'a', { cited: 1, mentioned: 1 }),+ day('2026-09-01', 'b', { mentioned: 1 }),+ day('2026-09-02', 'a', { failed: 1 }),+ day('2026-09-05', 'a', {}),+ day('2026-09-05', 'b', { cited: 1, mentioned: 1 }),+ // The latest day only failed, so the day before it is the latest with an answer.+ day('2026-09-10', 'a', { failed: 1 }),+ ])+ )++ expect(kpis).toEqual({+ answers: 4,+ days: 10,+ visibility: { average: 0.75, latest: 0.5, latestDate: '2026-09-05', delta: -0.25 },+ citation: { average: 0.5, latest: 0.5, latestDate: '2026-09-05', delta: 0 },+ })+ })++ it.each([+ ['no checks', []],+ ['only failed checks', [day('2026-09-03', 'a', { failed: 1 })]],+ ])('has no rates with %s', (_name, daily) => {+ const kpis = promptReportKpis(stats(daily))+ expect(kpis.answers).toBe(0)+ expect(kpis.visibility).toEqual({ average: null, latest: null, latestDate: null, delta: null })+ expect(kpis.citation).toEqual(kpis.visibility)+ })+ })++ it('builds the consensus matrix in engine order and drops engines without answers', () => {+ const matrix = promptConsensusMatrix(+ stats([], {+ consensus: [+ { engine: 'b', domain: 'x.example', checks_citing: 1, checks: 4 },+ { engine: 'b', domain: 'y.example', checks_citing: 4, checks: 4 },+ { engine: 'a', domain: 'x.example', checks_citing: 2, checks: 2 },+ { engine: 'a', domain: 'y.example', checks_citing: 0, checks: 2 },+ { engine: 'c', domain: 'x.example', checks_citing: 0, checks: 0 },+ ],+ }),+ ['a', 'b', 'c']+ )++ expect(matrix.domains).toEqual(['x.example', 'y.example'])+ expect(matrix.rows).toEqual([+ {+ engine: 'a',+ checks: 2,+ cells: [+ { domain: 'x.example', checksCiting: 2, share: 1 },+ { domain: 'y.example', checksCiting: 0, share: 0 },+ ],+ },+ {+ engine: 'b',+ checks: 4,+ cells: [+ { domain: 'x.example', checksCiting: 1, share: 0.25 },+ { domain: 'y.example', checksCiting: 4, share: 1 },+ ],+ },+ ])+ })++ it.each([+ [+ 'some scored',+ [+ { engine: 'a', positive: 3, neutral: 1, negative: 0, unscored: 2 },+ { engine: 'b', positive: 0, neutral: 0, negative: 0, unscored: 5 },+ ],+ {+ positive: 3,+ neutral: 1,+ negative: 0,+ unscored: 7,+ scored: 4,+ byEngine: [{ engine: 'a', scored: 4, positive: 3, neutral: 1, negative: 0 }],+ },+ ],+ // Scoring off: mentions exist but nothing is scored, so the UI must not claim there are no mentions.+ [+ 'mentions but nothing scored',+ [{ engine: 'a', positive: 0, neutral: 0, negative: 0, unscored: 4 }],+ { positive: 0, neutral: 0, negative: 0, unscored: 4, scored: 0, byEngine: [] },+ ],+ ])(+ 'sums sentiment across engines with %s and lists only engines with scored answers',+ (_name, sentiment, expected) => {+ expect(promptSentimentSummary(stats([], { sentiment }), ['b', 'a'])).toEqual(expected)+ }+ )+})diff --git a/products/web_analytics/frontend/contentAutopilot/promptReport.ts b/products/web_analytics/frontend/contentAutopilot/promptReport.tsnew file mode 100644index 000000000000..862bd860db5a--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/promptReport.ts@@ -0,0 +1,162 @@+import { dayjs } from 'lib/dayjs'++import type {+ AEOPromptDailyEngineStatsApi,+ AEOPromptStatsApi,+} from 'products/web_analytics/frontend/generated/api.schemas'++export type PromptVisibilityMetric = 'cited' | 'mentioned'++export interface PromptRateKpi {+ /** Pooled rate over the window: all answers that qualify over all answers. */+ average: number | null+ /** Rate on the latest day of the window that had an answer. */+ latest: number | null+ latestDate: string | null+ /** `latest - average`, in rate units (0.12 is 12 percentage points). */+ delta: number | null+}++export interface PromptReportKpis {+ answers: number+ days: number+ visibility: PromptRateKpi+ citation: PromptRateKpi+}++export interface PromptConsensusCell {+ domain: string+ checksCiting: number+ /** Share of the engine's answers that cited the domain. Null when the engine answered nothing. */+ share: number | null+}++export interface PromptConsensusMatrix {+ domains: string[]+ rows: { engine: string; checks: number; cells: PromptConsensusCell[] }[]+}++export interface PromptSentimentCounts {+ positive: number+ neutral: number+ negative: number+}++export interface PromptSentimentSummary extends PromptSentimentCounts {+ scored: number+ unscored: number+ byEngine: ({ engine: string; scored: number } & PromptSentimentCounts)[]+}++interface DailyTotals {+ checks: number+ cited: number+ mentioned: number+ failed: number+}++/** Every day of the stats window as YYYY-MM-DD, oldest first. */+export const statsWindowDates = (stats: Pick<AEOPromptStatsApi, 'daily_from' | 'daily_to'>): string[] => {+ const dates: string[] = []+ for (let day = dayjs(stats.daily_from); !day.isAfter(dayjs(stats.daily_to), 'day'); day = day.add(1, 'day')) {+ dates.push(day.format('YYYY-MM-DD'))+ }+ return dates+}++const sumDaily = (rows: AEOPromptDailyEngineStatsApi[]): DailyTotals =>+ rows.reduce(+ (totals, row) => ({+ checks: totals.checks + row.checks,+ cited: totals.cited + row.cited,+ mentioned: totals.mentioned + row.mentioned,+ failed: totals.failed + row.failed,+ }),+ { checks: 0, cited: 0, mentioned: 0, failed: 0 }+ )++const rate = (part: number, whole: number): number | null => (whole > 0 ? part / whole : null)++const rateKpi = (+ byDay: { date: string; checks: number; failed: number; count: number }[],+ totals: { checks: number; failed: number; count: number }+): PromptRateKpi => {+ const average = rate(totals.count, totals.checks - totals.failed)+ const latestDay = [...byDay].reverse().find((day) => day.checks - day.failed > 0) ?? null+ const latest = latestDay ? rate(latestDay.count, latestDay.checks - latestDay.failed) : null+ return {+ average,+ latest,+ latestDate: latestDay?.date ?? null,+ delta: average !== null && latest !== null ? latest - average : null,+ }+}++/** Window-wide visibility and citation rates, with the latest day and how far it sits from the average. */+export const promptReportKpis = (stats: AEOPromptStatsApi): PromptReportKpis => {+ const dates = statsWindowDates(stats)+ const byDate = new Map<string, AEOPromptDailyEngineStatsApi[]>()+ for (const row of stats.daily) {+ byDate.set(row.date, [...(byDate.get(row.date) ?? []), row])+ }+ const days = dates.map((date) => ({ date, ...sumDaily(byDate.get(date) ?? []) }))+ const totals = sumDaily(stats.daily)+ return {+ answers: totals.checks - totals.failed,+ days: dates.length,+ visibility: rateKpi(+ days.map((day) => ({ ...day, count: day.mentioned })),+ { ...totals, count: totals.mentioned }+ ),+ citation: rateKpi(+ days.map((day) => ({ ...day, count: day.cited })),+ { ...totals, count: totals.cited }+ ),+ }+}++/** Engines as rows, the most cited domains as columns, each cell the share of the engine's answers citing it. */+export const promptConsensusMatrix = (stats: AEOPromptStatsApi, engines: string[]): PromptConsensusMatrix => {+ const domains = [...new Set(stats.consensus.map((row) => row.domain))]+ const byKey = new Map(stats.consensus.map((row) => [`${row.engine}|${row.domain}`, row]))+ return {+ domains,+ rows: engines+ .map((engine) => {+ const checks = stats.consensus.find((row) => row.engine === engine)?.checks ?? 0+ return {+ engine,+ checks,+ cells: domains.map((domain) => {+ const checksCiting = byKey.get(`${engine}|${domain}`)?.checks_citing ?? 0+ return { domain, checksCiting, share: rate(checksCiting, checks) }+ }),+ }+ })+ .filter((row) => row.checks > 0),+ }+}++/** Sentiment counts across engines, plus one row per engine that has at least one scored answer. */+export const promptSentimentSummary = (stats: AEOPromptStatsApi, engines: string[]): PromptSentimentSummary => {+ const byEngine = engines.flatMap((engine) => {+ const row = stats.sentiment.find((item) => item.engine === engine)+ if (!row) {+ return []+ }+ const scored = row.positive + row.neutral + row.negative+ return scored > 0+ ? [{ engine, scored, positive: row.positive, neutral: row.neutral, negative: row.negative }]+ : []+ })+ const totals = stats.sentiment.reduce(+ (sum, row) => ({+ positive: sum.positive + row.positive,+ neutral: sum.neutral + row.neutral,+ negative: sum.negative + row.negative,+ unscored: sum.unscored + row.unscored,+ }),+ { positive: 0, neutral: 0, negative: 0, unscored: 0 }+ )+ return { ...totals, scored: totals.positive + totals.neutral + totals.negative, byEngine }+}diff --git a/products/web_analytics/frontend/contentAutopilot/promptReportColumns.tsx b/products/web_analytics/frontend/contentAutopilot/promptReportColumns.tsxnew file mode 100644index 000000000000..c712483a8499--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/promptReportColumns.tsx@@ -0,0 +1,79 @@+import { LemonTag, Link } from '@posthog/lemon-ui'++import { LemonProgress } from 'lib/lemon-ui/LemonProgress'+import { percentage } from 'lib/utils/numbers'++import type { QueryContext, QueryContextColumnComponent } from '~/queries/types'++import { PROMPT_CITATION_COLUMNS, PROMPT_SOURCE_COLUMNS } from './promptReportQueries'++// Engine output is third-party text: only plain http(s) URLs become links.+const isWebUrl = (url: string): boolean => /^https?:\/\//i.test(url)++/** HogQL rows reach the column renderers as arrays, so a renderer that needs a sibling column looks it up by position. */+const cellAt = (record: unknown, columns: readonly string[], name: string): unknown =>+ Array.isArray(record) ? record[columns.indexOf(name)] : undefined++const TargetTag = (): JSX.Element => (+ <LemonTag type="success" size="small">+ Your site+ </LemonTag>+)++const DomainCell: QueryContextColumnComponent = ({ value, record }) => (+ <span className="flex flex-wrap items-center gap-2">+ <span className="break-all">{String(value ?? '')}</span>+ {cellAt(record, PROMPT_SOURCE_COLUMNS, 'is_target') ? <TargetTag /> : null}+ </span>+)++const UrlCell: QueryContextColumnComponent = ({ value, record }) => {+ const url = String(value ?? '')+ return (+ <span className="flex flex-wrap items-center gap-2 min-w-0">+ {isWebUrl(url) ? (+ <Link to={url} target="_blank" className="break-all">+ {url}+ </Link>+ ) : (+ <span className="break-all">{url}</span>+ )}+ {cellAt(record, PROMPT_CITATION_COLUMNS, 'is_target') ? <TargetTag /> : null}+ </span>+ )+}++const ShareCell: QueryContextColumnComponent = ({ value }) => {+ const share = typeof value === 'number' ? value : Number(value ?? 0)+ return (+ <span className="flex items-center gap-2 min-w-32">+ <LemonProgress percent={share * 100} className="flex-1" />+ <span className="w-10 text-right tabular-nums">{percentage(share, 0)}</span>+ </span>+ )+}++const sharedColumns: QueryContext['columns'] = {+ answers: { title: 'Answers that link to it', align: 'right' },+ share_of_answers: { title: 'Share of all answers', render: ShareCell, align: 'right', width: 180 },+ is_target: { hidden: true },+}++export const promptSourcesContext: QueryContext = {+ showQueryEditor: false,+ columns: { ...sharedColumns, domain: { title: 'Site', render: DomainCell } },+}++export const promptCitationsContext: QueryContext = {+ showQueryEditor: false,+ columns: { ...sharedColumns, url: { title: 'Page', render: UrlCell } },+}++export const promptSearchesContext: QueryContext = {+ showQueryEditor: false,+ columns: {+ query: { title: 'Search' },+ searches: { title: 'Times run', align: 'right' },+ engines: { title: 'Engines that ran it', align: 'right' },+ },+}diff --git a/products/web_analytics/frontend/contentAutopilot/promptReportQueries.ts b/products/web_analytics/frontend/contentAutopilot/promptReportQueries.tsnew file mode 100644index 000000000000..599568c21550--- /dev/null+++ b/products/web_analytics/frontend/contentAutopilot/promptReportQueries.ts@@ -0,0 +1,158 @@+import { escapeRegex } from 'lib/utils/actions'++import { DataTableNode, DataVisualizationNode, HogQLQuery, NodeKind } from '~/queries/schema/schema-general'+import { ChartDisplayType } from '~/types'++import type { PromptVisibilityMetric } from './promptReport'++export interface PromptReportWindow {+ promptId: string+ /** First day of the window, YYYY-MM-DD in the project's timezone. */+ from: string+ /** The words an answer must contain to count as a brand mention. */+ brandNames: string[]+}++export const PROMPT_REPORT_TOP_SOURCES = 50+export const PROMPT_REPORT_TOP_CITATIONS = 20+export const PROMPT_REPORT_TOP_SEARCHES = 30++export const PROMPT_SOURCE_COLUMNS = ['domain', 'answers', 'share_of_answers', 'is_target'] as const+export const PROMPT_CITATION_COLUMNS = ['url', 'answers', 'share_of_answers', 'is_target'] as const+export const PROMPT_SEARCH_COLUMNS = ['query', 'searches', 'engines'] as const++// Every query is one prompt's answered checks inside the window. The prompt id and the window start travel+// as placeholder values, so the HogQL text never embeds user data.+const WHERE = 'prompt_id = {prompt_id} AND created_at >= toDateTime({window_start}) AND NOT check_failed'++const ANSWERED = `(SELECT count() FROM system.aeo_citation_checks WHERE ${WHERE})`++const hogql = (query: string, window: PromptReportWindow, values: Record<string, string> = {}): HogQLQuery => ({+ kind: NodeKind.HogQLQuery,+ query,+ values: { prompt_id: window.promptId, window_start: `${window.from} 00:00:00`, ...values },+})++/** RE2 pattern for `match()`: any brand name as a whole word, case folded by the caller. */+export const brandMentionPattern = (brandNames: string[]): string =>+ `\\b(${brandNames.map((name) => escapeRegex(name.toLowerCase())).join('|')})\\b`++const mentionedExpr = (brandNames: string[]): string =>+ brandNames.length ? "cited OR match(lower(coalesce(answer_text, '')), {brand_pattern})" : 'cited'++/** Daily citation (or brand mention) rate per engine. Days an engine did not run are gaps, not zeros. */+export const promptVisibilityQuery = (+ window: PromptReportWindow,+ metric: PromptVisibilityMetric+): DataVisualizationNode => {+ const counted = metric === 'mentioned' ? mentionedExpr(window.brandNames) : 'cited'+ return {+ kind: NodeKind.DataVisualizationNode,+ source: hogql(+ `SELECT toDate(created_at) AS day, engine, round(countIf(${counted}) / count(), 3) AS rate+FROM system.aeo_citation_checks+WHERE ${WHERE}+GROUP BY day, engine+ORDER BY day, engine`,+ window,+ metric === 'mentioned' && window.brandNames.length+ ? { brand_pattern: brandMentionPattern(window.brandNames) }+ : {}+ ),+ display: ChartDisplayType.ActionsLineGraph,+ chartSettings: {+ xAxis: { column: 'day' },+ yAxis: [{ column: 'rate', settings: { formatting: { style: 'percent', decimalPlaces: 0 } } }],+ seriesBreakdownColumn: 'engine',+ showLegend: true,+ },+ }+}++const citedUrls = `SELECT id, engine, arrayJoin(JSONExtract(cited_urls, 'Array(String)')) AS url, target_urls+FROM system.aeo_citation_checks+WHERE ${WHERE}`++const table = (source: HogQLQuery, columns: readonly string[]): DataTableNode => ({+ kind: NodeKind.DataTableNode,+ source,+ columns: [...columns],+ hiddenColumns: columns.includes('is_target') ? ['is_target'] : undefined,+ embedded: true,+ showActions: false,+ showOpenEditorButton: false,+})++/** Cited domains ranked by how many answers cited them, with the share of all answers that did. */+export const promptSourcesQuery = (window: PromptReportWindow): DataTableNode =>+ table(+ hogql(+ `SELECT domain(url) AS domain,+ uniq(id) AS answers,+ round(uniq(id) / ${ANSWERED}, 3) AS share_of_answers,+ max(has(JSONExtract(target_urls, 'Array(String)'), url)) AS is_target+FROM (${citedUrls})+WHERE domain != ''+GROUP BY domain+ORDER BY answers DESC, domain+LIMIT ${PROMPT_REPORT_TOP_SOURCES}`,+ window+ ),+ PROMPT_SOURCE_COLUMNS+ )++/** Cited URLs ranked by how many answers cited them. */+export const promptCitationsQuery = (window: PromptReportWindow): DataTableNode =>+ table(+ hogql(+ `SELECT url,+ uniq(id) AS answers,+ round(uniq(id) / ${ANSWERED}, 3) AS share_of_answers,+ max(has(JSONExtract(target_urls, 'Array(String)'), url)) AS is_target+FROM (${citedUrls})+GROUP BY url+ORDER BY answers DESC, url+LIMIT ${PROMPT_REPORT_TOP_CITATIONS}`,+ window+ ),+ PROMPT_CITATION_COLUMNS+ )++/** The searches the engines ran, with how often. Only engines that report their searches contribute. */+export const promptSearchesQuery = (window: PromptReportWindow): DataTableNode =>+ table(+ hogql(+ `SELECT query, count() AS searches, uniq(engine) AS engines+FROM (SELECT engine, arrayJoin(JSONExtract(search_queries, 'Array(String)')) AS query+ FROM system.aeo_citation_checks+ WHERE ${WHERE})+WHERE query != ''+GROUP BY query+ORDER BY searches DESC, query+LIMIT ${PROMPT_REPORT_TOP_SEARCHES}`,+ window+ ),+ PROMPT_SEARCH_COLUMNS+ )++/** Scored answers per engine, split by how they speak about the brand. Unscored answers are left out. */+export const promptSentimentQuery = (window: PromptReportWindow): DataVisualizationNode => ({+ kind: NodeKind.DataVisualizationNode,+ source: hogql(+ `SELECT engine,+ multiIf(brand_sentiment = 'positive', 'Positive', brand_sentiment = 'negative', 'Negative', 'Neutral') AS sentiment,+ count() AS answers+FROM system.aeo_citation_checks+WHERE ${WHERE} AND brand_sentiment IS NOT NULL+GROUP BY engine, sentiment+ORDER BY engine, sentiment`,+ window+ ),+ display: ChartDisplayType.ActionsStackedBar,+ chartSettings: {+ xAxis: { column: 'engine' },+ yAxis: [{ column: 'answers' }],+ seriesBreakdownColumn: 'sentiment',+ showLegend: true,+ },+})diff --git a/products/web_analytics/frontend/contentAutopilot/utils.ts b/products/web_analytics/frontend/contentAutopilot/utils.tsindex ba3ddd968f2f..a3791e3d2656 100644--- a/products/web_analytics/frontend/contentAutopilot/utils.ts+++ b/products/web_analytics/frontend/contentAutopilot/utils.ts@@ -1,9 +1,7 @@-import { dayjs } from 'lib/dayjs'import { parseWebAnalyticsURL } from 'scenes/web-analytics/common'import type {AEOPromptCheckApi,- AEOPromptStatsApi,ContentAutopilotOpportunityApi,ContentAutopilotProposalListApi,} from 'products/web_analytics/frontend/generated/api.schemas'@@ -26,48 +24,6 @@ export const sortEngines = (engines: Iterable<string>): string[] => {return [...new Set(engines)].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))}-export type PromptHistoryCellStatus = 'cited' | 'not_cited' | 'failed' | 'no_run'--export interface PromptHistoryCell {- date: string- status: PromptHistoryCellStatus- checks: number- cited: number- failed: number-}--export interface PromptHistoryGrid {- dates: string[]- rows: { engine: string; cells: PromptHistoryCell[] }[]-}--const historyCellStatus = ({- checks,- cited,- failed,-}: Omit<PromptHistoryCell, 'date' | 'status'>): PromptHistoryCellStatus =>- checks === 0 ? 'no_run' : cited > 0 ? 'cited' : failed === checks ? 'failed' : 'not_cited'--/** One cell per day and engine across the stats window, with days the engine did not run filled in. */-export const promptHistoryGrid = (stats: AEOPromptStatsApi, engines: string[]): PromptHistoryGrid => {- const dates: string[] = []- for (let day = dayjs(stats.daily_from); !day.isAfter(dayjs(stats.daily_to), 'day'); day = day.add(1, 'day')) {- dates.push(day.format('YYYY-MM-DD'))- }- const byKey = new Map(stats.daily.map((row) => [`${row.engine}|${row.date}`, row]))- return {- dates,- rows: engines.map((engine) => ({- engine,- cells: dates.map((date) => {- const row = byKey.get(`${engine}|${date}`)- const counts = { checks: row?.checks ?? 0, cited: row?.cited ?? 0, failed: row?.failed ?? 0 }- return { date, status: historyCellStatus(counts), ...counts }- }),- })),- }-}-export type ContentAutopilotPageTarget =| { isNewPage: true; path: string }| { isNewPage: false; path: string; url: string }diff --git a/products/web_analytics/frontend/generated/api.schemas.ts b/products/web_analytics/frontend/generated/api.schemas.tsindex 051a0eabf951..5c41a73282d6 100644--- a/products/web_analytics/frontend/generated/api.schemas.ts+++ b/products/web_analytics/frontend/generated/api.schemas.ts@@ -147,6 +147,16 @@ export interface AEOPromptCheckApi {* @nullable*/prompt_version: number | null+ /**+ * Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand.+ * @nullable+ */+ brand_sentiment?: string | null+ /**+ * Classifier confidence (0-1) in 'brand_sentiment'. Null when 'brand_sentiment' is null.+ * @nullable+ */+ brand_sentiment_score?: number | null}export interface PaginatedAEOPromptCheckListApi {@@ -164,12 +174,19 @@ export interface AEOPromptEngineStatsApi {* @nullable*/readonly citation_rate: number | null+ /**+ * Share (0-1) of answered checks whose answer names the brand. Failed checks are left out. Null when no check was answered.+ * @nullable+ */+ readonly mention_rate: number | null/** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */engine: string/** Checks of the prompt on this engine, failed ones included. */checks: number/** Checks whose answer cited a target domain. */cited: number+ /** Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */+ mentioned: number/** Checks where the engine did not answer. */failed: number/** When the engine first ran the prompt. */@@ -187,17 +204,58 @@ export interface AEOPromptDailyEngineStatsApi {checks: number/** Checks that day whose answer cited a target domain. */cited: number+ /** Checks that day whose answer names the brand. Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */+ mentioned: number/** Checks that day where the engine did not answer. */failed: number}+export interface AEOPromptEngineDomainShareApi {+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */+ engine: string+ /** One of the most cited domains across all engines in the window. */+ domain: string+ /** This engine's answered checks in the last 30 days that cited the domain. */+ checks_citing: number+ /** This engine's answered checks in the last 30 days. */+ checks: number+}++export interface AEOPromptEngineSentimentApi {+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */+ engine: string+ /** Answers in the last 30 days whose brand sentences read positive. */+ positive: number+ /** Answers in the last 30 days whose brand sentences read neutral. */+ neutral: number+ /** Answers in the last 30 days whose brand sentences read negative. */+ negative: number+ /** Answers in the last 30 days that name the brand but have no sentiment yet. */+ unscored: number+}++export interface AEOPromptSearchWordApi {+ /** A word from the web searches, lower-cased, common English words left out. */+ word: string+ /** How many searches used the word, weighted by how often each search ran. */+ count: number+}+export interface AEOPromptStatsApi {/** Totals per answer engine over the prompt's whole history. */engines: AEOPromptEngineStatsApi[]/** Counts per day and engine from 'daily_from' to 'daily_to' (30 days). Days and engines without a check are left out. */daily: AEOPromptDailyEngineStatsApi[]+ /** One row per engine and per domain for the 10 most cited domains of the last 30 days: how many of the engine's answers cited the domain. Shows which sources the engines agree on. */+ consensus: AEOPromptEngineDomainShareApi[]+ /** Brand sentiment of the answers in the last 30 days, one row per engine. Only the sentences that name the brand are scored. */+ sentiment: AEOPromptEngineSentimentApi[]+ /** Most used words across the web searches the engines ran in the last 30 days, at most 20. Only engines that report their searches contribute. */+ top_words: AEOPromptSearchWordApi[]/** Checks of the prompt across all engines, failed ones included. */total_checks: number+ /** Lower-cased brand names, derived from the project's target domains. An answer that contains one of them as a word counts as a mention. */+ brand_names: string[]/** First day of the daily series, in the project's timezone. */daily_from: string/** Last day of the daily series (today), in the project's timezone. */diff --git a/products/web_analytics/frontend/generated/api.ts b/products/web_analytics/frontend/generated/api.tsindex b979279ca9a8..3831844f7c9b 100644--- a/products/web_analytics/frontend/generated/api.ts+++ b/products/web_analytics/frontend/generated/api.ts@@ -227,7 +227,7 @@ export const getAeoPromptsStatsUrl = (projectId: string, id: string) => {}/**- * Gets citation totals per answer engine for one prompt, plus per-day counts for the last 30 days in the project's timezone.+ * Gets citation and brand-mention totals per answer engine for one prompt, plus, for the last 30 days in the project's timezone, per-day counts, which engines cite which domains, brand sentiment per engine, and the words of the web searches the engines ran.* @summary Get an AEO prompt's citation stats*/export const aeoPromptsStats = async (diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 2b8ad685c128..5f2a2c00d7b3 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -792,7 +792,7 @@"feature_flag": "aeo-citation-tracking"},"aeo-prompts-stats-get": {- "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `failed`, `citation_rate` (cited share of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Use it to compare engines or to see whether a prompt's citations are trending up or down.",+ "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `mentioned` (answers that name the brand), `failed`, `citation_rate` and `mention_rate` (shares of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Over the same 30 days, `consensus` says how many of each engine's answers cited each of the ten most cited domains, `sentiment` counts positive, neutral and negative answers toward the brand per engine (`unscored` answers name the brand but are not scored yet), and `top_words` lists the most used words of the web searches the engines ran. `brand_names` are the words an answer must contain to count as a mention; use them to filter `answer_text` in a HogQL query over `system.aeo_citation_checks`. Use it to compare engines, to see whether a prompt's citations are trending up or down, and to find which sources the engines agree on.","category": "AI citation tracking","feature": "aeo","summary": "Get a tracked AI prompt's citation stats",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex 5b04172e9b1c..12f8025df245 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -792,7 +792,7 @@"feature_flag": "aeo-citation-tracking"},"aeo-prompts-stats-get": {- "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `failed`, `citation_rate` (cited share of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Use it to compare engines or to see whether a prompt's citations are trending up or down.",+ "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `mentioned` (answers that name the brand), `failed`, `citation_rate` and `mention_rate` (shares of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Over the same 30 days, `consensus` says how many of each engine's answers cited each of the ten most cited domains, `sentiment` counts positive, neutral and negative answers toward the brand per engine (`unscored` answers name the brand but are not scored yet), and `top_words` lists the most used words of the web searches the engines ran. `brand_names` are the words an answer must contain to count as a mention; use them to filter `answer_text` in a HogQL query over `system.aeo_citation_checks`. Use it to compare engines, to see whether a prompt's citations are trending up or down, and to find which sources the engines agree on.","category": "AI citation tracking","feature": "aeo","summary": "Get a tracked AI prompt's citation stats",diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex fccaf380c037..d920f39d2b5f 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -146,6 +146,16 @@ export namespace Schemas {* @nullable*/prompt_version: number | null;+ /**+ * Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand.+ * @nullable+ */+ brand_sentiment?: string | null;+ /**+ * Classifier confidence (0-1) in 'brand_sentiment'. Null when 'brand_sentiment' is null.+ * @nullable+ */+ brand_sentiment_score?: number | null;}export interface AEOPromptDailyEngineStats {@@ -157,22 +167,55 @@ export namespace Schemas {checks: number;/** Checks that day whose answer cited a target domain. */cited: number;+ /** Checks that day whose answer names the brand. Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */+ mentioned: number;/** Checks that day where the engine did not answer. */failed: number;}+ export interface AEOPromptEngineDomainShare {+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */+ engine: string;+ /** One of the most cited domains across all engines in the window. */+ domain: string;+ /** This engine's answered checks in the last 30 days that cited the domain. */+ checks_citing: number;+ /** This engine's answered checks in the last 30 days. */+ checks: number;+ }++ export interface AEOPromptEngineSentiment {+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */+ engine: string;+ /** Answers in the last 30 days whose brand sentences read positive. */+ positive: number;+ /** Answers in the last 30 days whose brand sentences read neutral. */+ neutral: number;+ /** Answers in the last 30 days whose brand sentences read negative. */+ negative: number;+ /** Answers in the last 30 days that name the brand but have no sentiment yet. */+ unscored: number;+ }+export interface AEOPromptEngineStats {/*** Share (0-1) of answered checks that cited a target domain. Failed checks are left out. Null when no check was answered.* @nullable*/readonly citation_rate: number | null;+ /**+ * Share (0-1) of answered checks whose answer names the brand. Failed checks are left out. Null when no check was answered.+ * @nullable+ */+ readonly mention_rate: number | null;/** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */engine: string;/** Checks of the prompt on this engine, failed ones included. */checks: number;/** Checks whose answer cited a target domain. */cited: number;+ /** Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */+ mentioned: number;/** Checks where the engine did not answer. */failed: number;/** When the engine first ran the prompt. */@@ -190,13 +233,28 @@ export namespace Schemas {skipped: number;}+ export interface AEOPromptSearchWord {+ /** A word from the web searches, lower-cased, common English words left out. */+ word: string;+ /** How many searches used the word, weighted by how often each search ran. */+ count: number;+ }+export interface AEOPromptStats {/** Totals per answer engine over the prompt's whole history. */engines: AEOPromptEngineStats[];/** Counts per day and engine from 'daily_from' to 'daily_to' (30 days). Days and engines without a check are left out. */daily: AEOPromptDailyEngineStats[];+ /** One row per engine and per domain for the 10 most cited domains of the last 30 days: how many of the engine's answers cited the domain. Shows which sources the engines agree on. */+ consensus: AEOPromptEngineDomainShare[];+ /** Brand sentiment of the answers in the last 30 days, one row per engine. Only the sentences that name the brand are scored. */+ sentiment: AEOPromptEngineSentiment[];+ /** Most used words across the web searches the engines ran in the last 30 days, at most 20. Only engines that report their searches contribute. */+ top_words: AEOPromptSearchWord[];/** Checks of the prompt across all engines, failed ones included. */total_checks: number;+ /** Lower-cased brand names, derived from the project's target domains. An answer that contains one of them as a word counts as a mention. */+ brand_names: string[];/** First day of the daily series, in the project's timezone. */daily_from: string;/** Last day of the daily series (today), in the project's timezone. */diff --git a/services/mcp/src/generated/aeo/api.ts b/services/mcp/src/generated/aeo/api.tsindex c9d1956d2fda..86c9457ff852 100644--- a/services/mcp/src/generated/aeo/api.ts+++ b/services/mcp/src/generated/aeo/api.ts@@ -147,7 +147,7 @@ export const AeoPromptsChecksQueryParams = () => zod.object({})/**- * Gets citation totals per answer engine for one prompt, plus per-day counts for the last 30 days in the project's timezone.+ * Gets citation and brand-mention totals per answer engine for one prompt, plus, for the last 30 days in the project's timezone, per-day counts, which engines cite which domains, brand sentiment per engine, and the words of the web searches the engines ran.* @summary Get an AEO prompt's citation stats*/export const AeoPromptsStatsParams = () => zod.object({