Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aeo): report sources, engines and sentiment on the prompt detail (#122978)

Lucas Ricoy36e8e6f648f0
diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambr
index 3077b9a516f5..0908cc96bd17 100644
--- a/posthog/hogql/database/test/__snapshots__/test_database.ambr
+++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr
@@ -2889,6 +2889,26 @@
"table": null,
"type": "json"
},
+ "brand_sentiment": {
+ "chain": null,
+ "fields": null,
+ "hogql_value": "brand_sentiment",
+ "id": null,
+ "name": "brand_sentiment",
+ "schema_valid": true,
+ "table": null,
+ "type": "string"
+ },
+ "brand_sentiment_score": {
+ "chain": null,
+ "fields": null,
+ "hogql_value": "brand_sentiment_score",
+ "id": null,
+ "name": "brand_sentiment_score",
+ "schema_valid": true,
+ "table": null,
+ "type": "float"
+ },
"cost_usd": {
"chain": null,
"fields": null,
@@ -17537,6 +17557,26 @@
"table": null,
"type": "json"
},
+ "brand_sentiment": {
+ "chain": null,
+ "fields": null,
+ "hogql_value": "brand_sentiment",
+ "id": null,
+ "name": "brand_sentiment",
+ "schema_valid": true,
+ "table": null,
+ "type": "string"
+ },
+ "brand_sentiment_score": {
+ "chain": null,
+ "fields": null,
+ "hogql_value": "brand_sentiment_score",
+ "id": null,
+ "name": "brand_sentiment_score",
+ "schema_valid": true,
+ "table": null,
+ "type": "float"
+ },
"cost_usd": {
"chain": null,
"fields": null,
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 0b5c6eebd33d..5328e5c6cd84 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -156,6 +156,10 @@ from posthog.temporal.weekly_digest import (
WORKFLOWS as WEEKLY_DIGEST_WORKFLOWS,
)
+from products.aeo.backend.facade.temporal import (
+ ACTIVITIES as AEO_BRAND_SENTIMENT_ACTIVITIES,
+ WORKFLOWS as AEO_BRAND_SENTIMENT_WORKFLOWS,
+)
from products.alerts.backend.facade.temporal import (
PLATFORM_AI_QUEUE_ACTIVITIES as INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES,
PLATFORM_EVALUATION_ACTIVITIES as INSIGHT_PLATFORM_EVALUATION_ACTIVITIES,
@@ -580,9 +584,10 @@ _task_queue_specs = [
WEEKLY_DIGEST_ACTIVITIES + WA_DIGEST_ACTIVITIES + DATA_CATALOG_DIGEST_ACTIVITIES,
),
(
+ # AEO brand sentiment rides this queue because its image is the one that carries the ONNX sentiment model.
settings.LLMA_EVALS_TASK_QUEUE,
- LLM_ANALYTICS_EVAL_WORKFLOWS + LLM_ANALYTICS_TAGGER_WORKFLOWS,
- LLM_ANALYTICS_EVAL_ACTIVITIES + LLM_ANALYTICS_TAGGER_ACTIVITIES,
+ LLM_ANALYTICS_EVAL_WORKFLOWS + LLM_ANALYTICS_TAGGER_WORKFLOWS + AEO_BRAND_SENTIMENT_WORKFLOWS,
+ LLM_ANALYTICS_EVAL_ACTIVITIES + LLM_ANALYTICS_TAGGER_ACTIVITIES + AEO_BRAND_SENTIMENT_ACTIVITIES,
),
(
settings.LLMA_TASK_QUEUE,
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index ff605a97f505..fd90723bdcca 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -1467,6 +1467,8 @@ MCP_STORE_SLACK_DEV_ALLOWED_TEAM_IDS = get_list(get_from_env("MCP_STORE_SLACK_DE
# teams in this allowlist AND with the `aeo-citation-tracking` flag enabled.
AEO_CITATION_TEAM_IDS = get_list(get_from_env("AEO_CITATION_TEAM_IDS", ""))
AEO_TARGET_DOMAINS = get_list(get_from_env("AEO_TARGET_DOMAINS", "posthog.com"))
+# Scores brand sentiment on the evals Temporal queue after each run. Off until that worker carries the model.
+AEO_BRAND_SENTIMENT_ENABLED = get_from_env("AEO_BRAND_SENTIMENT_ENABLED", False, type_cast=str_to_bool)
AEO_ANTHROPIC_MODEL = get_from_env("AEO_ANTHROPIC_MODEL", "claude-sonnet-5")
AEO_OPENAI_MODEL = get_from_env("AEO_OPENAI_MODEL", "gpt-5")
EXA_API_KEY = get_from_env("EXA_API_KEY", "")
diff --git a/products/aeo/backend/brand.py b/products/aeo/backend/brand.py
new file mode 100644
index 000000000000..87f339740e39
--- /dev/null
+++ b/products/aeo/backend/brand.py
@@ -0,0 +1,47 @@
+"""How an answer is judged to name the brand.
+
+The brand names are the registrable names of the target domains (`example` for
+`docs.example.co.uk`), so the same list drives the SQL mention filter, the brand
+regex the sentiment scorer uses, and the names the frontend builds its HogQL
+mention filter from.
+"""
+
+from __future__ import annotations
+
+import re
+
+from django.db.models import Q
+
+import tldextract
+
+_DOMAIN_PARTS = tldextract.TLDExtract(suffix_list_urls=())
+
+
+def brand_names(target_domains: list[str]) -> tuple[str, ...]:
+ """Lower-cased registrable names of the target domains, sorted, without duplicates."""
+ return tuple(sorted({(_DOMAIN_PARTS(domain).domain or domain).lower() for domain in target_domains if domain}))
+
+
+def brand_regex(target_domains: list[str]) -> str | None:
+ """Postgres regex (word-bounded, for `iregex`) that matches any brand name. None without a brand."""
+ names = brand_names(target_domains)
+ if not names:
+ return None
+ return r"\m(" + "|".join(re.escape(name) for name in names) + r")\M"
+
+
+def brand_pattern(target_domains: list[str]) -> re.Pattern[str] | None:
+ """Python equivalent of `brand_regex`, for text already loaded from the database."""
+ names = brand_names(target_domains)
+ if not names:
+ return None
+ return re.compile(r"\b(" + "|".join(re.escape(name) for name in names) + r")\b", re.IGNORECASE)
+
+
+def mention_filter(target_domains: list[str]) -> Q:
+ """Checks whose answer names the brand. A cited answer counts as a mention even when the
+ text only carries the URL, so the mention rate is never below the citation rate."""
+ mention = Q(cited=True)
+ if regex := brand_regex(target_domains):
+ mention |= Q(answer_text__iregex=regex)
+ return mention
diff --git a/products/aeo/backend/engines.py b/products/aeo/backend/engines.py
index e8706d4eeadb..db345841f8d2 100644
--- a/products/aeo/backend/engines.py
+++ b/products/aeo/backend/engines.py
@@ -245,13 +245,19 @@ def parse_exa_citations(body: dict[str, Any]) -> ParsedAnswer:
)
+def is_target_host(host: str, target_domains: list[str]) -> bool:
+ """True when the host is one of the target domains or a subdomain of it."""
+ host = host.lower()
+ return any(host == domain or host.endswith(f".{domain}") for domain in (d.lower() for d in target_domains))
+
+
def is_target_url(url: str, target_domains: list[str]) -> bool:
"""True when the URL's host is one of the target domains or a subdomain of it."""
try:
- host = (urlparse(url).hostname or "").lower()
+ host = urlparse(url).hostname or ""
except ValueError:
return False
- return any(host == domain or host.endswith(f".{domain}") for domain in (d.lower() for d in target_domains))
+ return is_target_host(host, target_domains)
def target_position(cited_urls: list[str], target_domains: list[str]) -> int | None:
diff --git a/products/aeo/backend/facade/api.py b/products/aeo/backend/facade/api.py
index af9ffa63ff4f..3b759994f85d 100644
--- a/products/aeo/backend/facade/api.py
+++ b/products/aeo/backend/facade/api.py
@@ -6,22 +6,25 @@ capabilities. Celery wiring is re-exported from facade/tasks.py.
from __future__ import annotations
+import re
import uuid
import datetime as dt
-from collections import defaultdict
+from collections import Counter, defaultdict
from typing import Any
from zoneinfo import ZoneInfo
+from django.conf import settings
from django.core.exceptions import ValidationError
from django.db import IntegrityError, transaction
-from django.db.models import Count, Max, Min, Q
+from django.db.models import Count, Max, Min, Q, QuerySet
from django.db.models.functions import TruncDate
from django.utils import timezone
from posthog.models.team import Team
from products.aeo.backend import gaps
-from products.aeo.backend.engines import MAX_PROMPT_LENGTH
+from products.aeo.backend.brand import brand_names, mention_filter
+from products.aeo.backend.engines import MAX_PROMPT_LENGTH, top_domains
from products.aeo.backend.facade.contracts import (
PROMPT_TOPIC_MAX_LENGTH,
CitationGap,
@@ -31,10 +34,13 @@ from products.aeo.backend.facade.contracts import (
PromptCheck,
PromptDailyEngineStats,
PromptEngineCheck,
+ PromptEngineDomainShare,
+ PromptEngineSentiment,
PromptEngineStats,
PromptImport,
PromptImportResult,
PromptNotFoundError,
+ PromptSearchWord,
PromptStats,
PromptTextConflictError,
PromptTopic,
@@ -182,12 +188,17 @@ def list_prompt_checks(
def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days: int) -> PromptStats:
- """Totals per engine over the prompt's whole history, plus per-day counts for the last `days` days
+ """Totals per engine over the prompt's whole history, plus per-day counts, engine consensus on
+ the cited domains, brand sentiment and the words of the web searches over the last `days` days
in the team's timezone."""
prompt = _get_prompt_row(team_id, prompt_id)
+ target_domains: list[str] = settings.AEO_TARGET_DOMAINS
checks = AEOCitationCheck.objects.for_team(team_id).filter(prompt_id=prompt.id)
+ # `mentioned` filters on the `cited` column, and an annotation named `cited` would shadow that
+ # column for every later annotation in the same call, so it must come first.
counts = {
"checks": Count("id"),
+ "mentioned": Count("id", filter=Q(check_failed=False) & mention_filter(target_domains)),
"cited": Count("id", filter=Q(cited=True)),
"failed": Count("id", filter=Q(check_failed=True)),
}
@@ -197,6 +208,7 @@ def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days:
engine=row["engine"],
checks=row["checks"],
cited=row["cited"],
+ mentioned=row["mentioned"],
failed=row["failed"],
first_checked_at=row["first_checked_at"],
last_checked_at=row["last_checked_at"],
@@ -210,26 +222,117 @@ def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days:
daily_to = timezone.now().astimezone(tz).date()
daily_from = daily_to - dt.timedelta(days=days - 1)
window_start = dt.datetime.combine(daily_from, dt.time.min, tzinfo=tz)
+ in_window = checks.filter(created_at__gte=window_start)
daily = tuple(
PromptDailyEngineStats(
- date=row["date"], engine=row["engine"], checks=row["checks"], cited=row["cited"], failed=row["failed"]
+ date=row["date"],
+ engine=row["engine"],
+ checks=row["checks"],
+ cited=row["cited"],
+ mentioned=row["mentioned"],
+ failed=row["failed"],
)
- for row in checks.filter(created_at__gte=window_start)
- .annotate(date=TruncDate("created_at", tzinfo=tz))
+ for row in in_window.annotate(date=TruncDate("created_at", tzinfo=tz))
.values("date", "engine")
.annotate(**counts)
.order_by("date", "engine")
)
+ answered = in_window.filter(check_failed=False)
+ sentiment = tuple(
+ PromptEngineSentiment(
+ engine=row["engine"],
+ positive=row["positive"],
+ neutral=row["neutral"],
+ negative=row["negative"],
+ unscored=row["unscored"],
+ )
+ for row in answered.values("engine")
+ .annotate(
+ positive=Count("id", filter=Q(brand_sentiment="positive")),
+ neutral=Count("id", filter=Q(brand_sentiment="neutral")),
+ negative=Count("id", filter=Q(brand_sentiment="negative")),
+ unscored=Count("id", filter=Q(brand_sentiment__isnull=True) & mention_filter(target_domains)),
+ )
+ .order_by("engine")
+ )
+ report = _PromptWindowReport.from_checks(answered)
return PromptStats(
total_checks=sum(engine.checks for engine in engines),
+ brand_names=brand_names(target_domains),
engines=engines,
daily_from=daily_from,
daily_to=daily_to,
daily=daily,
+ consensus=report.consensus(),
+ sentiment=sentiment,
+ top_words=report.top_words(),
)
+MAX_CONSENSUS_DOMAINS = 10
+MAX_STATS_WORDS = 20
+MIN_SEARCH_WORD_LENGTH = 3
+_WORD = re.compile(r"[^\W_]+", re.UNICODE)
+SEARCH_STOP_WORDS = frozenset(
+ """a an and are as at be best by can do does for from how i in is it its my of on or that the this to top vs
+ what which who with without your you""".split()
+)
+
+
+class _PromptWindowReport:
+ """Cited domains and web searches across one prompt's answered checks in the window.
+
+ Counts are per check: a domain cited twice in one answer counts once, so every number reads as
+ "how many answers", the same unit as the citation rate."""
+
+ def __init__(self) -> None:
+ self._domain_checks: Counter[str] = Counter()
+ self._engine_checks: Counter[str] = Counter()
+ self._engine_domain_checks: Counter[tuple[str, str]] = Counter()
+ self._queries: Counter[str] = Counter()
+
+ @classmethod
+ def from_checks(cls, answered: QuerySet[AEOCitationCheck]) -> _PromptWindowReport:
+ report = cls()
+ for row in answered.values("engine", "cited_urls", "search_queries"):
+ report.add(row["engine"], _strings(row["cited_urls"]), _strings(row["search_queries"]))
+ return report
+
+ def add(self, engine: str, cited_urls: tuple[str, ...], search_queries: tuple[str, ...]) -> None:
+ self._engine_checks[engine] += 1
+ domains = top_domains(list(cited_urls))
+ self._domain_checks.update(domains)
+ self._engine_domain_checks.update((engine, domain) for domain in domains)
+ self._queries.update(key for key in (" ".join(query.casefold().split()) for query in search_queries) if key)
+
+ def consensus(self) -> tuple[PromptEngineDomainShare, ...]:
+ domains = [domain for domain, _ in self._ranked(self._domain_checks, MAX_CONSENSUS_DOMAINS)]
+ return tuple(
+ PromptEngineDomainShare(
+ engine=engine,
+ domain=domain,
+ checks_citing=self._engine_domain_checks[(engine, domain)],
+ checks=checks,
+ )
+ for engine, checks in sorted(self._engine_checks.items())
+ for domain in domains
+ )
+
+ def top_words(self) -> tuple[PromptSearchWord, ...]:
+ words: Counter[str] = Counter()
+ for key, count in self._queries.items():
+ for word in set(_WORD.findall(key)):
+ if len(word) >= MIN_SEARCH_WORD_LENGTH and word not in SEARCH_STOP_WORDS:
+ words[word] += count
+ return tuple(PromptSearchWord(word=word, count=count) for word, count in self._ranked(words, MAX_STATS_WORDS))
+
+ @staticmethod
+ def _ranked(counter: Counter[str], limit: int) -> list[tuple[str, int]]:
+ # Ties break on the value, so the order is stable between two requests.
+ return sorted(counter.items(), key=lambda item: (-item[1], item[0]))[:limit]
+
+
def import_prompts(team_id: int, prompts: list[PromptImport]) -> PromptImportResult:
"""Add prompts to the team's set, or update the topic and evidence of
prompts that already exist. Existing prompts keep their active state."""
@@ -373,6 +476,8 @@ _CHECK_FIELDS = (
"search_queries",
"answer_text",
"prompt_version",
+ "brand_sentiment",
+ "brand_sentiment_score",
)
@@ -397,6 +502,8 @@ def _to_prompt_check(row: dict[str, Any]) -> PromptCheck:
search_queries=_strings(row["search_queries"]),
answer_text=row["answer_text"],
prompt_version=row["prompt_version"],
+ brand_sentiment=row["brand_sentiment"],
+ brand_sentiment_score=row["brand_sentiment_score"],
)
diff --git a/products/aeo/backend/facade/contracts.py b/products/aeo/backend/facade/contracts.py
index 21dff934d5de..7bf4131837ef 100644
--- a/products/aeo/backend/facade/contracts.py
+++ b/products/aeo/backend/facade/contracts.py
@@ -114,6 +114,9 @@ class PromptCheck:
search_queries: tuple[str, ...]
answer_text: str | None
prompt_version: int | None
+ # Sentiment of the sentences that name the brand. None until scored, and when the brand is not named.
+ brand_sentiment: str | None = None
+ brand_sentiment_score: float | None = None
@frozen
@@ -121,6 +124,8 @@ class PromptEngineStats:
engine: str
checks: int
cited: int
+ # Answers that name the brand, cited ones included: the same definition the citation gaps use.
+ mentioned: int
failed: int
first_checked_at: dt.datetime
last_checked_at: dt.datetime
@@ -130,6 +135,11 @@ class PromptEngineStats:
answered = self.checks - self.failed
return self.cited / answered if answered else None
+ @property
+ def mention_rate(self) -> float | None:
+ answered = self.checks - self.failed
+ return self.mentioned / answered if answered else None
+
@frozen
class PromptDailyEngineStats:
@@ -137,16 +147,50 @@ class PromptDailyEngineStats:
engine: str
checks: int
cited: int
+ mentioned: int
failed: int
+@frozen
+class PromptEngineDomainShare:
+ """How many of one engine's answered checks in the window cited a domain."""
+
+ engine: str
+ domain: str
+ checks_citing: int
+ checks: int
+
+
+@frozen
+class PromptEngineSentiment:
+ """Brand sentiment of one engine's answered checks in the window. `unscored` answers name the
+ brand but have no sentiment yet."""
+
+ engine: str
+ positive: int
+ neutral: int
+ negative: int
+ unscored: int
+
+
+@frozen
+class PromptSearchWord:
+ word: str
+ count: int
+
+
@frozen
class PromptStats:
total_checks: int
+ # The names an answer must contain to count as a mention, so a client can filter answers the same way.
+ brand_names: tuple[str, ...]
engines: tuple[PromptEngineStats, ...]
daily_from: dt.date
daily_to: dt.date
daily: tuple[PromptDailyEngineStats, ...]
+ consensus: tuple[PromptEngineDomainShare, ...]
+ sentiment: tuple[PromptEngineSentiment, ...]
+ top_words: tuple[PromptSearchWord, ...]
@frozen
diff --git a/products/aeo/backend/facade/hogql.py b/products/aeo/backend/facade/hogql.py
index d5e1ad64fa75..6d2c1a705eb1 100644
--- a/products/aeo/backend/facade/hogql.py
+++ b/products/aeo/backend/facade/hogql.py
@@ -88,6 +88,16 @@ aeo_citation_checks: PostgresTable = PostgresTable(
"top_cited_domains": StringJSONDatabaseField(
name="top_cited_domains", description="JSON array of distinct hosts across the cited URLs."
),
+ "brand_sentiment": StringDatabaseField(
+ name="brand_sentiment",
+ nullable=True,
+ description="Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand.",
+ ),
+ "brand_sentiment_score": FloatDatabaseField(
+ name="brand_sentiment_score",
+ nullable=True,
+ description="Classifier confidence (0-1) in brand_sentiment.",
+ ),
"cost_usd": FloatDatabaseField(
name="cost_usd", nullable=True, description="Engine-reported cost, where the engine reports one."
),
diff --git a/products/aeo/backend/facade/temporal.py b/products/aeo/backend/facade/temporal.py
new file mode 100644
index 000000000000..efff02307d98
--- /dev/null
+++ b/products/aeo/backend/facade/temporal.py
@@ -0,0 +1,6 @@
+"""Temporal workflows and activities this product registers on the evals queue, which is the
+only worker image that carries the sentiment model."""
+
+from products.aeo.backend.temporal.brand_sentiment import ACTIVITIES, WORKFLOWS
+
+__all__ = ["ACTIVITIES", "WORKFLOWS"]
diff --git a/products/aeo/backend/gaps.py b/products/aeo/backend/gaps.py
index 5487b6260d57..0c51e5c12b35 100644
--- a/products/aeo/backend/gaps.py
+++ b/products/aeo/backend/gaps.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import re
import datetime as dt
from collections import Counter, defaultdict
from typing import Any
@@ -8,8 +7,7 @@ from typing import Any
from django.conf import settings
from django.db.models import BooleanField, Case, F, Q, Value, When
-import tldextract
-
+from products.aeo.backend.brand import mention_filter
from products.aeo.backend.engines import is_target_url, top_domains
from products.aeo.backend.facade.contracts import CitationGap, EngineAnswer
from products.aeo.backend.models import AEOCitationCheck, AEOPromptVersion
@@ -17,20 +15,12 @@ from products.aeo.backend.seeding import prompt_hash
MAX_GAP_URLS = 10
MAX_GAP_QUERIES = 10
-_DOMAIN_PARTS = tldextract.TLDExtract(suffix_list_urls=())
def _ranked(counter: Counter[str], limit: int) -> tuple[str, ...]:
return tuple(value for value, _ in counter.most_common(limit))
-def _brand_regex(target_domains: list[str]) -> str | None:
- names = {(_DOMAIN_PARTS(domain).domain or domain).lower() for domain in target_domains if domain}
- if not names:
- return None
- return r"\m(" + "|".join(re.escape(name) for name in sorted(names)) + r")\M"
-
-
def _gap_keys(first_hashes: dict[str, str], current_hashes: dict[str, str]) -> dict[str, str]:
"""Key each prompt's gap by its first version's hash, so a text edit keeps the key.
@@ -60,12 +50,13 @@ def list_citation_gaps(team_id: int, *, since: dt.datetime) -> list[CitationGap]
.filter(created_at__gte=since, check_failed=False, prompt__active=True)
.filter(current_version)
)
- mention = Q(cited=True)
- if brand := _brand_regex(target_domains):
- mention |= Q(answer_text__iregex=brand)
rows = (
checks_in_window.annotate(
- mentioned=Case(When(mention, then=Value(True)), default=Value(False), output_field=BooleanField())
+ mentioned=Case(
+ When(mention_filter(target_domains), then=Value(True)),
+ default=Value(False),
+ output_field=BooleanField(),
+ )
)
.order_by("-created_at")
.values(
diff --git a/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py b/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py
new file mode 100644
index 000000000000..3df58b8408f8
--- /dev/null
+++ b/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py
@@ -0,0 +1,31 @@
+# Generated by Django 5.2.18 on 2026-10-23 23:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("aeo", "0006_backfill_prompt_versions"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="aeocitationcheck",
+ name="brand_sentiment",
+ field=models.CharField(
+ blank=True,
+ help_text="Sentiment of the sentences that name the brand: positive, neutral or negative. Null until scored, and when the answer never names the brand.",
+ max_length=16,
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="aeocitationcheck",
+ name="brand_sentiment_score",
+ field=models.FloatField(
+ blank=True,
+ help_text="Classifier confidence (0-1) in `brand_sentiment`, averaged over the brand sentences.",
+ null=True,
+ ),
+ ),
+ ]
diff --git a/products/aeo/backend/migrations/max_migration.txt b/products/aeo/backend/migrations/max_migration.txt
index a1727f32d961..637c4c4a811e 100644
--- a/products/aeo/backend/migrations/max_migration.txt
+++ b/products/aeo/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0006_backfill_prompt_versions
+0007_citation_check_brand_sentiment
diff --git a/products/aeo/backend/models.py b/products/aeo/backend/models.py
index 2e1c729af35d..ead33a5ce13e 100644
--- a/products/aeo/backend/models.py
+++ b/products/aeo/backend/models.py
@@ -188,6 +188,18 @@ class AEOCitationCheck(TeamScopedRootMixin, CreatedMetaFields, UUIDTModel):
target_urls = models.JSONField(default=list, blank=True, help_text="Cited URLs on a target domain.")
top_cited_domains = models.JSONField(default=list, blank=True, help_text="Distinct hosts across the cited URLs.")
+ brand_sentiment = models.CharField(
+ max_length=16,
+ null=True,
+ blank=True,
+ help_text="Sentiment of the sentences that name the brand: positive, neutral or negative. Null until scored, and when the answer never names the brand.",
+ )
+ brand_sentiment_score = models.FloatField(
+ null=True,
+ blank=True,
+ help_text="Classifier confidence (0-1) in `brand_sentiment`, averaged over the brand sentences.",
+ )
+
cost_usd = models.FloatField(null=True, blank=True, help_text="Engine-reported cost, where the engine reports one.")
gateway_trace_id = models.CharField(
max_length=64,
diff --git a/products/aeo/backend/presentation/views.py b/products/aeo/backend/presentation/views.py
index b11ecf0e2371..5b88e8956031 100644
--- a/products/aeo/backend/presentation/views.py
+++ b/products/aeo/backend/presentation/views.py
@@ -16,6 +16,8 @@ from posthog.exceptions import Conflict
from posthog.permissions import posthog_feature_flag_enabled
from products.aeo.backend.facade.api import (
+ MAX_CONSENSUS_DOMAINS,
+ MAX_STATS_WORDS,
archive_prompt,
get_prompt,
get_prompt_stats,
@@ -35,10 +37,13 @@ from products.aeo.backend.facade.contracts import (
PromptCheck,
PromptDailyEngineStats,
PromptEngineCheck,
+ PromptEngineDomainShare,
+ PromptEngineSentiment,
PromptEngineStats,
PromptImport,
PromptImportResult,
PromptNotFoundError,
+ PromptSearchWord,
PromptStats,
PromptTextConflictError,
PromptTopic,
@@ -171,6 +176,12 @@ class AEOPromptCheckSerializer(DataclassSerializer):
"prompt_version": {
"help_text": "Version of the prompt that ran. Null for checks recorded before prompts had versions."
},
+ "brand_sentiment": {
+ "help_text": "Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand."
+ },
+ "brand_sentiment_score": {
+ "help_text": "Classifier confidence (0-1) in 'brand_sentiment'. Null when 'brand_sentiment' is null."
+ },
}
@@ -187,12 +198,23 @@ class AEOPromptChecksQuerySerializer(serializers.Serializer):
)
+MENTIONED_HELP = (
+ "Checks whose answer names the brand (the registrable name of a target domain, case ignored). "
+ "A cited check always counts as mentioned."
+)
+
+
class AEOPromptEngineStatsSerializer(DataclassSerializer):
citation_rate = serializers.FloatField(
read_only=True,
allow_null=True,
help_text="Share (0-1) of answered checks that cited a target domain. Failed checks are left out. Null when no check was answered.",
)
+ mention_rate = serializers.FloatField(
+ read_only=True,
+ allow_null=True,
+ help_text="Share (0-1) of answered checks whose answer names the brand. Failed checks are left out. Null when no check was answered.",
+ )
class Meta:
dataclass = PromptEngineStats
@@ -201,6 +223,7 @@ class AEOPromptEngineStatsSerializer(DataclassSerializer):
"engine": {"help_text": ENGINE_HELP},
"checks": {"help_text": "Checks of the prompt on this engine, failed ones included."},
"cited": {"help_text": "Checks whose answer cited a target domain."},
+ "mentioned": {"help_text": MENTIONED_HELP},
"failed": {"help_text": "Checks where the engine did not answer."},
"first_checked_at": {"help_text": "When the engine first ran the prompt."},
"last_checked_at": {"help_text": "When the engine last ran the prompt."},
@@ -216,10 +239,50 @@ class AEOPromptDailyEngineStatsSerializer(DataclassSerializer):
"engine": {"help_text": ENGINE_HELP},
"checks": {"help_text": "Checks on this engine that day, failed ones included."},
"cited": {"help_text": "Checks that day whose answer cited a target domain."},
+ "mentioned": {"help_text": "Checks that day whose answer names the brand. " + MENTIONED_HELP},
"failed": {"help_text": "Checks that day where the engine did not answer."},
}
+class AEOPromptEngineDomainShareSerializer(DataclassSerializer):
+ class Meta:
+ dataclass = PromptEngineDomainShare
+ ref_name = "AEOPromptEngineDomainShare"
+ extra_kwargs = {
+ "engine": {"help_text": ENGINE_HELP},
+ "domain": {"help_text": "One of the most cited domains across all engines in the window."},
+ "checks_citing": {
+ "help_text": f"This engine's answered checks in the last {STATS_DAYS} days that cited the domain."
+ },
+ "checks": {"help_text": f"This engine's answered checks in the last {STATS_DAYS} days."},
+ }
+
+
+class AEOPromptEngineSentimentSerializer(DataclassSerializer):
+ class Meta:
+ dataclass = PromptEngineSentiment
+ ref_name = "AEOPromptEngineSentiment"
+ extra_kwargs = {
+ "engine": {"help_text": ENGINE_HELP},
+ "positive": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read positive."},
+ "neutral": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read neutral."},
+ "negative": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read negative."},
+ "unscored": {
+ "help_text": f"Answers in the last {STATS_DAYS} days that name the brand but have no sentiment yet."
+ },
+ }
+
+
+class AEOPromptSearchWordSerializer(DataclassSerializer):
+ class Meta:
+ dataclass = PromptSearchWord
+ ref_name = "AEOPromptSearchWord"
+ extra_kwargs = {
+ "word": {"help_text": "A word from the web searches, lower-cased, common English words left out."},
+ "count": {"help_text": "How many searches used the word, weighted by how often each search ran."},
+ }
+
+
class AEOPromptStatsSerializer(DataclassSerializer):
engines = AEOPromptEngineStatsSerializer(
many=True, help_text="Totals per answer engine over the prompt's whole history."
@@ -228,12 +291,27 @@ class AEOPromptStatsSerializer(DataclassSerializer):
many=True,
help_text=f"Counts per day and engine from 'daily_from' to 'daily_to' ({STATS_DAYS} days). Days and engines without a check are left out.",
)
+ consensus = AEOPromptEngineDomainShareSerializer(
+ many=True,
+ help_text=f"One row per engine and per domain for the {MAX_CONSENSUS_DOMAINS} most cited domains of the last {STATS_DAYS} days: how many of the engine's answers cited the domain. Shows which sources the engines agree on.",
+ )
+ sentiment = AEOPromptEngineSentimentSerializer(
+ many=True,
+ help_text=f"Brand sentiment of the answers in the last {STATS_DAYS} days, one row per engine. Only the sentences that name the brand are scored.",
+ )
+ top_words = AEOPromptSearchWordSerializer(
+ many=True,
+ help_text=f"Most used words across the web searches the engines ran in the last {STATS_DAYS} days, at most {MAX_STATS_WORDS}. Only engines that report their searches contribute.",
+ )
class Meta:
dataclass = PromptStats
ref_name = "AEOPromptStats"
extra_kwargs = {
"total_checks": {"help_text": "Checks of the prompt across all engines, failed ones included."},
+ "brand_names": {
+ "help_text": "Lower-cased brand names, derived from the project's target domains. An answer that contains one of them as a word counts as a mention."
+ },
"daily_from": {"help_text": "First day of the daily series, in the project's timezone."},
"daily_to": {"help_text": "Last day of the daily series (today), in the project's timezone."},
}
@@ -512,7 +590,7 @@ class AEOPromptViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet):
@validated_request(
operation_id="aeo_prompts_stats",
summary="Get an AEO prompt's citation stats",
- description=f"Gets citation totals per answer engine for one prompt, plus per-day counts for the last {STATS_DAYS} days in the project's timezone.",
+ description=f"Gets citation and brand-mention totals per answer engine for one prompt, plus, for the last {STATS_DAYS} days in the project's timezone, per-day counts, which engines cite which domains, brand sentiment per engine, and the words of the web searches the engines ran.",
responses={200: OpenApiResponse(response=AEOPromptStatsSerializer)},
tags=["web_analytics"],
)
diff --git a/products/aeo/backend/runner.py b/products/aeo/backend/runner.py
index 414116d6fc89..f9cfeb74e416 100644
--- a/products/aeo/backend/runner.py
+++ b/products/aeo/backend/runner.py
@@ -30,6 +30,7 @@ from posthog.models.team import Team
from products.aeo.backend.engines import CitationEngine, available_engines, build_check_fields
from products.aeo.backend.facade.contracts import CitationRunSummary
from products.aeo.backend.models import AEOCitationCheck, AEOPrompt
+from products.aeo.backend.temporal.brand_sentiment import start_brand_sentiment_workflow
logger = structlog.get_logger(__name__)
@@ -144,6 +145,8 @@ def run_citation_checks(
rows_written=summary.rows_written,
write_failures=summary.write_failures,
)
+ if record and rows_written and settings.AEO_BRAND_SENTIMENT_ENABLED:
+ start_brand_sentiment_workflow(canonical_team_id, run_id)
return summary, checks
diff --git a/products/aeo/backend/sentiment.py b/products/aeo/backend/sentiment.py
new file mode 100644
index 000000000000..641b25825b39
--- /dev/null
+++ b/products/aeo/backend/sentiment.py
@@ -0,0 +1,56 @@
+"""How an answer engine's reply speaks about the brand.
+
+Only the sentences that name the brand are scored, so praise of a competitor in
+the same answer does not count as praise of the brand. The classifier itself is
+the AI observability ONNX model; it lives on the evals Temporal queue, and this
+module only prepares its input and folds its output back into one label per answer.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, Sequence
+
+from posthog.dataclasses import frozen
+from posthog.hogql_queries.ai.sentiment_labeling import select_sentiment_label
+
+from products.aeo.backend.brand import brand_pattern
+
+SENTIMENT_LABELS = ("negative", "neutral", "positive")
+# Bounds the classifier input per answer: an answer that names the brand in more sentences than
+# this is scored on the first ones, and the model truncates a sentence longer than this anyway.
+MAX_BRAND_SENTENCES = 20
+MAX_SENTENCE_CHARS = 1000
+_SENTENCE_BREAK = re.compile(r"(?<=[.!?])\s+|\n+")
+
+
+@frozen
+class AnswerSentiment:
+ label: str
+ score: float
+
+
+def brand_sentences(answer_text: str | None, target_domains: list[str]) -> list[str]:
+ """The sentences of an answer that name the brand, in order, trimmed and bounded."""
+ pattern = brand_pattern(target_domains)
+ if not answer_text or pattern is None:
+ return []
+ sentences = (sentence.strip() for sentence in _SENTENCE_BREAK.split(answer_text))
+ matching = (sentence[:MAX_SENTENCE_CHARS] for sentence in sentences if sentence and pattern.search(sentence))
+ return [sentence for _, sentence in zip(range(MAX_BRAND_SENTENCES), matching)]
+
+
+def aggregate_sentiment(sentence_scores: Sequence[Mapping[str, float]]) -> AnswerSentiment | None:
+ """One label for the answer from its brand sentences' class scores.
+
+ The class scores are averaged over the sentences before the label is picked, so one strongly
+ negative sentence among neutral ones pulls the answer toward negative without a majority vote
+ flipping it, and the same neutral margin as AI observability keeps near-ties neutral."""
+ if not sentence_scores:
+ return None
+ mean_scores = {
+ label: round(sum(scores.get(label, 0.0) for scores in sentence_scores) / len(sentence_scores), 4)
+ for label in SENTIMENT_LABELS
+ }
+ label = select_sentiment_label(mean_scores)
+ return AnswerSentiment(label=label, score=mean_scores[label])
diff --git a/products/aeo/backend/temporal/__init__.py b/products/aeo/backend/temporal/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/aeo/backend/temporal/brand_sentiment.py b/products/aeo/backend/temporal/brand_sentiment.py
new file mode 100644
index 000000000000..43528c65a239
--- /dev/null
+++ b/products/aeo/backend/temporal/brand_sentiment.py
@@ -0,0 +1,175 @@
+"""Scores brand sentiment for one citation run.
+
+The ONNX sentiment model and its dependencies are only in the image that serves
+the evals Temporal queue, so the runner cannot classify inline: it starts this
+workflow, and the activity reads the run's answers back from Postgres, scores the
+sentences that name the brand, and writes the label and score to the check rows.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime as dt
+import dataclasses
+from collections.abc import Sequence
+from typing import Any
+
+from django.conf import settings
+from django.utils import timezone
+
+import structlog
+from temporalio import activity, workflow
+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy
+from temporalio.exceptions import WorkflowAlreadyStartedError
+
+from posthog.exceptions_capture import capture_exception
+from posthog.sync import database_sync_to_async
+from posthog.temporal.common.base import PostHogWorkflow
+from posthog.temporal.common.client import sync_connect
+from posthog.temporal.common.utils import close_db_connections
+
+from products.aeo.backend.brand import brand_regex
+from products.aeo.backend.models import AEOCitationCheck
+from products.aeo.backend.sentiment import aggregate_sentiment, brand_sentences
+
+logger = structlog.get_logger(__name__)
+
+WORKFLOW_NAME = "aeo-brand-sentiment"
+# One run is at most ~150 answers x 20 sentences on a CPU model, well inside this.
+CLASSIFY_TIMEOUT = dt.timedelta(minutes=15)
+CLASSIFY_BATCH = 32
+# Answers scored before the setting was turned on, or missed by a failed run, are picked up from
+# this far back, which matches the window the prompt report shows.
+BACKLOG_WINDOW = dt.timedelta(days=30)
+MAX_ROWS_PER_RUN = 2000
+
+
[email protected](frozen=True)
+class AEOBrandSentimentInputs:
+ team_id: int
+ run_id: str
+
+
[email protected](frozen=True)
+class _PendingAnswer:
+ check_id: str
+ sentences: list[str]
+
+
+def _load_pending(inputs: AEOBrandSentimentInputs) -> list[_PendingAnswer]:
+ """The run's unscored answers, then the team's other unscored answers from the backlog window,
+ oldest first, capped so one activity never scans an unbounded history.
+
+ Only answers that name the brand count toward the cap: an answer without a mention stays NULL
+ forever, so letting it fill the slice would starve newer answers that can be scored.
+
+ Two concurrent runs can both pick up the same backlog row. The writes are idempotent (the same
+ text scores the same), so that only costs a duplicate classification, and no transaction is
+ held across the model call that a row lock could ride on."""
+ target_domains: list[str] = settings.AEO_TARGET_DOMAINS
+ regex = brand_regex(target_domains)
+ if regex is None:
+ return []
+ unscored = AEOCitationCheck.objects.for_team(inputs.team_id).filter(
+ check_failed=False, brand_sentiment__isnull=True, answer_text__iregex=regex
+ )
+ own_rows = list(unscored.filter(run_id=inputs.run_id).order_by("created_at").values_list("id", "answer_text"))
+ backlog_rows = list(
+ unscored.exclude(run_id=inputs.run_id)
+ .filter(created_at__gte=timezone.now() - BACKLOG_WINDOW)
+ .order_by("created_at")
+ .values_list("id", "answer_text")[: max(0, MAX_ROWS_PER_RUN - len(own_rows))]
+ )
+ pending = [
+ _PendingAnswer(check_id=str(check_id), sentences=brand_sentences(text, target_domains))
+ for check_id, text in [*own_rows, *backlog_rows]
+ ]
+ return [answer for answer in pending if answer.sentences]
+
+
+def _write_scores(team_id: int, scored: Sequence[tuple[str, str, float]]) -> None:
+ rows = [
+ AEOCitationCheck(id=check_id, brand_sentiment=label, brand_sentiment_score=score)
+ for check_id, label, score in scored
+ ]
+ AEOCitationCheck.objects.for_team(team_id).bulk_update(rows, ["brand_sentiment", "brand_sentiment_score"])
+
+
+@close_db_connections
+async def classify_aeo_brand_sentiment(inputs: AEOBrandSentimentInputs) -> dict[str, Any]:
+ """Score every answer of the run that names the brand and has no sentiment yet.
+
+ A missing model is a deployment problem, not a data problem, so the activity reports it and
+ leaves the rows unscored instead of retrying: the next run picks nothing up either, and the
+ rows read as "not scored yet" until the worker image carries the model."""
+ pending = await database_sync_to_async(_load_pending, thread_sensitive=False)(inputs)
+ if not pending:
+ return {"scored": 0, "answers": 0}
+
+ texts = [sentence for answer in pending for sentence in answer.sentences]
+ try:
+ import posthog.temporal.ai_observability.sentiment.model as sentiment_model # noqa: PLC0415 -- ONNX deps
+
+ results = []
+ for start in range(0, len(texts), CLASSIFY_BATCH):
+ results.extend(await asyncio.to_thread(sentiment_model.classify, texts[start : start + CLASSIFY_BATCH]))
+ except (ImportError, FileNotFoundError) as error:
+ logger.warning(
+ "aeo_brand_sentiment_model_unavailable", team_id=inputs.team_id, run_id=inputs.run_id, error=str(error)
+ )
+ return {"scored": 0, "answers": len(pending), "error": "model_unavailable"}
+
+ scored: list[tuple[str, str, float]] = []
+ position = 0
+ for answer in pending:
+ sentence_scores = [result.scores for result in results[position : position + len(answer.sentences)]]
+ position += len(answer.sentences)
+ sentiment = aggregate_sentiment(sentence_scores)
+ if sentiment is not None:
+ scored.append((answer.check_id, sentiment.label, sentiment.score))
+ await database_sync_to_async(_write_scores, thread_sensitive=False)(inputs.team_id, scored)
+ logger.info("aeo_brand_sentiment_scored", team_id=inputs.team_id, run_id=inputs.run_id, scored=len(scored))
+ return {"scored": len(scored), "answers": len(pending)}
+
+
[email protected](name=WORKFLOW_NAME)
+class AEOBrandSentimentWorkflow(PostHogWorkflow):
+ inputs_cls = AEOBrandSentimentInputs
+
+ @workflow.run
+ async def run(self, inputs: AEOBrandSentimentInputs) -> dict[str, Any]:
+ return await workflow.execute_activity(
+ classify_aeo_brand_sentiment,
+ inputs,
+ start_to_close_timeout=CLASSIFY_TIMEOUT,
+ retry_policy=RetryPolicy(maximum_attempts=3, initial_interval=dt.timedelta(seconds=30)),
+ )
+
+
+def start_brand_sentiment_workflow(team_id: int, run_id: str) -> None:
+ """Fire-and-forget: the citation run is already recorded, so a Temporal outage only means
+ this run's answers stay unscored."""
+ inputs = AEOBrandSentimentInputs(team_id=team_id, run_id=run_id)
+ try:
+ client = sync_connect()
+ asyncio.run(
+ client.start_workflow(
+ WORKFLOW_NAME,
+ inputs,
+ id=f"{WORKFLOW_NAME}-{team_id}-{run_id}",
+ task_queue=settings.LLMA_EVALS_TASK_QUEUE,
+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
+ )
+ )
+ except WorkflowAlreadyStartedError:
+ logger.info("aeo_brand_sentiment_already_started", team_id=team_id, run_id=run_id)
+ except Exception as error:
+ logger.exception("aeo_brand_sentiment_start_failed", team_id=team_id, run_id=run_id, error=str(error)[:300])
+ capture_exception(error)
+ else:
+ logger.info("aeo_brand_sentiment_started", team_id=team_id, run_id=run_id)
+
+
+WORKFLOWS = [AEOBrandSentimentWorkflow]
+ACTIVITIES = [classify_aeo_brand_sentiment]
diff --git a/products/aeo/backend/test/test_api.py b/products/aeo/backend/test/test_api.py
index 16c56c35ddca..9d4e57581b02 100644
--- a/products/aeo/backend/test/test_api.py
+++ b/products/aeo/backend/test/test_api.py
@@ -6,6 +6,7 @@ from posthog.test.base import APIBaseTest
from unittest.mock import patch
from django.db import connection
+from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from django.utils import timezone
@@ -41,6 +42,8 @@ class TestAEOPromptAPI(APIBaseTest):
cited: bool = False,
check_failed: bool = False,
answer_text: str | None = None,
+ cited_urls: list[str] | None = None,
+ search_queries: list[str] | None = None,
) -> str:
with team_scope(self.team.pk):
prompt = AEOPrompt.objects.get(id=prompt_id)
@@ -57,6 +60,8 @@ class TestAEOPromptAPI(APIBaseTest):
cited=cited,
check_failed=check_failed,
answer_text=answer_text,
+ cited_urls=cited_urls or [],
+ search_queries=search_queries or [],
)
AEOCitationCheck.objects.filter(id=check.id).update(
created_at=timezone.now() - dt.timedelta(minutes=minutes_ago)
@@ -424,26 +429,146 @@ class TestAEOPromptAPI(APIBaseTest):
prompt_id = self._import_one_prompt()
day = 24 * 60
self._record_check(prompt_id, "exa-answer", minutes_ago=10, cited=True)
- self._record_check(prompt_id, "exa-answer", minutes_ago=20)
+ self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example Analytics is one option.")
self._record_check(prompt_id, "exa-answer", minutes_ago=day, check_failed=True)
self._record_check(prompt_id, "claude-web-search", minutes_ago=2 * day, cited=True)
self._record_check(prompt_id, "claude-web-search", minutes_ago=40 * day)
- response = self.client.get(f"{self.url}{prompt_id}/stats/")
+ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
+ response = self.client.get(f"{self.url}{prompt_id}/stats/")
assert response.status_code == status.HTTP_200_OK, response.json()
stats = response.json()
assert stats["total_checks"] == 5
assert (stats["daily_from"], stats["daily_to"]) == ("2026-09-21", "2026-10-20")
engines = {
- row["engine"]: (row["checks"], row["cited"], row["failed"], row["citation_rate"])
+ row["engine"]: (
+ row["checks"],
+ row["cited"],
+ row["mentioned"],
+ row["failed"],
+ row["citation_rate"],
+ row["mention_rate"],
+ )
for row in stats["engines"]
}
- assert engines == {"claude-web-search": (2, 1, 0, 0.5), "exa-answer": (3, 1, 1, 0.5)}
- assert [(row["date"], row["engine"], row["checks"], row["cited"], row["failed"]) for row in stats["daily"]] == [
- ("2026-10-18", "claude-web-search", 1, 1, 0),
- ("2026-10-19", "exa-answer", 1, 0, 1),
- ("2026-10-20", "exa-answer", 2, 1, 0),
+ assert engines == {"claude-web-search": (2, 1, 1, 0, 0.5, 0.5), "exa-answer": (3, 1, 2, 1, 0.5, 1.0)}
+ assert [
+ (row["date"], row["engine"], row["checks"], row["cited"], row["mentioned"], row["failed"])
+ for row in stats["daily"]
+ ] == [
+ ("2026-10-18", "claude-web-search", 1, 1, 1, 0),
+ ("2026-10-19", "exa-answer", 1, 0, 0, 1),
+ ("2026-10-20", "exa-answer", 2, 1, 2, 0),
+ ]
+ assert stats["brand_names"] == ["example"]
+ assert (stats["consensus"], stats["top_words"]) == ([], [])
+ assert [(row["engine"], row["unscored"]) for row in stats["sentiment"]] == [
+ ("claude-web-search", 1),
+ ("exa-answer", 2),
+ ]
+
+ @parameterized.expand(
+ [
+ ("brand_named_in_the_answer", {"answer_text": "Try Example Analytics for funnels."}, 1),
+ ("brand_case_ignored", {"answer_text": "EXAMPLE ships session replay."}, 1),
+ ("brand_inside_another_word", {"answer_text": "Counterexample: nothing fits."}, 0),
+ ("cited_without_the_name_in_the_text", {"cited": True, "answer_text": "See [1]."}, 1),
+ ("failed_check", {"check_failed": True}, 0),
+ ]
+ )
+ def test_stats_count_a_mention_when_the_answer_names_the_brand(
+ self, _name: str, check: dict, expected_mentioned: int
+ ) -> None:
+ prompt_id = self._import_one_prompt()
+ self._record_check(prompt_id, "exa-answer", minutes_ago=10, **check)
+
+ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
+ stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()
+
+ assert [row["mentioned"] for row in stats["engines"]] == [expected_mentioned]
+ assert [row["mentioned"] for row in stats["daily"]] == [expected_mentioned]
+
+ def test_stats_count_brand_sentiment_per_engine_and_checks_expose_it(self) -> None:
+ prompt_id = self._import_one_prompt()
+ scored = self._record_check(prompt_id, "exa-answer", minutes_ago=10, answer_text="Example is great.")
+ self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example is slow.")
+ self._record_check(prompt_id, "exa-answer", minutes_ago=30, answer_text="Example exists.")
+ self._record_check(prompt_id, "claude-web-search", minutes_ago=40, answer_text="Nothing about the brand.")
+ with team_scope(self.team.pk):
+ for text, label, score in (("great", "positive", 0.9), ("slow", "negative", 0.7)):
+ AEOCitationCheck.objects.filter(answer_text__contains=text).update(
+ brand_sentiment=label, brand_sentiment_score=score
+ )
+
+ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
+ stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()
+ checks = self.client.get(f"{self.url}{prompt_id}/checks/?engine=exa-answer").json()["results"]
+
+ assert [
+ (row["engine"], row["positive"], row["neutral"], row["negative"], row["unscored"])
+ for row in stats["sentiment"]
+ ] == [("claude-web-search", 0, 0, 0, 0), ("exa-answer", 1, 0, 1, 1)]
+ assert {check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks} == {
+ scored: ("positive", 0.9),
+ **{check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks[1:]},
+ }
+ assert [check["brand_sentiment"] for check in checks] == ["positive", "negative", None]
+
+ @time_machine.travel("2026-10-20T12:00:00Z", tick=False)
+ def test_stats_rank_engine_consensus_and_search_words_over_the_window(self) -> None:
+ prompt_id = self._import_one_prompt()
+ day = 24 * 60
+ ours = "https://docs.example.com/product-analytics"
+ rival = "https://rival.example.org/best-tools"
+ self._record_check(
+ prompt_id,
+ "claude-web-search",
+ minutes_ago=10,
+ cited=True,
+ cited_urls=[rival, ours, "https://rival.example.org/pricing", rival],
+ search_queries=["Best product analytics tools", "product analytics for startups"],
+ )
+ self._record_check(
+ prompt_id,
+ "claude-web-search",
+ minutes_ago=day,
+ cited_urls=[rival],
+ search_queries=["best product analytics tools"],
+ )
+ self._record_check(
+ prompt_id,
+ "openai-web-search",
+ minutes_ago=2 * day,
+ cited=True,
+ cited_urls=["https://community.example.net/thread", ours],
+ search_queries=["open source analytics"],
+ )
+ self._record_check(prompt_id, "exa-answer", minutes_ago=3 * day, cited_urls=[])
+ self._record_check(prompt_id, "openai-web-search", minutes_ago=4 * day, check_failed=True)
+ self._record_check(prompt_id, "openai-web-search", minutes_ago=40 * day, cited_urls=[rival, rival, rival])
+
+ with override_settings(AEO_TARGET_DOMAINS=["example.com"]):
+ stats = self.client.get(f"{self.url}{prompt_id}/stats/").json()
+
+ assert [(row["engine"], row["domain"], row["checks_citing"], row["checks"]) for row in stats["consensus"]] == [
+ ("claude-web-search", "docs.example.com", 1, 2),
+ ("claude-web-search", "rival.example.org", 2, 2),
+ ("claude-web-search", "community.example.net", 0, 2),
+ ("exa-answer", "docs.example.com", 0, 1),
+ ("exa-answer", "rival.example.org", 0, 1),
+ ("exa-answer", "community.example.net", 0, 1),
+ ("openai-web-search", "docs.example.com", 1, 1),
+ ("openai-web-search", "rival.example.org", 0, 1),
+ ("openai-web-search", "community.example.net", 1, 1),
+ ]
+ assert [(row["word"], row["count"]) for row in stats["top_words"]] == [
+ ("analytics", 4),
+ ("product", 3),
+ ("tools", 2),
+ ("open", 1),
+ ("source", 1),
+ ("startups", 1),
]
@parameterized.expand([("above_range", 140), ("not_a_number", "NaN"), ("infinite", "Infinity")])
diff --git a/products/aeo/backend/test/test_brand_sentiment.py b/products/aeo/backend/test/test_brand_sentiment.py
new file mode 100644
index 000000000000..d94352334b40
--- /dev/null
+++ b/products/aeo/backend/test/test_brand_sentiment.py
@@ -0,0 +1,178 @@
+import uuid
+import datetime as dt
+
+import pytest
+from unittest.mock import MagicMock, patch
+
+from django.utils import timezone
+
+from asgiref.sync import sync_to_async
+from temporalio.testing import ActivityEnvironment
+
+from posthog.models.organization import Organization
+from posthog.models.scoping import team_scope
+from posthog.models.team import Team
+from posthog.temporal.ai_observability.sentiment.schema import SentimentResult
+
+from products.aeo.backend.engines import CitationCheck
+from products.aeo.backend.models import AEOCitationCheck, AEOPrompt
+from products.aeo.backend.runner import run_citation_checks
+from products.aeo.backend.seeding import PromptCandidate, upsert_prompts
+from products.aeo.backend.temporal.brand_sentiment import AEOBrandSentimentInputs, classify_aeo_brand_sentiment
+
+_MODULE = "products.aeo.backend.temporal.brand_sentiment"
+
+
+def _team() -> Team:
+ organization = Organization.objects.create(name="aeo sentiment org")
+ return Team.objects.create(organization=organization, name="aeo sentiment team")
+
+
+def _check(team: Team, prompt: AEOPrompt, run_id: uuid.UUID, answer_text: str | None, **fields) -> AEOCitationCheck:
+ with team_scope(team.pk):
+ return AEOCitationCheck.objects.create(
+ team=team,
+ prompt=prompt,
+ run_id=run_id,
+ prompt_text=prompt.prompt,
+ prompt_source=prompt.prompt_source,
+ prompt_hash=prompt.prompt_hash,
+ engine="exa-answer",
+ model="exa-answer",
+ answer_text=answer_text,
+ **fields,
+ )
+
+
+def _seed_run(team: Team) -> tuple[uuid.UUID, dict[str, AEOCitationCheck]]:
+ with team_scope(team.pk):
+ prompt = AEOPrompt.objects.create(
+ team=team, prompt="best tool?", prompt_hash="h", prompt_source=AEOPrompt.Source.MANUAL
+ )
+ run_id = uuid.uuid4()
+ return run_id, {
+ "praise": _check(team, prompt, run_id, "Example is fast. Rival is slow."),
+ "two_sentences": _check(team, prompt, run_id, "Example is slow. Still, Example is cheap."),
+ "no_brand": _check(team, prompt, run_id, "Rival is the only option."),
+ "failed": _check(team, prompt, run_id, None, check_failed=True),
+ "scored": _check(team, prompt, run_id, "Example again.", brand_sentiment="neutral", brand_sentiment_score=0.5),
+ "other_run": _check(team, prompt, uuid.uuid4(), "Example elsewhere."),
+ "no_brand_backlog": _check(team, prompt, uuid.uuid4(), "Rival elsewhere."),
+ "too_old": _check(team, prompt, uuid.uuid4(), "Example long ago."),
+ }
+
+
+def _age(check: AEOCitationCheck, days: int) -> None:
+ with team_scope(check.team_id):
+ AEOCitationCheck.objects.filter(id=check.id).update(created_at=timezone.now() - dt.timedelta(days=days))
+
+
+def _result(**scores: float) -> SentimentResult:
+ label = max(scores, key=scores.__getitem__)
+ return SentimentResult(label=label, score=scores[label], scores=scores)
+
+
[email protected]_db(transaction=True)
+async def test_activity_scores_the_runs_answers_then_the_recent_unscored_backlog(settings) -> None:
+ settings.AEO_TARGET_DOMAINS = ["example.com"]
+ team = await sync_to_async(_team)()
+ run_id, checks = await sync_to_async(_seed_run)(team)
+ await sync_to_async(_age)(checks["other_run"], 10)
+ await sync_to_async(_age)(checks["no_brand_backlog"], 20)
+ await sync_to_async(_age)(checks["too_old"], 31)
+ classify = MagicMock(
+ return_value=[
+ _result(positive=0.9, neutral=0.08, negative=0.02),
+ _result(positive=0.05, neutral=0.15, negative=0.8),
+ _result(positive=0.75, neutral=0.2, negative=0.05),
+ _result(positive=0.1, neutral=0.85, negative=0.05),
+ ]
+ )
+
+ # Cap of 4 leaves one backlog slot after the run's own rows. The older brand-less backlog row
+ # must not take it from the newer answer that names the brand.
+ with (
+ patch("posthog.temporal.ai_observability.sentiment.model.classify", classify),
+ patch(f"{_MODULE}.MAX_ROWS_PER_RUN", 4),
+ ):
+ result = await ActivityEnvironment().run(
+ classify_aeo_brand_sentiment, AEOBrandSentimentInputs(team_id=team.id, run_id=str(run_id))
+ )
+
+ assert result == {"scored": 3, "answers": 3}
+ classify.assert_called_once_with(
+ ["Example is fast.", "Example is slow.", "Still, Example is cheap.", "Example elsewhere."]
+ )
+
+ def sentiments() -> dict[str, tuple[str | None, float | None]]:
+ with team_scope(team.pk):
+ rows = AEOCitationCheck.objects.in_bulk([check.id for check in checks.values()])
+ return {
+ name: (rows[check.id].brand_sentiment, rows[check.id].brand_sentiment_score)
+ for name, check in checks.items()
+ }
+
+ assert await sync_to_async(sentiments)() == {
+ "praise": ("positive", 0.9),
+ "two_sentences": ("negative", 0.425),
+ "no_brand": (None, None),
+ "failed": (None, None),
+ "scored": ("neutral", 0.5),
+ "other_run": ("neutral", 0.85),
+ "no_brand_backlog": (None, None),
+ "too_old": (None, None),
+ }
+
+
[email protected]_db(transaction=True)
[email protected]("error", [ImportError("No module named 'onnxruntime'"), FileNotFoundError("model.onnx")])
+async def test_activity_leaves_rows_unscored_when_the_model_is_unavailable(settings, error: Exception) -> None:
+ settings.AEO_TARGET_DOMAINS = ["example.com"]
+ team = await sync_to_async(_team)()
+ run_id, checks = await sync_to_async(_seed_run)(team)
+
+ with patch("posthog.temporal.ai_observability.sentiment.model.classify", MagicMock(side_effect=error)):
+ result = await ActivityEnvironment().run(
+ classify_aeo_brand_sentiment, AEOBrandSentimentInputs(team_id=team.id, run_id=str(run_id))
+ )
+
+ assert result == {"scored": 0, "answers": 4, "error": "model_unavailable"}
+ unscored = await sync_to_async(
+ lambda: AEOCitationCheck.objects.for_team(team.id).filter(brand_sentiment__isnull=True).count()
+ )()
+ assert unscored == len(checks) - 1
+
+
+class _StubEngine:
+ name = "stub-engine"
+ model = "stub"
+
+ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck:
+ return CitationCheck(engine=self.name, model=self.model, answer_text="Example is fine.")
+
+
+ "enabled, record, expected_starts",
+ [(True, True, 1), (False, True, 0), (True, False, 0)],
+ ids=["on", "off", "dry_run"],
+)
+def test_runner_starts_the_sentiment_workflow_only_when_enabled_and_recording(
+ settings, enabled: bool, record: bool, expected_starts: int
+) -> None:
+ settings.AEO_BRAND_SENTIMENT_ENABLED = enabled
+ team = _team()
+ upsert_prompts(team, [PromptCandidate(text="q?", source="imported")])
+
+ with patch(f"{_MODULE}.sync_connect") as connect, patch(f"{_MODULE}.asyncio.run") as run:
+ summary, _ = run_citation_checks(team, engines=[_StubEngine()], record=record)
+
+ assert summary.rows_written == (1 if record else 0)
+ assert run.call_count == expected_starts
+ if expected_starts:
+ start = connect.return_value.start_workflow
+ start.assert_called_once()
+ assert start.call_args.kwargs["id"] == f"aeo-brand-sentiment-{team.id}-{summary.run_id}"
+ assert start.call_args.kwargs["task_queue"] == settings.LLMA_EVALS_TASK_QUEUE
diff --git a/products/aeo/backend/test/test_sentiment.py b/products/aeo/backend/test/test_sentiment.py
new file mode 100644
index 000000000000..3adef29e30b2
--- /dev/null
+++ b/products/aeo/backend/test/test_sentiment.py
@@ -0,0 +1,74 @@
+import pytest
+
+from products.aeo.backend.brand import brand_names, brand_regex
+from products.aeo.backend.sentiment import (
+ MAX_BRAND_SENTENCES,
+ MAX_SENTENCE_CHARS,
+ AnswerSentiment,
+ aggregate_sentiment,
+ brand_sentences,
+)
+
+TARGET = ["docs.example.co.uk", "Example.com"]
+
+
+ "domains, names, regex",
+ [
+ (TARGET, ("example",), r"\m(example)\M"),
+ (["posthog.com", "rival.io"], ("posthog", "rival"), r"\m(posthog|rival)\M"),
+ ([], (), None),
+ (["", "example.com"], ("example",), r"\m(example)\M"),
+ ],
+)
+def test_brand_names_come_from_the_registrable_domain(domains: list[str], names: tuple[str, ...], regex: str | None):
+ assert brand_names(domains) == names
+ assert brand_regex(domains) == regex
+
+
+ "answer, expected",
+ [
+ (
+ "Example is fast. Rival is cheaper! Pick Example if you self-host?",
+ ["Example is fast.", "Pick Example if you self-host?"],
+ ),
+ ("EXAMPLE works\nRival too\nexample's docs help", ["EXAMPLE works", "example's docs help"]),
+ ("A counterexample: nothing here. Examples abound.", []),
+ ("", []),
+ (None, []),
+ ],
+)
+def test_brand_sentences_keep_only_sentences_that_name_the_brand(answer: str | None, expected: list[str]):
+ assert brand_sentences(answer, TARGET) == expected
+
+
+def test_brand_sentences_are_bounded():
+ answer = " ".join(f"Example point {index}." for index in range(MAX_BRAND_SENTENCES + 5)) + " Example " + "x" * 2000
+ sentences = brand_sentences(answer, TARGET)
+ assert len(sentences) == MAX_BRAND_SENTENCES
+ assert all(len(sentence) <= MAX_SENTENCE_CHARS for sentence in sentences)
+ assert brand_sentences("Example is fine.", []) == []
+
+
+ "scores, expected",
+ [
+ ([], None),
+ ([{"positive": 0.9, "neutral": 0.08, "negative": 0.02}], AnswerSentiment(label="positive", score=0.9)),
+ # The mean decides: one strongly negative sentence beats two mildly neutral ones.
+ (
+ [
+ {"positive": 0.1, "neutral": 0.5, "negative": 0.4},
+ {"positive": 0.05, "neutral": 0.45, "negative": 0.5},
+ {"positive": 0.0, "neutral": 0.05, "negative": 0.95},
+ ],
+ AnswerSentiment(label="negative", score=0.6167),
+ ),
+ # A polar label that only just beats neutral stays neutral, as in AI observability.
+ ([{"positive": 0.5, "neutral": 0.4, "negative": 0.1}], AnswerSentiment(label="neutral", score=0.4)),
+ ([{"positive": 1.0}], AnswerSentiment(label="positive", score=1.0)),
+ ],
+)
+def test_aggregate_sentiment_averages_sentence_scores_before_labeling(scores: list[dict[str, float]], expected):
+ assert aggregate_sentiment(scores) == expected
diff --git a/products/aeo/mcp/tools.yaml b/products/aeo/mcp/tools.yaml
index 14b2e1593dc3..87d29dd32963 100644
--- a/products/aeo/mcp/tools.yaml
+++ b/products/aeo/mcp/tools.yaml
@@ -91,10 +91,16 @@ tools:
title: Get a tracked AI prompt's citation stats
description: >
Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the
- prompt's whole history: `checks`, `cited`, `failed`, `citation_rate` (cited share of answered checks, 0-1,
- failed checks left out), and the first and last check times. `daily` has counts per day and engine for the
- last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Use
- it to compare engines or to see whether a prompt's citations are trending up or down.
+ prompt's whole history: `checks`, `cited`, `mentioned` (answers that name the brand), `failed`,
+ `citation_rate` and `mention_rate` (shares of answered checks, 0-1, failed checks left out), and the first
+ and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to
+ `daily_to`, in the project's timezone); days without a check are left out. Over the same 30 days,
+ `consensus` says how many of each engine's answers cited each of the ten most cited domains, `sentiment`
+ counts positive, neutral and negative answers toward the brand per engine (`unscored` answers name the brand
+ but are not scored yet), and `top_words` lists the most used words of the web searches the engines ran.
+ `brand_names` are the words an answer must contain to count as a mention; use them to filter `answer_text`
+ in a HogQL query over `system.aeo_citation_checks`. Use it to compare engines, to see whether a prompt's
+ citations are trending up or down, and to find which sources the engines agree on.
param_overrides:
id:
description: The prompt id (a UUID), from `aeo-prompts-list`.
diff --git a/products/aeo/turbo.json b/products/aeo/turbo.json
index a38de87df10f..2abbe99ac41c 100644
--- a/products/aeo/turbo.json
+++ b/products/aeo/turbo.json
@@ -5,6 +5,7 @@
"inputs": [
"backend/facade/**",
"backend/tasks/**",
+ "backend/temporal/**",
"backend/models.py",
"backend/migrations/**",
"backend/routes.py",
diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptConsensus.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptConsensus.tsx
new file mode 100644
index 000000000000..453be6c54e06
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptConsensus.tsx
@@ -0,0 +1,67 @@
+import clsx from 'clsx'
+
+import { LemonTable, LemonTableColumn, Tooltip } from '@posthog/lemon-ui'
+
+import { percentage } from 'lib/utils/numbers'
+import { pluralize } from 'lib/utils/strings'
+
+import type { PromptConsensusCell, PromptConsensusMatrix } from './promptReport'
+import { engineLabel } from './utils'
+
+type ConsensusRow = PromptConsensusMatrix['rows'][number]
+
+const heatClass = (share: number | null): string =>
+ share === null || share === 0
+ ? ''
+ : share < 0.25
+ ? 'bg-success/15'
+ : share < 0.5
+ ? 'bg-success/30'
+ : share < 0.75
+ ? 'bg-success/50'
+ : 'bg-success/70'
+
+const HeatCell = ({ cell, checks }: { cell: PromptConsensusCell; checks: number }): JSX.Element => (
+ <Tooltip title={`${cell.checksCiting} of ${pluralize(checks, 'answer')} link to ${cell.domain}`}>
+ <span className={clsx('block rounded px-2 py-1 text-right tabular-nums', heatClass(cell.share))}>
+ {cell.share === null ? '-' : percentage(cell.share, 0)}
+ </span>
+ </Tooltip>
+)
+
+export const ContentAutopilotPromptConsensus = ({ matrix }: { matrix: PromptConsensusMatrix }): JSX.Element => {
+ const columns: LemonTableColumn<ConsensusRow, keyof ConsensusRow | undefined>[] = [
+ {
+ title: 'Engine',
+ key: 'engine',
+ render: (_, row) => (
+ <span className="whitespace-nowrap">
+ {engineLabel(row.engine)} <span className="text-muted">({pluralize(row.checks, 'answer')})</span>
+ </span>
+ ),
+ },
+ ...matrix.domains.map(
+ (domain, index): LemonTableColumn<ConsensusRow, keyof ConsensusRow | undefined> => ({
+ title: (
+ <Tooltip title={domain}>
+ <span className="block max-w-32 truncate">{domain}</span>
+ </Tooltip>
+ ),
+ key: domain,
+ align: 'right',
+ render: (_, row) => <HeatCell cell={row.cells[index]} checks={row.checks} />,
+ })
+ ),
+ ]
+ return (
+ <div className="overflow-x-auto">
+ <LemonTable
+ size="small"
+ dataSource={matrix.rows}
+ rowKey="engine"
+ columns={columns}
+ emptyState="No answer in the last 30 days linked to a site."
+ />
+ </div>
+ )
+}
diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsx
index f6c5f549dbb3..5db8e1329afb 100644
--- a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsx
+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptDetail.tsx
@@ -1,6 +1,6 @@
import { useActions, useValues } from 'kea'
-import { LemonBanner, LemonButton, LemonModal, LemonSkeleton, LemonTag } from '@posthog/lemon-ui'
+import { LemonBanner, LemonButton, LemonModal, LemonSegmentedButton, LemonSkeleton, LemonTag } from '@posthog/lemon-ui'
import { TZLabel } from 'lib/components/TZLabel'
import { pluralize } from 'lib/utils/strings'
@@ -9,16 +9,53 @@ import { ContentAutopilotPromptActiveSwitch } from './ContentAutopilotPromptActi
import { ContentAutopilotPromptAnswer } from './ContentAutopilotPromptAnswer'
import { ContentAutopilotPromptArchiveBanner } from './ContentAutopilotPromptArchiveBanner'
import { ContentAutopilotPromptChecks } from './ContentAutopilotPromptChecks'
+import { ContentAutopilotPromptConsensus } from './ContentAutopilotPromptConsensus'
import { contentAutopilotPromptDetailLogic } from './contentAutopilotPromptDetailLogic'
-import { ContentAutopilotPromptEngineSummary } from './ContentAutopilotPromptEngineSummary'
-import { ContentAutopilotPromptHistoryGrid } from './ContentAutopilotPromptHistoryGrid'
+import { ContentAutopilotPromptKpiCard } from './ContentAutopilotPromptKpiCard'
+import { ContentAutopilotPromptQueryTile } from './ContentAutopilotPromptQueryTile'
+import { ContentAutopilotPromptSentiment } from './ContentAutopilotPromptSentiment'
import { ContentAutopilotPromptVersions } from './ContentAutopilotPromptVersions'
+import { promptCitationsContext, promptSearchesContext, promptSourcesContext } from './promptReportColumns'
+
+const Section = ({
+ title,
+ description,
+ children,
+}: {
+ title: string
+ description?: string
+ children: React.ReactNode
+}): JSX.Element => (
+ <section className="flex flex-col gap-3">
+ <div>
+ <h4 className="m-0">{title}</h4>
+ {description ? <p className="m-0 text-xs text-muted">{description}</p> : null}
+ </div>
+ {children}
+ </section>
+)
export const ContentAutopilotPromptDetail = (): JSX.Element | null => {
- const { promptId, prompt, stats, historyGrid, lastCheckedAt, detailError, archiveConfirming } = useValues(
+ const {
+ promptId,
+ prompt,
+ stats,
+ lastCheckedAt,
+ detailError,
+ archiveConfirming,
+ reportKpis,
+ consensusMatrix,
+ sentimentSummary,
+ visibilityMetric,
+ visibilityQuery,
+ sourcesQuery,
+ citationsQuery,
+ searchesQuery,
+ sentimentQuery,
+ } = useValues(contentAutopilotPromptDetailLogic)
+ const { closePromptDetail, openPromptDetail, setArchiveConfirming, setVisibilityMetric } = useActions(
contentAutopilotPromptDetailLogic
)
- const { closePromptDetail, openPromptDetail, setArchiveConfirming } = useActions(contentAutopilotPromptDetailLogic)
if (!promptId) {
return null
@@ -34,20 +71,32 @@ export const ContentAutopilotPromptDetail = (): JSX.Element | null => {
)
}
+ const reportReady =
+ prompt &&
+ stats &&
+ reportKpis &&
+ consensusMatrix &&
+ sentimentSummary &&
+ visibilityQuery &&
+ sourcesQuery &&
+ citationsQuery &&
+ searchesQuery &&
+ sentimentQuery
+
return (
<LemonModal isOpen onClose={closePromptDetail} title={prompt?.prompt ?? 'Loading prompt'} width={960}>
{detailError === 'failed' ? (
<LemonBanner type="error" action={{ children: 'Try again', onClick: () => openPromptDetail(promptId) }}>
Couldn't load this prompt. Try again, and if it keeps happening contact support.
</LemonBanner>
- ) : !prompt || !stats || !historyGrid ? (
+ ) : !reportReady ? (
<div className="flex flex-col gap-3">
<LemonSkeleton className="h-8 w-1/2" />
<LemonSkeleton className="h-24 w-full" />
<LemonSkeleton className="h-40 w-full" />
</div>
) : (
- <div className="flex flex-col gap-6">
+ <div className="@container/prompt-report flex flex-col gap-6">
<div className="flex flex-wrap items-center gap-x-4 gap-y-2 text-sm">
{prompt.topic ? (
<LemonTag>{prompt.topic}</LemonTag>
@@ -84,10 +133,9 @@ export const ContentAutopilotPromptDetail = (): JSX.Element | null => {
</div>
<ContentAutopilotPromptArchiveBanner />
{prompt.version_count > 1 ? (
- <section className="flex flex-col gap-2">
- <h4 className="m-0">Versions</h4>
+ <Section title="Versions">
<ContentAutopilotPromptVersions promptId={prompt.id} />
- </section>
+ </Section>
) : null}
{stats.engines.length === 0 ? (
@@ -99,22 +147,123 @@ export const ContentAutopilotPromptDetail = (): JSX.Element | null => {
</p>
) : (
<>
- <section className="flex flex-col gap-2">
- <h4 className="m-0">Citation rate by engine</h4>
- <ContentAutopilotPromptEngineSummary engines={stats.engines} />
- </section>
- <section className="flex flex-col gap-2">
- <h4 className="m-0">Last 30 days</h4>
- <ContentAutopilotPromptHistoryGrid grid={historyGrid} />
- </section>
- <section className="flex flex-col gap-2">
- <h4 className="m-0">Answers</h4>
+ <Section
+ title="Last 30 days, all engines"
+ description={`${pluralize(reportKpis.answers, 'answer')} across ${pluralize(reportKpis.days, 'day')}.`}
+ >
+ <div className="grid grid-cols-1 @2xl/prompt-report:grid-cols-2 gap-3">
+ <ContentAutopilotPromptKpiCard
+ label="Mentioned in answers"
+ description="Share of answers that name you. An answer that cites your site always counts."
+ kpi={reportKpis.visibility}
+ data-attr="content-autopilot-prompt-kpi-mentioned"
+ />
+ <ContentAutopilotPromptKpiCard
+ label="Cited as a source"
+ description="Share of answers that link to a page on your site."
+ kpi={reportKpis.citation}
+ data-attr="content-autopilot-prompt-kpi-cited"
+ />
+ </div>
+ </Section>
+
+ <Section
+ title="How often engines mention or cite you"
+ description="One line per engine, one point per day it ran."
+ >
+ <ContentAutopilotPromptQueryTile
+ title={
+ visibilityMetric === 'cited'
+ ? 'Share of answers that cite you'
+ : 'Share of answers that mention you'
+ }
+ query={visibilityQuery}
+ uniqueKey={`aeo-prompt-visibility-${promptId}-${visibilityMetric}`}
+ bodyClassName="h-72 p-2"
+ actions={
+ <LemonSegmentedButton
+ size="xsmall"
+ value={visibilityMetric}
+ onChange={setVisibilityMetric}
+ options={[
+ { value: 'cited', label: 'Cited' },
+ { value: 'mentioned', label: 'Mentioned' },
+ ]}
+ data-attr="content-autopilot-prompt-visibility-metric"
+ />
+ }
+ data-attr="content-autopilot-prompt-visibility-tile"
+ />
+ </Section>
+
+ <Section title="Answers" description="The latest answer from each engine.">
<ContentAutopilotPromptAnswer />
- </section>
- <section className="flex flex-col gap-2">
- <h4 className="m-0">All checks</h4>
+ </Section>
+
+ <Section
+ title="Where the answers come from"
+ description="The sites and pages the engines link to when they answer this prompt."
+ >
+ <div className="grid grid-cols-1 @3xl/prompt-report:grid-cols-2 gap-4">
+ <ContentAutopilotPromptQueryTile
+ title="Sites the engines cite for this prompt"
+ query={sourcesQuery}
+ uniqueKey={`aeo-prompt-sources-${promptId}`}
+ context={promptSourcesContext}
+ data-attr="content-autopilot-prompt-sources-tile"
+ />
+ <ContentAutopilotPromptQueryTile
+ title="Pages the engines cite most"
+ query={citationsQuery}
+ uniqueKey={`aeo-prompt-citations-${promptId}`}
+ context={promptCitationsContext}
+ data-attr="content-autopilot-prompt-citations-tile"
+ />
+ </div>
+ </Section>
+
+ <Section
+ title="Which sites each engine relies on"
+ description="The share of each engine's answers that link to a site. A darker cell means the engine links to that site more often."
+ >
+ <ContentAutopilotPromptConsensus matrix={consensusMatrix} />
+ </Section>
+
+ <Section
+ title="How answers talk about you"
+ description="Scored on the sentences that name you, so praise of another tool does not count."
+ >
+ <ContentAutopilotPromptSentiment summary={sentimentSummary} query={sentimentQuery} />
+ </Section>
+
+ <Section
+ title="What the engines searched for"
+ description="The searches an engine ran before it answered. Engines that do not report their searches are left out."
+ >
+ <ContentAutopilotPromptQueryTile
+ title="Searches"
+ query={searchesQuery}
+ uniqueKey={`aeo-prompt-searches-${promptId}`}
+ context={promptSearchesContext}
+ footer={
+ stats.top_words.length ? (
+ <span className="flex flex-wrap items-center gap-1">
+ <span className="mr-1">Most common words in those searches:</span>
+ {stats.top_words.map((word) => (
+ <LemonTag key={word.word} size="small">
+ {word.word} <span className="text-muted">{word.count}</span>
+ </LemonTag>
+ ))}
+ </span>
+ ) : undefined
+ }
+ data-attr="content-autopilot-prompt-searches-tile"
+ />
+ </Section>
+
+ <Section title="All checks">
<ContentAutopilotPromptChecks />
- </section>
+ </Section>
</>
)}
</div>
diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptEngineSummary.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptEngineSummary.tsx
deleted file mode 100644
index e8670a639083..000000000000
--- a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptEngineSummary.tsx
+++ /dev/null
@@ -1,30 +0,0 @@
-import { LemonCard } from '@posthog/lemon-ui'
-
-import { pluralize } from 'lib/utils/strings'
-
-import type { AEOPromptEngineStatsApi } from 'products/web_analytics/frontend/generated/api.schemas'
-
-import { engineLabel } from './utils'
-
-export const ContentAutopilotPromptEngineSummary = ({
- engines,
-}: {
- engines: AEOPromptEngineStatsApi[]
-}): JSX.Element => (
- <div className="grid grid-cols-[repeat(auto-fill,minmax(12rem,1fr))] gap-2">
- {engines.map((engine) => (
- <LemonCard key={engine.engine} hoverEffect={false} className="p-3 flex flex-col gap-1">
- <span className="text-muted text-xs font-semibold uppercase">{engineLabel(engine.engine)}</span>
- <span className="text-2xl font-semibold">
- {engine.citation_rate === null ? 'No answers' : `${Math.round(engine.citation_rate * 100)}%`}
- </span>
- <span className="text-muted text-xs">
- Cited in {engine.cited} of {pluralize(engine.checks - engine.failed, 'answer')}
- </span>
- <span className="text-muted text-xs">
- {pluralize(engine.checks, 'check')}, {engine.failed} failed
- </span>
- </LemonCard>
- ))}
- </div>
-)
diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptHistoryGrid.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptHistoryGrid.tsx
deleted file mode 100644
index c6085003d196..000000000000
--- a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptHistoryGrid.tsx
+++ /dev/null
@@ -1,70 +0,0 @@
-import clsx from 'clsx'
-
-import { Tooltip } from '@posthog/lemon-ui'
-
-import { dayjs } from 'lib/dayjs'
-
-import { PromptHistoryCell, PromptHistoryCellStatus, PromptHistoryGrid, engineLabel } from './utils'
-
-const STATUS_LABELS: Record<PromptHistoryCellStatus, string> = {
- cited: 'Cited',
- not_cited: 'Not cited',
- failed: 'Failed',
- no_run: 'No run',
-}
-
-const STATUS_CLASSES: Record<PromptHistoryCellStatus, string> = {
- cited: 'bg-success',
- not_cited: 'bg-warning',
- failed: 'bg-danger',
- no_run: 'bg-border',
-}
-
-const cellTooltip = ({ date, status, checks, cited, failed }: PromptHistoryCell): string => {
- const day = dayjs(date).format('MMM D, YYYY')
- if (status === 'no_run') {
- return `${day}: no run`
- }
- const answered = checks - failed
- const failures = failed ? `, ${failed} failed` : ''
- return `${day}: ${STATUS_LABELS[status].toLowerCase()} (cited in ${cited} of ${answered} answers${failures})`
-}
-
-export const ContentAutopilotPromptHistoryGrid = ({ grid }: { grid: PromptHistoryGrid }): JSX.Element => (
- <div className="flex flex-col gap-2">
- <div className="overflow-x-auto">
- <div className="flex flex-col gap-1 w-max">
- {grid.rows.map(({ engine, cells }) => (
- <div key={engine} className="flex items-center gap-2">
- <span className="w-20 shrink-0 text-xs text-muted truncate">{engineLabel(engine)}</span>
- <div className="flex gap-0.5">
- {cells.map((cell) => (
- <Tooltip key={cell.date} title={cellTooltip(cell)}>
- <span
- role="img"
- aria-label={cellTooltip(cell)}
- className={clsx('block size-4 rounded-sm', STATUS_CLASSES[cell.status])}
- />
- </Tooltip>
- ))}
- </div>
- </div>
- ))}
- {grid.dates.length > 0 ? (
- <div className="flex justify-between pl-22 text-xs text-muted">
- <span>{dayjs(grid.dates[0]).format('MMM D')}</span>
- <span>{dayjs(grid.dates[grid.dates.length - 1]).format('MMM D')}</span>
- </div>
- ) : null}
- </div>
- </div>
- <div className="flex flex-wrap gap-3 text-xs text-muted">
- {(Object.keys(STATUS_LABELS) as PromptHistoryCellStatus[]).map((status) => (
- <span key={status} className="flex items-center gap-1">
- <span className={clsx('block size-3 rounded-sm', STATUS_CLASSES[status])} />
- {STATUS_LABELS[status]}
- </span>
- ))}
- </div>
- </div>
-)
diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptKpiCard.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptKpiCard.tsx
new file mode 100644
index 000000000000..c7371334f6da
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptKpiCard.tsx
@@ -0,0 +1,54 @@
+import { IconTrending } from '@posthog/icons'
+import { LemonCard, LemonTag, Tooltip } from '@posthog/lemon-ui'
+
+import { dayjs } from 'lib/dayjs'
+import { IconTrendingDown, IconTrendingFlat } from 'lib/lemon-ui/icons'
+import { percentage } from 'lib/utils/numbers'
+
+import type { PromptRateKpi } from './promptReport'
+
+const DeltaTag = ({ delta }: { delta: number }): JSX.Element => {
+ const points = Math.round(delta * 100)
+ const Icon = points === 0 ? IconTrendingFlat : points > 0 ? IconTrending : IconTrendingDown
+ return (
+ <Tooltip title="The latest day against the 30-day average, in percentage points">
+ <LemonTag
+ type={points === 0 ? 'muted' : points > 0 ? 'success' : 'danger'}
+ size="small"
+ icon={<Icon />}
+ className="tabular-nums"
+ >
+ {points === 0 ? 'No change' : `${points > 0 ? '+' : ''}${points} pts`}
+ </LemonTag>
+ </Tooltip>
+ )
+}
+
+export const ContentAutopilotPromptKpiCard = ({
+ label,
+ description,
+ kpi,
+ 'data-attr': dataAttr,
+}: {
+ label: string
+ description: string
+ kpi: PromptRateKpi
+ 'data-attr'?: string
+}): JSX.Element => (
+ <LemonCard hoverEffect={false} className="p-3 flex flex-col gap-1 min-w-0" data-attr={dataAttr}>
+ <div className="flex items-center justify-between gap-2">
+ <Tooltip title={description}>
+ <span className="text-muted text-xs font-semibold uppercase truncate">{label}</span>
+ </Tooltip>
+ {kpi.delta !== null ? <DeltaTag delta={kpi.delta} /> : null}
+ </div>
+ <span className="text-2xl font-semibold tabular-nums">
+ {kpi.average === null ? 'No answers' : percentage(kpi.average, 0)}
+ </span>
+ <span className="text-muted text-xs">
+ {kpi.latest === null || !kpi.latestDate
+ ? 'Average over the last 30 days'
+ : `Average over the last 30 days. On ${dayjs(kpi.latestDate).format('MMM D')}: ${percentage(kpi.latest, 0)}`}
+ </span>
+ </LemonCard>
+)
diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptQueryTile.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptQueryTile.tsx
new file mode 100644
index 000000000000..448badc4c714
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptQueryTile.tsx
@@ -0,0 +1,61 @@
+import clsx from 'clsx'
+import { ReactNode } from 'react'
+
+import { LemonButton } from '@posthog/lemon-ui'
+
+import { IconOpenInNew } from 'lib/lemon-ui/icons'
+import { urls } from 'scenes/urls'
+import { PagePerformanceCard } from 'scenes/web-analytics/PagePerformanceCard'
+import { PagePerformanceCardHeader } from 'scenes/web-analytics/PagePerformanceCardHeader'
+
+import { Query } from '~/queries/Query/Query'
+import { DataTableNode, DataVisualizationNode } from '~/queries/schema/schema-general'
+import { QueryContext } from '~/queries/types'
+
+/** One report tile: a card with the web analytics tile header, a read-only query inside, and a way to open
+ * the same query as an insight so it can be saved or added to a dashboard. */
+export const ContentAutopilotPromptQueryTile = ({
+ title,
+ query,
+ uniqueKey,
+ context,
+ actions,
+ footer,
+ className,
+ bodyClassName,
+ 'data-attr': dataAttr,
+}: {
+ title: string
+ query: DataTableNode | DataVisualizationNode
+ uniqueKey: string
+ context?: QueryContext
+ actions?: ReactNode
+ footer?: ReactNode
+ className?: string
+ bodyClassName?: string
+ 'data-attr': string
+}): JSX.Element => (
+ <PagePerformanceCard className={clsx('min-w-0', className)} footer={footer}>
+ <PagePerformanceCardHeader
+ title={title}
+ actions={
+ <div className="flex flex-wrap items-center gap-2">
+ {actions}
+ <LemonButton
+ size="small"
+ icon={<IconOpenInNew />}
+ to={urls.insightNew({ query })}
+ targetBlank
+ tooltip="Open as insight"
+ aria-label="Open as insight"
+ data-attr={`${dataAttr}-open-insight`}
+ />
+ </div>
+ }
+ />
+ {/* No flex-1 here: in the card's column layout it would override a fixed chart height from `bodyClassName`. */}
+ <div className={clsx('flex flex-col min-h-0 min-w-0', bodyClassName)} data-attr={dataAttr}>
+ <Query uniqueKey={uniqueKey} query={query} readOnly embedded context={context} />
+ </div>
+ </PagePerformanceCard>
+)
diff --git a/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptSentiment.tsx b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptSentiment.tsx
new file mode 100644
index 000000000000..b0b27a0af4a5
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/ContentAutopilotPromptSentiment.tsx
@@ -0,0 +1,53 @@
+import { percentage } from 'lib/utils/numbers'
+import { pluralize } from 'lib/utils/strings'
+
+import type { DataVisualizationNode } from '~/queries/schema/schema-general'
+
+import { ContentAutopilotPromptQueryTile } from './ContentAutopilotPromptQueryTile'
+import type { PromptSentimentCounts, PromptSentimentSummary } from './promptReport'
+
+const LABELS: { key: keyof PromptSentimentCounts; label: string }[] = [
+ { key: 'positive', label: 'positive' },
+ { key: 'neutral', label: 'neutral' },
+ { key: 'negative', label: 'negative' },
+]
+
+export const ContentAutopilotPromptSentiment = ({
+ summary,
+ query,
+}: {
+ summary: PromptSentimentSummary
+ query: DataVisualizationNode
+}): JSX.Element => {
+ if (summary.scored === 0) {
+ return (
+ <p className="m-0 text-muted">
+ {summary.unscored > 0
+ ? `${pluralize(summary.unscored, 'answer')} mention you but have not been scored yet.`
+ : 'No sentences mention you yet, so there is nothing to score.'}
+ </p>
+ )
+ }
+ return (
+ <div className="flex flex-col gap-3">
+ <p className="m-0 text-sm">
+ {pluralize(summary.scored, 'scored answer')} in the last 30 days:{' '}
+ {LABELS.map(({ key, label }, index) => (
+ <span key={key}>
+ {index > 0 ? ', ' : ''}
+ <span className="font-semibold tabular-nums">{summary[key]}</span> {label} (
+ {percentage(summary[key] / summary.scored, 0)})
+ </span>
+ ))}
+ {summary.unscored > 0 ? `. ${pluralize(summary.unscored, 'answer')} not scored yet.` : '.'}
+ </p>
+ <ContentAutopilotPromptQueryTile
+ title="Scored answers per engine"
+ query={query}
+ uniqueKey={`aeo-prompt-sentiment-${query.source.values?.prompt_id ?? ''}`}
+ bodyClassName="h-64 p-2"
+ data-attr="content-autopilot-prompt-sentiment-tile"
+ />
+ </div>
+ )
+}
diff --git a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.ts b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.ts
index 36ee312931f4..a33167888bba 100644
--- a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.ts
+++ b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.test.ts
@@ -5,6 +5,7 @@ import { expectLogic } from 'kea-test-utils'
import { ApiError } from 'lib/api-error'
import { resumeKeaLoadersErrors, silenceKeaLoadersErrors } from '~/initKea'
+import type { HogQLQuery } from '~/queries/schema/schema-general'
import { initKeaTests } from '~/test/init'
import {
@@ -77,22 +78,38 @@ describe('contentAutopilotPromptDetailLogic', () => {
expect(mockChecks).toHaveBeenCalledWith(TEAM_ID, PROMPT.id, { engine, limit: 1 })
}
expect(logic.values.prompt?.id).toBe(PROMPT.id)
- expect(logic.values.historyGrid?.dates).toHaveLength(30)
- expect(logic.values.historyGrid?.rows.map((row) => row.engine)).toEqual([
- 'claude-web-search',
- 'openai-web-search',
- 'exa-answer',
- ])
- expect(logic.values.historyGrid?.rows[0].cells.slice(0, 4).map((cell) => cell.status)).toEqual([
- 'cited',
- 'cited',
- 'no_run',
- 'not_cited',
- ])
+ expect(logic.values.engines).toEqual(['claude-web-search', 'openai-web-search', 'exa-answer'])
expect(logic.values.activeEngine).toBe('claude-web-search')
expect(logic.values.shownCheck?.id).toBe(EXAMPLE_PROMPT_CHECKS[0].id)
expect(mockVersions).toHaveBeenCalledWith(TEAM_ID, PROMPT.id)
expect(logic.values.versions).toEqual(EXAMPLE_PROMPT_VERSIONS)
+ expect(logic.values.reportKpis?.days).toBe(30)
+ expect(logic.values.consensusMatrix?.rows.map((row) => row.engine)).toEqual([
+ 'claude-web-search',
+ 'openai-web-search',
+ 'exa-answer',
+ ])
+ })
+
+ it('scopes the report queries to the prompt and the stats window, and switches the visibility metric', async () => {
+ logic.actions.openPromptDetail(PROMPT.id)
+ await expectLogic(logic).toFinishAllListeners()
+
+ const citedQuery = logic.values.visibilityQuery!
+ expect(citedQuery.source.values).toEqual({ prompt_id: PROMPT.id, window_start: '2026-09-01 00:00:00' })
+ expect(citedQuery.source.query).toContain('countIf(cited)')
+ expect((logic.values.sourcesQuery!.source as HogQLQuery).values).toEqual(citedQuery.source.values)
+ expect(logic.values.sentimentQuery!.source.values).toEqual(citedQuery.source.values)
+ expect(logic.values.sentimentQuery!.source.query).toContain('brand_sentiment IS NOT NULL')
+
+ logic.actions.setVisibilityMetric('mentioned')
+ const mentionedQuery = logic.values.visibilityQuery!
+ expect(mentionedQuery.source.query).toContain('{brand_pattern}')
+ expect(mentionedQuery.source.values?.brand_pattern).toBe('\\b(example)\\b')
+
+ // A new prompt starts over on the citation metric.
+ logic.actions.openPromptDetail(EXAMPLE_PROMPTS[1].id)
+ expect(logic.values.visibilityMetric).toBe('cited')
})
it('does not load versions for a prompt with one version', async () => {
diff --git a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.ts b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.ts
index 36929e1573c1..7c636837692b 100644
--- a/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.ts
+++ b/products/web_analytics/frontend/contentAutopilot/contentAutopilotPromptDetailLogic.ts
@@ -6,6 +6,8 @@ import { ApiError } from 'lib/api-error'
import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
import { teamLogic } from 'scenes/teamLogic'
+import type { DataTableNode, DataVisualizationNode } from '~/queries/schema/schema-general'
+
import {
aeoPromptsChecks,
aeoPromptsPartialUpdate,
@@ -22,7 +24,24 @@ import type {
} from 'products/web_analytics/frontend/generated/api.schemas'
import { contentAutopilotPromptsLogic } from './contentAutopilotPromptsLogic'
-import { PromptHistoryGrid, promptHistoryGrid, sortEngines } from './utils'
+import {
+ PromptConsensusMatrix,
+ PromptReportKpis,
+ PromptSentimentSummary,
+ PromptVisibilityMetric,
+ promptConsensusMatrix,
+ promptReportKpis,
+ promptSentimentSummary,
+} from './promptReport'
+import {
+ PromptReportWindow,
+ promptCitationsQuery,
+ promptSearchesQuery,
+ promptSentimentQuery,
+ promptSourcesQuery,
+ promptVisibilityQuery,
+} from './promptReportQueries'
+import { sortEngines } from './utils'
export const PROMPT_CHECKS_PAGE_SIZE = 10
@@ -37,9 +56,10 @@ export interface contentAutopilotPromptDetailLogicValues {
checksFailed: boolean
checksLoading: boolean
checksPage: number
+ citationsQuery: DataTableNode | null
+ consensusMatrix: PromptConsensusMatrix | null
detailError: ContentAutopilotPromptDetailError | null
engines: string[]
- historyGrid: PromptHistoryGrid | null
lastCheckedAt: string | null
latestChecks: Record<string, AEOPromptCheckApi>
latestChecksFailed: boolean
@@ -48,14 +68,22 @@ export interface contentAutopilotPromptDetailLogicValues {
promptArchiving: boolean
promptId: string | null
promptLoading: boolean
+ reportKpis: PromptReportKpis | null
+ reportWindow: PromptReportWindow | null
+ searchesQuery: DataTableNode | null
selectedCheck: AEOPromptCheckApi | null
selectedEngine: string | null
+ sentimentQuery: DataVisualizationNode | null
+ sentimentSummary: PromptSentimentSummary | null
shownCheck: AEOPromptCheckApi | null
+ sourcesQuery: DataTableNode | null
stats: AEOPromptStatsApi | null
statsLoading: boolean
versions: AEOPromptVersionApi[] | null
versionsFailed: boolean
versionsLoading: boolean
+ visibilityMetric: PromptVisibilityMetric
+ visibilityQuery: DataVisualizationNode | null
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
@@ -196,6 +224,9 @@ export interface contentAutopilotPromptDetailLogicActions {
setPromptArchivedSuccess: (prompt: AEOPromptApi) => {
prompt: AEOPromptApi
}
+ setVisibilityMetric: (metric: PromptVisibilityMetric) => {
+ metric: PromptVisibilityMetric
+ }
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
@@ -208,8 +239,19 @@ export interface contentAutopilotPromptDetailLogicMeta {
activeEngine: string | null,
latestChecks: Record<string, AEOPromptCheckApi>
) => AEOPromptCheckApi | null
- historyGrid: (stats: AEOPromptStatsApi | null, engines: string[]) => PromptHistoryGrid | null
lastCheckedAt: (stats: AEOPromptStatsApi | null) => string | null
+ reportKpis: (stats: AEOPromptStatsApi | null) => PromptReportKpis | null
+ consensusMatrix: (stats: AEOPromptStatsApi | null, engines: string[]) => PromptConsensusMatrix | null
+ sentimentSummary: (stats: AEOPromptStatsApi | null, engines: string[]) => PromptSentimentSummary | null
+ reportWindow: (promptId: string | null, stats: AEOPromptStatsApi | null) => PromptReportWindow | null
+ visibilityQuery: (
+ reportWindow: PromptReportWindow | null,
+ visibilityMetric: PromptVisibilityMetric
+ ) => DataVisualizationNode | null
+ sourcesQuery: (reportWindow: PromptReportWindow | null) => DataTableNode | null
+ citationsQuery: (reportWindow: PromptReportWindow | null) => DataTableNode | null
+ searchesQuery: (reportWindow: PromptReportWindow | null) => DataTableNode | null
+ sentimentQuery: (reportWindow: PromptReportWindow | null) => DataVisualizationNode | null
}
}
@@ -245,6 +287,7 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetai
selectEngine: (engine: string) => ({ engine }),
selectCheck: (check: AEOPromptCheckApi) => ({ check }),
setChecksPage: (page: number) => ({ page }),
+ setVisibilityMetric: (metric: PromptVisibilityMetric) => ({ metric }),
}),
loaders(({ values }) => ({
prompt: [
@@ -379,6 +422,13 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetai
selectCheck: (_, { check }) => check,
},
],
+ visibilityMetric: [
+ 'cited' as PromptVisibilityMetric,
+ {
+ openPromptDetail: () => 'cited',
+ setVisibilityMetric: (_, { metric }) => metric,
+ },
+ ],
detailError: [
null as ContentAutopilotPromptDetailError | null,
{
@@ -421,11 +471,6 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetai
): AEOPromptCheckApi | null =>
selectedCheck ?? (activeEngine ? (latestChecks[activeEngine] ?? null) : null),
],
- historyGrid: [
- (s) => [s.stats, s.engines],
- (stats: AEOPromptStatsApi | null, engines: string[]): PromptHistoryGrid | null =>
- stats ? promptHistoryGrid(stats, engines) : null,
- ],
lastCheckedAt: [
(s) => [s.stats],
(stats: AEOPromptStatsApi | null): string | null =>
@@ -435,6 +480,50 @@ export const contentAutopilotPromptDetailLogic = kea<contentAutopilotPromptDetai
null
),
],
+ reportKpis: [
+ (s) => [s.stats],
+ (stats: AEOPromptStatsApi | null): PromptReportKpis | null => (stats ? promptReportKpis(stats) : null),
+ ],
+ consensusMatrix: [
+ (s) => [s.stats, s.engines],
+ (stats: AEOPromptStatsApi | null, engines: string[]): PromptConsensusMatrix | null =>
+ stats ? promptConsensusMatrix(stats, engines) : null,
+ ],
+ sentimentSummary: [
+ (s) => [s.stats, s.engines],
+ (stats: AEOPromptStatsApi | null, engines: string[]): PromptSentimentSummary | null =>
+ stats ? promptSentimentSummary(stats, engines) : null,
+ ],
+ reportWindow: [
+ (s) => [s.promptId, s.stats],
+ (promptId: string | null, stats: AEOPromptStatsApi | null): PromptReportWindow | null =>
+ promptId && stats ? { promptId, from: stats.daily_from, brandNames: stats.brand_names } : null,
+ ],
+ visibilityQuery: [
+ (s) => [s.reportWindow, s.visibilityMetric],
+ (reportWindow: PromptReportWindow | null, visibilityMetric: PromptVisibilityMetric) =>
+ reportWindow ? promptVisibilityQuery(reportWindow, visibilityMetric) : null,
+ ],
+ sourcesQuery: [
+ (s) => [s.reportWindow],
+ (reportWindow: PromptReportWindow | null): DataTableNode | null =>
+ reportWindow ? promptSourcesQuery(reportWindow) : null,
+ ],
+ citationsQuery: [
+ (s) => [s.reportWindow],
+ (reportWindow: PromptReportWindow | null): DataTableNode | null =>
+ reportWindow ? promptCitationsQuery(reportWindow) : null,
+ ],
+ searchesQuery: [
+ (s) => [s.reportWindow],
+ (reportWindow: PromptReportWindow | null): DataTableNode | null =>
+ reportWindow ? promptSearchesQuery(reportWindow) : null,
+ ],
+ sentimentQuery: [
+ (s) => [s.reportWindow],
+ (reportWindow: PromptReportWindow | null): DataVisualizationNode | null =>
+ reportWindow ? promptSentimentQuery(reportWindow) : null,
+ ],
}),
listeners(({ actions, values }) => ({
openPromptDetail: ({ promptId }) => {
diff --git a/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.ts b/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.ts
index 88007a0eb620..373787c9dac4 100644
--- a/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.ts
+++ b/products/web_analytics/frontend/contentAutopilot/contentAutopilotStoryFixtures.ts
@@ -348,6 +348,8 @@ const EXAMPLE_CHECK: AEOPromptCheckApi = {
top_cited_domains: ['analytics-review.example.com', 'example.com', 'community.example.org'],
search_queries: ['best open source product analytics', 'self-hosted product analytics for startups'],
prompt_version: 2,
+ brand_sentiment: 'positive',
+ brand_sentiment_score: 0.86,
answer_text:
'For a small startup, a few open source product analytics tools stand out.\n\n' +
'1. Example Analytics: event tracking, funnels and retention, with a generous free tier.\n' +
@@ -367,6 +369,8 @@ export const EXAMPLE_PROMPT_CHECKS: AEOPromptCheckApi[] = [
target_best_position: null,
top_cited_domains: ['analytics-review.example.com'],
search_queries: ['open source product analytics startup'],
+ brand_sentiment: null,
+ brand_sentiment_score: null,
answer_text: 'Most startups begin with a hosted tool.\nCompare pricing tiers before you commit.',
},
{
@@ -383,23 +387,27 @@ export const EXAMPLE_PROMPT_CHECKS: AEOPromptCheckApi[] = [
target_best_position: null,
top_cited_domains: [],
search_queries: [],
+ brand_sentiment: null,
+ brand_sentiment_score: null,
answer_text: null,
},
]
-const EXAMPLE_ENGINE_DAYS: Record<string, ('cited' | 'not_cited' | 'failed' | null)[]> = {
- 'claude-web-search': ['cited', 'cited', null, 'not_cited', 'cited', 'cited', 'cited', 'failed', 'cited', 'cited'],
+type ExampleDay = 'cited' | 'mentioned' | 'not_cited' | 'failed' | null
+
+const EXAMPLE_ENGINE_DAYS: Record<string, ExampleDay[]> = {
+ 'claude-web-search': ['cited', 'cited', null, 'mentioned', 'cited', 'cited', 'cited', 'failed', 'cited', 'cited'],
'openai-web-search': [
'not_cited',
- 'not_cited',
+ 'mentioned',
null,
'cited',
'not_cited',
- 'not_cited',
+ 'mentioned',
null,
'not_cited',
'cited',
- 'not_cited',
+ 'mentioned',
],
'exa-answer': ['cited', 'failed', null, 'cited', 'cited', 'not_cited', 'cited', 'cited', 'failed', 'cited'],
}
@@ -413,29 +421,62 @@ const EXAMPLE_PROMPT_DAILY: AEOPromptStatsApi['daily'] = Object.entries(EXAMPLE_
engine,
checks: 1,
cited: status === 'cited' ? 1 : 0,
+ mentioned: status === 'cited' || status === 'mentioned' ? 1 : 0,
failed: status === 'failed' ? 1 : 0,
}))
)
const exampleEngineStats = (engine: string): AEOPromptStatsApi['engines'][number] => {
const days = EXAMPLE_PROMPT_DAILY.filter((row) => row.engine === engine)
- const sum = (key: 'checks' | 'cited' | 'failed'): number => days.reduce((total, row) => total + row[key], 0)
+ const sum = (key: 'checks' | 'cited' | 'mentioned' | 'failed'): number =>
+ days.reduce((total, row) => total + row[key], 0)
const answered = sum('checks') - sum('failed')
return {
engine,
checks: sum('checks'),
cited: sum('cited'),
+ mentioned: sum('mentioned'),
failed: sum('failed'),
citation_rate: answered ? sum('cited') / answered : null,
+ mention_rate: answered ? sum('mentioned') / answered : null,
first_checked_at: `${days[0].date}T09:00:00Z`,
last_checked_at: `${days[days.length - 1].date}T09:00:00Z`,
}
}
+const EXAMPLE_CONSENSUS_DOMAINS = [
+ 'analytics-review.example.com',
+ 'example.com',
+ 'community.example.org',
+ 'rival.example.net',
+ 'docs.rival.example.net',
+]
+
export const EXAMPLE_PROMPT_STATS: AEOPromptStatsApi = {
total_checks: EXAMPLE_PROMPT_DAILY.length,
+ brand_names: ['example'],
daily_from: '2026-09-01',
daily_to: '2026-09-30',
engines: Object.keys(EXAMPLE_ENGINE_DAYS).map(exampleEngineStats),
daily: EXAMPLE_PROMPT_DAILY,
+ consensus: Object.keys(EXAMPLE_ENGINE_DAYS).flatMap((engine, engineIndex) =>
+ EXAMPLE_CONSENSUS_DOMAINS.map((domain, domainIndex) => ({
+ engine,
+ domain,
+ checks: 24,
+ checks_citing: Math.max(0, 22 - domainIndex * 5 - engineIndex * 3),
+ }))
+ ),
+ sentiment: [
+ { engine: 'claude-web-search', positive: 14, neutral: 6, negative: 1, unscored: 2 },
+ { engine: 'openai-web-search', positive: 5, neutral: 9, negative: 3, unscored: 0 },
+ { engine: 'exa-answer', positive: 0, neutral: 0, negative: 0, unscored: 18 },
+ ],
+ top_words: [
+ { word: 'analytics', count: 41 },
+ { word: 'product', count: 38 },
+ { word: 'open', count: 22 },
+ { word: 'source', count: 22 },
+ { word: 'startups', count: 11 },
+ ],
}
diff --git a/products/web_analytics/frontend/contentAutopilot/promptReport.test.ts b/products/web_analytics/frontend/contentAutopilot/promptReport.test.ts
new file mode 100644
index 000000000000..ebb2e543727b
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/promptReport.test.ts
@@ -0,0 +1,116 @@
+import type { AEOPromptStatsApi } from '../generated/api.schemas'
+import { EXAMPLE_PROMPT_STATS } from './contentAutopilotStoryFixtures'
+import { promptConsensusMatrix, promptReportKpis, promptSentimentSummary } from './promptReport'
+
+const day = (
+ date: string,
+ engine: string,
+ counts: Partial<Pick<AEOPromptStatsApi['daily'][number], 'checks' | 'cited' | 'mentioned' | 'failed'>>
+): AEOPromptStatsApi['daily'][number] => ({ date, engine, checks: 1, cited: 0, mentioned: 0, failed: 0, ...counts })
+
+const stats = (daily: AEOPromptStatsApi['daily'], overrides: Partial<AEOPromptStatsApi> = {}): AEOPromptStatsApi => ({
+ ...EXAMPLE_PROMPT_STATS,
+ daily_from: '2026-09-01',
+ daily_to: '2026-09-10',
+ daily,
+ ...overrides,
+})
+
+describe('promptReport', () => {
+ describe('promptReportKpis', () => {
+ it('pools the window and compares the latest answered day against it', () => {
+ const kpis = promptReportKpis(
+ stats([
+ day('2026-09-01', 'a', { cited: 1, mentioned: 1 }),
+ day('2026-09-01', 'b', { mentioned: 1 }),
+ day('2026-09-02', 'a', { failed: 1 }),
+ day('2026-09-05', 'a', {}),
+ day('2026-09-05', 'b', { cited: 1, mentioned: 1 }),
+ // The latest day only failed, so the day before it is the latest with an answer.
+ day('2026-09-10', 'a', { failed: 1 }),
+ ])
+ )
+
+ expect(kpis).toEqual({
+ answers: 4,
+ days: 10,
+ visibility: { average: 0.75, latest: 0.5, latestDate: '2026-09-05', delta: -0.25 },
+ citation: { average: 0.5, latest: 0.5, latestDate: '2026-09-05', delta: 0 },
+ })
+ })
+
+ it.each([
+ ['no checks', []],
+ ['only failed checks', [day('2026-09-03', 'a', { failed: 1 })]],
+ ])('has no rates with %s', (_name, daily) => {
+ const kpis = promptReportKpis(stats(daily))
+ expect(kpis.answers).toBe(0)
+ expect(kpis.visibility).toEqual({ average: null, latest: null, latestDate: null, delta: null })
+ expect(kpis.citation).toEqual(kpis.visibility)
+ })
+ })
+
+ it('builds the consensus matrix in engine order and drops engines without answers', () => {
+ const matrix = promptConsensusMatrix(
+ stats([], {
+ consensus: [
+ { engine: 'b', domain: 'x.example', checks_citing: 1, checks: 4 },
+ { engine: 'b', domain: 'y.example', checks_citing: 4, checks: 4 },
+ { engine: 'a', domain: 'x.example', checks_citing: 2, checks: 2 },
+ { engine: 'a', domain: 'y.example', checks_citing: 0, checks: 2 },
+ { engine: 'c', domain: 'x.example', checks_citing: 0, checks: 0 },
+ ],
+ }),
+ ['a', 'b', 'c']
+ )
+
+ expect(matrix.domains).toEqual(['x.example', 'y.example'])
+ expect(matrix.rows).toEqual([
+ {
+ engine: 'a',
+ checks: 2,
+ cells: [
+ { domain: 'x.example', checksCiting: 2, share: 1 },
+ { domain: 'y.example', checksCiting: 0, share: 0 },
+ ],
+ },
+ {
+ engine: 'b',
+ checks: 4,
+ cells: [
+ { domain: 'x.example', checksCiting: 1, share: 0.25 },
+ { domain: 'y.example', checksCiting: 4, share: 1 },
+ ],
+ },
+ ])
+ })
+
+ it.each([
+ [
+ 'some scored',
+ [
+ { engine: 'a', positive: 3, neutral: 1, negative: 0, unscored: 2 },
+ { engine: 'b', positive: 0, neutral: 0, negative: 0, unscored: 5 },
+ ],
+ {
+ positive: 3,
+ neutral: 1,
+ negative: 0,
+ unscored: 7,
+ scored: 4,
+ byEngine: [{ engine: 'a', scored: 4, positive: 3, neutral: 1, negative: 0 }],
+ },
+ ],
+ // Scoring off: mentions exist but nothing is scored, so the UI must not claim there are no mentions.
+ [
+ 'mentions but nothing scored',
+ [{ engine: 'a', positive: 0, neutral: 0, negative: 0, unscored: 4 }],
+ { positive: 0, neutral: 0, negative: 0, unscored: 4, scored: 0, byEngine: [] },
+ ],
+ ])(
+ 'sums sentiment across engines with %s and lists only engines with scored answers',
+ (_name, sentiment, expected) => {
+ expect(promptSentimentSummary(stats([], { sentiment }), ['b', 'a'])).toEqual(expected)
+ }
+ )
+})
diff --git a/products/web_analytics/frontend/contentAutopilot/promptReport.ts b/products/web_analytics/frontend/contentAutopilot/promptReport.ts
new file mode 100644
index 000000000000..862bd860db5a
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/promptReport.ts
@@ -0,0 +1,162 @@
+import { dayjs } from 'lib/dayjs'
+
+import type {
+ AEOPromptDailyEngineStatsApi,
+ AEOPromptStatsApi,
+} from 'products/web_analytics/frontend/generated/api.schemas'
+
+export type PromptVisibilityMetric = 'cited' | 'mentioned'
+
+export interface PromptRateKpi {
+ /** Pooled rate over the window: all answers that qualify over all answers. */
+ average: number | null
+ /** Rate on the latest day of the window that had an answer. */
+ latest: number | null
+ latestDate: string | null
+ /** `latest - average`, in rate units (0.12 is 12 percentage points). */
+ delta: number | null
+}
+
+export interface PromptReportKpis {
+ answers: number
+ days: number
+ visibility: PromptRateKpi
+ citation: PromptRateKpi
+}
+
+export interface PromptConsensusCell {
+ domain: string
+ checksCiting: number
+ /** Share of the engine's answers that cited the domain. Null when the engine answered nothing. */
+ share: number | null
+}
+
+export interface PromptConsensusMatrix {
+ domains: string[]
+ rows: { engine: string; checks: number; cells: PromptConsensusCell[] }[]
+}
+
+export interface PromptSentimentCounts {
+ positive: number
+ neutral: number
+ negative: number
+}
+
+export interface PromptSentimentSummary extends PromptSentimentCounts {
+ scored: number
+ unscored: number
+ byEngine: ({ engine: string; scored: number } & PromptSentimentCounts)[]
+}
+
+interface DailyTotals {
+ checks: number
+ cited: number
+ mentioned: number
+ failed: number
+}
+
+/** Every day of the stats window as YYYY-MM-DD, oldest first. */
+export const statsWindowDates = (stats: Pick<AEOPromptStatsApi, 'daily_from' | 'daily_to'>): string[] => {
+ const dates: string[] = []
+ for (let day = dayjs(stats.daily_from); !day.isAfter(dayjs(stats.daily_to), 'day'); day = day.add(1, 'day')) {
+ dates.push(day.format('YYYY-MM-DD'))
+ }
+ return dates
+}
+
+const sumDaily = (rows: AEOPromptDailyEngineStatsApi[]): DailyTotals =>
+ rows.reduce(
+ (totals, row) => ({
+ checks: totals.checks + row.checks,
+ cited: totals.cited + row.cited,
+ mentioned: totals.mentioned + row.mentioned,
+ failed: totals.failed + row.failed,
+ }),
+ { checks: 0, cited: 0, mentioned: 0, failed: 0 }
+ )
+
+const rate = (part: number, whole: number): number | null => (whole > 0 ? part / whole : null)
+
+const rateKpi = (
+ byDay: { date: string; checks: number; failed: number; count: number }[],
+ totals: { checks: number; failed: number; count: number }
+): PromptRateKpi => {
+ const average = rate(totals.count, totals.checks - totals.failed)
+ const latestDay = [...byDay].reverse().find((day) => day.checks - day.failed > 0) ?? null
+ const latest = latestDay ? rate(latestDay.count, latestDay.checks - latestDay.failed) : null
+ return {
+ average,
+ latest,
+ latestDate: latestDay?.date ?? null,
+ delta: average !== null && latest !== null ? latest - average : null,
+ }
+}
+
+/** Window-wide visibility and citation rates, with the latest day and how far it sits from the average. */
+export const promptReportKpis = (stats: AEOPromptStatsApi): PromptReportKpis => {
+ const dates = statsWindowDates(stats)
+ const byDate = new Map<string, AEOPromptDailyEngineStatsApi[]>()
+ for (const row of stats.daily) {
+ byDate.set(row.date, [...(byDate.get(row.date) ?? []), row])
+ }
+ const days = dates.map((date) => ({ date, ...sumDaily(byDate.get(date) ?? []) }))
+ const totals = sumDaily(stats.daily)
+ return {
+ answers: totals.checks - totals.failed,
+ days: dates.length,
+ visibility: rateKpi(
+ days.map((day) => ({ ...day, count: day.mentioned })),
+ { ...totals, count: totals.mentioned }
+ ),
+ citation: rateKpi(
+ days.map((day) => ({ ...day, count: day.cited })),
+ { ...totals, count: totals.cited }
+ ),
+ }
+}
+
+/** Engines as rows, the most cited domains as columns, each cell the share of the engine's answers citing it. */
+export const promptConsensusMatrix = (stats: AEOPromptStatsApi, engines: string[]): PromptConsensusMatrix => {
+ const domains = [...new Set(stats.consensus.map((row) => row.domain))]
+ const byKey = new Map(stats.consensus.map((row) => [`${row.engine}|${row.domain}`, row]))
+ return {
+ domains,
+ rows: engines
+ .map((engine) => {
+ const checks = stats.consensus.find((row) => row.engine === engine)?.checks ?? 0
+ return {
+ engine,
+ checks,
+ cells: domains.map((domain) => {
+ const checksCiting = byKey.get(`${engine}|${domain}`)?.checks_citing ?? 0
+ return { domain, checksCiting, share: rate(checksCiting, checks) }
+ }),
+ }
+ })
+ .filter((row) => row.checks > 0),
+ }
+}
+
+/** Sentiment counts across engines, plus one row per engine that has at least one scored answer. */
+export const promptSentimentSummary = (stats: AEOPromptStatsApi, engines: string[]): PromptSentimentSummary => {
+ const byEngine = engines.flatMap((engine) => {
+ const row = stats.sentiment.find((item) => item.engine === engine)
+ if (!row) {
+ return []
+ }
+ const scored = row.positive + row.neutral + row.negative
+ return scored > 0
+ ? [{ engine, scored, positive: row.positive, neutral: row.neutral, negative: row.negative }]
+ : []
+ })
+ const totals = stats.sentiment.reduce(
+ (sum, row) => ({
+ positive: sum.positive + row.positive,
+ neutral: sum.neutral + row.neutral,
+ negative: sum.negative + row.negative,
+ unscored: sum.unscored + row.unscored,
+ }),
+ { positive: 0, neutral: 0, negative: 0, unscored: 0 }
+ )
+ return { ...totals, scored: totals.positive + totals.neutral + totals.negative, byEngine }
+}
diff --git a/products/web_analytics/frontend/contentAutopilot/promptReportColumns.tsx b/products/web_analytics/frontend/contentAutopilot/promptReportColumns.tsx
new file mode 100644
index 000000000000..c712483a8499
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/promptReportColumns.tsx
@@ -0,0 +1,79 @@
+import { LemonTag, Link } from '@posthog/lemon-ui'
+
+import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
+import { percentage } from 'lib/utils/numbers'
+
+import type { QueryContext, QueryContextColumnComponent } from '~/queries/types'
+
+import { PROMPT_CITATION_COLUMNS, PROMPT_SOURCE_COLUMNS } from './promptReportQueries'
+
+// Engine output is third-party text: only plain http(s) URLs become links.
+const isWebUrl = (url: string): boolean => /^https?:\/\//i.test(url)
+
+/** HogQL rows reach the column renderers as arrays, so a renderer that needs a sibling column looks it up by position. */
+const cellAt = (record: unknown, columns: readonly string[], name: string): unknown =>
+ Array.isArray(record) ? record[columns.indexOf(name)] : undefined
+
+const TargetTag = (): JSX.Element => (
+ <LemonTag type="success" size="small">
+ Your site
+ </LemonTag>
+)
+
+const DomainCell: QueryContextColumnComponent = ({ value, record }) => (
+ <span className="flex flex-wrap items-center gap-2">
+ <span className="break-all">{String(value ?? '')}</span>
+ {cellAt(record, PROMPT_SOURCE_COLUMNS, 'is_target') ? <TargetTag /> : null}
+ </span>
+)
+
+const UrlCell: QueryContextColumnComponent = ({ value, record }) => {
+ const url = String(value ?? '')
+ return (
+ <span className="flex flex-wrap items-center gap-2 min-w-0">
+ {isWebUrl(url) ? (
+ <Link to={url} target="_blank" className="break-all">
+ {url}
+ </Link>
+ ) : (
+ <span className="break-all">{url}</span>
+ )}
+ {cellAt(record, PROMPT_CITATION_COLUMNS, 'is_target') ? <TargetTag /> : null}
+ </span>
+ )
+}
+
+const ShareCell: QueryContextColumnComponent = ({ value }) => {
+ const share = typeof value === 'number' ? value : Number(value ?? 0)
+ return (
+ <span className="flex items-center gap-2 min-w-32">
+ <LemonProgress percent={share * 100} className="flex-1" />
+ <span className="w-10 text-right tabular-nums">{percentage(share, 0)}</span>
+ </span>
+ )
+}
+
+const sharedColumns: QueryContext['columns'] = {
+ answers: { title: 'Answers that link to it', align: 'right' },
+ share_of_answers: { title: 'Share of all answers', render: ShareCell, align: 'right', width: 180 },
+ is_target: { hidden: true },
+}
+
+export const promptSourcesContext: QueryContext = {
+ showQueryEditor: false,
+ columns: { ...sharedColumns, domain: { title: 'Site', render: DomainCell } },
+}
+
+export const promptCitationsContext: QueryContext = {
+ showQueryEditor: false,
+ columns: { ...sharedColumns, url: { title: 'Page', render: UrlCell } },
+}
+
+export const promptSearchesContext: QueryContext = {
+ showQueryEditor: false,
+ columns: {
+ query: { title: 'Search' },
+ searches: { title: 'Times run', align: 'right' },
+ engines: { title: 'Engines that ran it', align: 'right' },
+ },
+}
diff --git a/products/web_analytics/frontend/contentAutopilot/promptReportQueries.ts b/products/web_analytics/frontend/contentAutopilot/promptReportQueries.ts
new file mode 100644
index 000000000000..599568c21550
--- /dev/null
+++ b/products/web_analytics/frontend/contentAutopilot/promptReportQueries.ts
@@ -0,0 +1,158 @@
+import { escapeRegex } from 'lib/utils/actions'
+
+import { DataTableNode, DataVisualizationNode, HogQLQuery, NodeKind } from '~/queries/schema/schema-general'
+import { ChartDisplayType } from '~/types'
+
+import type { PromptVisibilityMetric } from './promptReport'
+
+export interface PromptReportWindow {
+ promptId: string
+ /** First day of the window, YYYY-MM-DD in the project's timezone. */
+ from: string
+ /** The words an answer must contain to count as a brand mention. */
+ brandNames: string[]
+}
+
+export const PROMPT_REPORT_TOP_SOURCES = 50
+export const PROMPT_REPORT_TOP_CITATIONS = 20
+export const PROMPT_REPORT_TOP_SEARCHES = 30
+
+export const PROMPT_SOURCE_COLUMNS = ['domain', 'answers', 'share_of_answers', 'is_target'] as const
+export const PROMPT_CITATION_COLUMNS = ['url', 'answers', 'share_of_answers', 'is_target'] as const
+export const PROMPT_SEARCH_COLUMNS = ['query', 'searches', 'engines'] as const
+
+// Every query is one prompt's answered checks inside the window. The prompt id and the window start travel
+// as placeholder values, so the HogQL text never embeds user data.
+const WHERE = 'prompt_id = {prompt_id} AND created_at >= toDateTime({window_start}) AND NOT check_failed'
+
+const ANSWERED = `(SELECT count() FROM system.aeo_citation_checks WHERE ${WHERE})`
+
+const hogql = (query: string, window: PromptReportWindow, values: Record<string, string> = {}): HogQLQuery => ({
+ kind: NodeKind.HogQLQuery,
+ query,
+ values: { prompt_id: window.promptId, window_start: `${window.from} 00:00:00`, ...values },
+})
+
+/** RE2 pattern for `match()`: any brand name as a whole word, case folded by the caller. */
+export const brandMentionPattern = (brandNames: string[]): string =>
+ `\\b(${brandNames.map((name) => escapeRegex(name.toLowerCase())).join('|')})\\b`
+
+const mentionedExpr = (brandNames: string[]): string =>
+ brandNames.length ? "cited OR match(lower(coalesce(answer_text, '')), {brand_pattern})" : 'cited'
+
+/** Daily citation (or brand mention) rate per engine. Days an engine did not run are gaps, not zeros. */
+export const promptVisibilityQuery = (
+ window: PromptReportWindow,
+ metric: PromptVisibilityMetric
+): DataVisualizationNode => {
+ const counted = metric === 'mentioned' ? mentionedExpr(window.brandNames) : 'cited'
+ return {
+ kind: NodeKind.DataVisualizationNode,
+ source: hogql(
+ `SELECT toDate(created_at) AS day, engine, round(countIf(${counted}) / count(), 3) AS rate
+FROM system.aeo_citation_checks
+WHERE ${WHERE}
+GROUP BY day, engine
+ORDER BY day, engine`,
+ window,
+ metric === 'mentioned' && window.brandNames.length
+ ? { brand_pattern: brandMentionPattern(window.brandNames) }
+ : {}
+ ),
+ display: ChartDisplayType.ActionsLineGraph,
+ chartSettings: {
+ xAxis: { column: 'day' },
+ yAxis: [{ column: 'rate', settings: { formatting: { style: 'percent', decimalPlaces: 0 } } }],
+ seriesBreakdownColumn: 'engine',
+ showLegend: true,
+ },
+ }
+}
+
+const citedUrls = `SELECT id, engine, arrayJoin(JSONExtract(cited_urls, 'Array(String)')) AS url, target_urls
+FROM system.aeo_citation_checks
+WHERE ${WHERE}`
+
+const table = (source: HogQLQuery, columns: readonly string[]): DataTableNode => ({
+ kind: NodeKind.DataTableNode,
+ source,
+ columns: [...columns],
+ hiddenColumns: columns.includes('is_target') ? ['is_target'] : undefined,
+ embedded: true,
+ showActions: false,
+ showOpenEditorButton: false,
+})
+
+/** Cited domains ranked by how many answers cited them, with the share of all answers that did. */
+export const promptSourcesQuery = (window: PromptReportWindow): DataTableNode =>
+ table(
+ hogql(
+ `SELECT domain(url) AS domain,
+ uniq(id) AS answers,
+ round(uniq(id) / ${ANSWERED}, 3) AS share_of_answers,
+ max(has(JSONExtract(target_urls, 'Array(String)'), url)) AS is_target
+FROM (${citedUrls})
+WHERE domain != ''
+GROUP BY domain
+ORDER BY answers DESC, domain
+LIMIT ${PROMPT_REPORT_TOP_SOURCES}`,
+ window
+ ),
+ PROMPT_SOURCE_COLUMNS
+ )
+
+/** Cited URLs ranked by how many answers cited them. */
+export const promptCitationsQuery = (window: PromptReportWindow): DataTableNode =>
+ table(
+ hogql(
+ `SELECT url,
+ uniq(id) AS answers,
+ round(uniq(id) / ${ANSWERED}, 3) AS share_of_answers,
+ max(has(JSONExtract(target_urls, 'Array(String)'), url)) AS is_target
+FROM (${citedUrls})
+GROUP BY url
+ORDER BY answers DESC, url
+LIMIT ${PROMPT_REPORT_TOP_CITATIONS}`,
+ window
+ ),
+ PROMPT_CITATION_COLUMNS
+ )
+
+/** The searches the engines ran, with how often. Only engines that report their searches contribute. */
+export const promptSearchesQuery = (window: PromptReportWindow): DataTableNode =>
+ table(
+ hogql(
+ `SELECT query, count() AS searches, uniq(engine) AS engines
+FROM (SELECT engine, arrayJoin(JSONExtract(search_queries, 'Array(String)')) AS query
+ FROM system.aeo_citation_checks
+ WHERE ${WHERE})
+WHERE query != ''
+GROUP BY query
+ORDER BY searches DESC, query
+LIMIT ${PROMPT_REPORT_TOP_SEARCHES}`,
+ window
+ ),
+ PROMPT_SEARCH_COLUMNS
+ )
+
+/** Scored answers per engine, split by how they speak about the brand. Unscored answers are left out. */
+export const promptSentimentQuery = (window: PromptReportWindow): DataVisualizationNode => ({
+ kind: NodeKind.DataVisualizationNode,
+ source: hogql(
+ `SELECT engine,
+ multiIf(brand_sentiment = 'positive', 'Positive', brand_sentiment = 'negative', 'Negative', 'Neutral') AS sentiment,
+ count() AS answers
+FROM system.aeo_citation_checks
+WHERE ${WHERE} AND brand_sentiment IS NOT NULL
+GROUP BY engine, sentiment
+ORDER BY engine, sentiment`,
+ window
+ ),
+ display: ChartDisplayType.ActionsStackedBar,
+ chartSettings: {
+ xAxis: { column: 'engine' },
+ yAxis: [{ column: 'answers' }],
+ seriesBreakdownColumn: 'sentiment',
+ showLegend: true,
+ },
+})
diff --git a/products/web_analytics/frontend/contentAutopilot/utils.ts b/products/web_analytics/frontend/contentAutopilot/utils.ts
index ba3ddd968f2f..a3791e3d2656 100644
--- a/products/web_analytics/frontend/contentAutopilot/utils.ts
+++ b/products/web_analytics/frontend/contentAutopilot/utils.ts
@@ -1,9 +1,7 @@
-import { dayjs } from 'lib/dayjs'
import { parseWebAnalyticsURL } from 'scenes/web-analytics/common'
import type {
AEOPromptCheckApi,
- AEOPromptStatsApi,
ContentAutopilotOpportunityApi,
ContentAutopilotProposalListApi,
} from 'products/web_analytics/frontend/generated/api.schemas'
@@ -26,48 +24,6 @@ export const sortEngines = (engines: Iterable<string>): string[] => {
return [...new Set(engines)].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))
}
-export type PromptHistoryCellStatus = 'cited' | 'not_cited' | 'failed' | 'no_run'
-
-export interface PromptHistoryCell {
- date: string
- status: PromptHistoryCellStatus
- checks: number
- cited: number
- failed: number
-}
-
-export interface PromptHistoryGrid {
- dates: string[]
- rows: { engine: string; cells: PromptHistoryCell[] }[]
-}
-
-const historyCellStatus = ({
- checks,
- cited,
- failed,
-}: Omit<PromptHistoryCell, 'date' | 'status'>): PromptHistoryCellStatus =>
- checks === 0 ? 'no_run' : cited > 0 ? 'cited' : failed === checks ? 'failed' : 'not_cited'
-
-/** One cell per day and engine across the stats window, with days the engine did not run filled in. */
-export const promptHistoryGrid = (stats: AEOPromptStatsApi, engines: string[]): PromptHistoryGrid => {
- const dates: string[] = []
- for (let day = dayjs(stats.daily_from); !day.isAfter(dayjs(stats.daily_to), 'day'); day = day.add(1, 'day')) {
- dates.push(day.format('YYYY-MM-DD'))
- }
- const byKey = new Map(stats.daily.map((row) => [`${row.engine}|${row.date}`, row]))
- return {
- dates,
- rows: engines.map((engine) => ({
- engine,
- cells: dates.map((date) => {
- const row = byKey.get(`${engine}|${date}`)
- const counts = { checks: row?.checks ?? 0, cited: row?.cited ?? 0, failed: row?.failed ?? 0 }
- return { date, status: historyCellStatus(counts), ...counts }
- }),
- })),
- }
-}
-
export type ContentAutopilotPageTarget =
| { isNewPage: true; path: string }
| { isNewPage: false; path: string; url: string }
diff --git a/products/web_analytics/frontend/generated/api.schemas.ts b/products/web_analytics/frontend/generated/api.schemas.ts
index 051a0eabf951..5c41a73282d6 100644
--- a/products/web_analytics/frontend/generated/api.schemas.ts
+++ b/products/web_analytics/frontend/generated/api.schemas.ts
@@ -147,6 +147,16 @@ export interface AEOPromptCheckApi {
* @nullable
*/
prompt_version: number | null
+ /**
+ * Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand.
+ * @nullable
+ */
+ brand_sentiment?: string | null
+ /**
+ * Classifier confidence (0-1) in 'brand_sentiment'. Null when 'brand_sentiment' is null.
+ * @nullable
+ */
+ brand_sentiment_score?: number | null
}
export interface PaginatedAEOPromptCheckListApi {
@@ -164,12 +174,19 @@ export interface AEOPromptEngineStatsApi {
* @nullable
*/
readonly citation_rate: number | null
+ /**
+ * Share (0-1) of answered checks whose answer names the brand. Failed checks are left out. Null when no check was answered.
+ * @nullable
+ */
+ readonly mention_rate: number | null
/** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */
engine: string
/** Checks of the prompt on this engine, failed ones included. */
checks: number
/** Checks whose answer cited a target domain. */
cited: number
+ /** Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */
+ mentioned: number
/** Checks where the engine did not answer. */
failed: number
/** When the engine first ran the prompt. */
@@ -187,17 +204,58 @@ export interface AEOPromptDailyEngineStatsApi {
checks: number
/** Checks that day whose answer cited a target domain. */
cited: number
+ /** Checks that day whose answer names the brand. Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */
+ mentioned: number
/** Checks that day where the engine did not answer. */
failed: number
}
+export interface AEOPromptEngineDomainShareApi {
+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */
+ engine: string
+ /** One of the most cited domains across all engines in the window. */
+ domain: string
+ /** This engine's answered checks in the last 30 days that cited the domain. */
+ checks_citing: number
+ /** This engine's answered checks in the last 30 days. */
+ checks: number
+}
+
+export interface AEOPromptEngineSentimentApi {
+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */
+ engine: string
+ /** Answers in the last 30 days whose brand sentences read positive. */
+ positive: number
+ /** Answers in the last 30 days whose brand sentences read neutral. */
+ neutral: number
+ /** Answers in the last 30 days whose brand sentences read negative. */
+ negative: number
+ /** Answers in the last 30 days that name the brand but have no sentiment yet. */
+ unscored: number
+}
+
+export interface AEOPromptSearchWordApi {
+ /** A word from the web searches, lower-cased, common English words left out. */
+ word: string
+ /** How many searches used the word, weighted by how often each search ran. */
+ count: number
+}
+
export interface AEOPromptStatsApi {
/** Totals per answer engine over the prompt's whole history. */
engines: AEOPromptEngineStatsApi[]
/** Counts per day and engine from 'daily_from' to 'daily_to' (30 days). Days and engines without a check are left out. */
daily: AEOPromptDailyEngineStatsApi[]
+ /** One row per engine and per domain for the 10 most cited domains of the last 30 days: how many of the engine's answers cited the domain. Shows which sources the engines agree on. */
+ consensus: AEOPromptEngineDomainShareApi[]
+ /** Brand sentiment of the answers in the last 30 days, one row per engine. Only the sentences that name the brand are scored. */
+ sentiment: AEOPromptEngineSentimentApi[]
+ /** Most used words across the web searches the engines ran in the last 30 days, at most 20. Only engines that report their searches contribute. */
+ top_words: AEOPromptSearchWordApi[]
/** Checks of the prompt across all engines, failed ones included. */
total_checks: number
+ /** Lower-cased brand names, derived from the project's target domains. An answer that contains one of them as a word counts as a mention. */
+ brand_names: string[]
/** First day of the daily series, in the project's timezone. */
daily_from: string
/** Last day of the daily series (today), in the project's timezone. */
diff --git a/products/web_analytics/frontend/generated/api.ts b/products/web_analytics/frontend/generated/api.ts
index b979279ca9a8..3831844f7c9b 100644
--- a/products/web_analytics/frontend/generated/api.ts
+++ b/products/web_analytics/frontend/generated/api.ts
@@ -227,7 +227,7 @@ export const getAeoPromptsStatsUrl = (projectId: string, id: string) => {
}
/**
- * Gets citation totals per answer engine for one prompt, plus per-day counts for the last 30 days in the project's timezone.
+ * Gets citation and brand-mention totals per answer engine for one prompt, plus, for the last 30 days in the project's timezone, per-day counts, which engines cite which domains, brand sentiment per engine, and the words of the web searches the engines ran.
* @summary Get an AEO prompt's citation stats
*/
export const aeoPromptsStats = async (
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 2b8ad685c128..5f2a2c00d7b3 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -792,7 +792,7 @@
"feature_flag": "aeo-citation-tracking"
},
"aeo-prompts-stats-get": {
- "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `failed`, `citation_rate` (cited share of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Use it to compare engines or to see whether a prompt's citations are trending up or down.",
+ "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `mentioned` (answers that name the brand), `failed`, `citation_rate` and `mention_rate` (shares of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Over the same 30 days, `consensus` says how many of each engine's answers cited each of the ten most cited domains, `sentiment` counts positive, neutral and negative answers toward the brand per engine (`unscored` answers name the brand but are not scored yet), and `top_words` lists the most used words of the web searches the engines ran. `brand_names` are the words an answer must contain to count as a mention; use them to filter `answer_text` in a HogQL query over `system.aeo_citation_checks`. Use it to compare engines, to see whether a prompt's citations are trending up or down, and to find which sources the engines agree on.",
"category": "AI citation tracking",
"feature": "aeo",
"summary": "Get a tracked AI prompt's citation stats",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index 5b04172e9b1c..12f8025df245 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -792,7 +792,7 @@
"feature_flag": "aeo-citation-tracking"
},
"aeo-prompts-stats-get": {
- "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `failed`, `citation_rate` (cited share of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Use it to compare engines or to see whether a prompt's citations are trending up or down.",
+ "description": "Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the prompt's whole history: `checks`, `cited`, `mentioned` (answers that name the brand), `failed`, `citation_rate` and `mention_rate` (shares of answered checks, 0-1, failed checks left out), and the first and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Over the same 30 days, `consensus` says how many of each engine's answers cited each of the ten most cited domains, `sentiment` counts positive, neutral and negative answers toward the brand per engine (`unscored` answers name the brand but are not scored yet), and `top_words` lists the most used words of the web searches the engines ran. `brand_names` are the words an answer must contain to count as a mention; use them to filter `answer_text` in a HogQL query over `system.aeo_citation_checks`. Use it to compare engines, to see whether a prompt's citations are trending up or down, and to find which sources the engines agree on.",
"category": "AI citation tracking",
"feature": "aeo",
"summary": "Get a tracked AI prompt's citation stats",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index fccaf380c037..d920f39d2b5f 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -146,6 +146,16 @@ export namespace Schemas {
* @nullable
*/
prompt_version: number | null;
+ /**
+ * Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand.
+ * @nullable
+ */
+ brand_sentiment?: string | null;
+ /**
+ * Classifier confidence (0-1) in 'brand_sentiment'. Null when 'brand_sentiment' is null.
+ * @nullable
+ */
+ brand_sentiment_score?: number | null;
}
export interface AEOPromptDailyEngineStats {
@@ -157,22 +167,55 @@ export namespace Schemas {
checks: number;
/** Checks that day whose answer cited a target domain. */
cited: number;
+ /** Checks that day whose answer names the brand. Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */
+ mentioned: number;
/** Checks that day where the engine did not answer. */
failed: number;
}
+ export interface AEOPromptEngineDomainShare {
+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */
+ engine: string;
+ /** One of the most cited domains across all engines in the window. */
+ domain: string;
+ /** This engine's answered checks in the last 30 days that cited the domain. */
+ checks_citing: number;
+ /** This engine's answered checks in the last 30 days. */
+ checks: number;
+ }
+
+ export interface AEOPromptEngineSentiment {
+ /** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */
+ engine: string;
+ /** Answers in the last 30 days whose brand sentences read positive. */
+ positive: number;
+ /** Answers in the last 30 days whose brand sentences read neutral. */
+ neutral: number;
+ /** Answers in the last 30 days whose brand sentences read negative. */
+ negative: number;
+ /** Answers in the last 30 days that name the brand but have no sentiment yet. */
+ unscored: number;
+ }
+
export interface AEOPromptEngineStats {
/**
* Share (0-1) of answered checks that cited a target domain. Failed checks are left out. Null when no check was answered.
* @nullable
*/
readonly citation_rate: number | null;
+ /**
+ * Share (0-1) of answered checks whose answer names the brand. Failed checks are left out. Null when no check was answered.
+ * @nullable
+ */
+ readonly mention_rate: number | null;
/** Answer engine that ran the prompt, for example 'claude-web-search', 'openai-web-search' or 'exa-answer'. */
engine: string;
/** Checks of the prompt on this engine, failed ones included. */
checks: number;
/** Checks whose answer cited a target domain. */
cited: number;
+ /** Checks whose answer names the brand (the registrable name of a target domain, case ignored). A cited check always counts as mentioned. */
+ mentioned: number;
/** Checks where the engine did not answer. */
failed: number;
/** When the engine first ran the prompt. */
@@ -190,13 +233,28 @@ export namespace Schemas {
skipped: number;
}
+ export interface AEOPromptSearchWord {
+ /** A word from the web searches, lower-cased, common English words left out. */
+ word: string;
+ /** How many searches used the word, weighted by how often each search ran. */
+ count: number;
+ }
+
export interface AEOPromptStats {
/** Totals per answer engine over the prompt's whole history. */
engines: AEOPromptEngineStats[];
/** Counts per day and engine from 'daily_from' to 'daily_to' (30 days). Days and engines without a check are left out. */
daily: AEOPromptDailyEngineStats[];
+ /** One row per engine and per domain for the 10 most cited domains of the last 30 days: how many of the engine's answers cited the domain. Shows which sources the engines agree on. */
+ consensus: AEOPromptEngineDomainShare[];
+ /** Brand sentiment of the answers in the last 30 days, one row per engine. Only the sentences that name the brand are scored. */
+ sentiment: AEOPromptEngineSentiment[];
+ /** Most used words across the web searches the engines ran in the last 30 days, at most 20. Only engines that report their searches contribute. */
+ top_words: AEOPromptSearchWord[];
/** Checks of the prompt across all engines, failed ones included. */
total_checks: number;
+ /** Lower-cased brand names, derived from the project's target domains. An answer that contains one of them as a word counts as a mention. */
+ brand_names: string[];
/** First day of the daily series, in the project's timezone. */
daily_from: string;
/** Last day of the daily series (today), in the project's timezone. */
diff --git a/services/mcp/src/generated/aeo/api.ts b/services/mcp/src/generated/aeo/api.ts
index c9d1956d2fda..86c9457ff852 100644
--- a/services/mcp/src/generated/aeo/api.ts
+++ b/services/mcp/src/generated/aeo/api.ts
@@ -147,7 +147,7 @@ export const AeoPromptsChecksQueryParams = () => zod.object({
})
/**
- * Gets citation totals per answer engine for one prompt, plus per-day counts for the last 30 days in the project's timezone.
+ * Gets citation and brand-mention totals per answer engine for one prompt, plus, for the last 30 days in the project's timezone, per-day counts, which engines cite which domains, brand sentiment per engine, and the words of the web searches the engines ran.
* @summary Get an AEO prompt's citation stats
*/
export const AeoPromptsStatsParams = () => zod.object({