Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(cloro): read names instead of ChatGPT entity tags in answers (#1097)
Cihanb8a6e3212135
diff --git a/server/src/lib/cloro-scraper.js b/server/src/lib/cloro-scraper.jsindex 7f7699d39..9e6e1f739 100644--- a/server/src/lib/cloro-scraper.js+++ b/server/src/lib/cloro-scraper.js@@ -8,6 +8,7 @@*/import logger from './logger.js';+import { stripEntityMarkup } from './entity-markup.js';const CLORO_API = 'https://api.cloro.dev';@@ -141,6 +142,12 @@ function normalizeSearchQueries(result, scraperId) {return [];}+/**+ * Parse a completed Cloro task into the answer we store. The text loses+ * ChatGPT's entity tags here (entity-markup.js), so the saved answer and every+ * metric derived from it read the names instead. Only ChatGPT has been seen+ * to send them, but the cleanup is harmless on the other engines.+ */export function parseScraperResponse(result, scraperId) {if (scraperId === 'chatgpt-shopping') {// ChatGPT Shopping returns a normal ChatGPT response shape *plus*@@ -149,7 +156,7 @@ export function parseScraperResponse(result, scraperId) {// Model is forced to the shopping-specific id so Insights aggregates// (which exclude `platform = 'chatgpt-shopping'`) never mix this row// into brand-level visibility numbers.- const text = result.markdown || result.text || '';+ const text = stripEntityMarkup(result.markdown || result.text || '');const citations = (result.sources || []).map((src, idx) => ({url: src.url || '',title: src.label || '',@@ -171,7 +178,7 @@ export function parseScraperResponse(result, scraperId) {throw new Error('Google did not return an AI Overview for this query');}- const text = aio.markdown || aio.text || '';+ const text = stripEntityMarkup(aio.markdown || aio.text || '');const citations = (aio.sources || []).map((src, idx) => ({url: src.url || '',title: src.label || '',@@ -184,7 +191,7 @@ export function parseScraperResponse(result, scraperId) {if (scraperId === 'google-aimode') {const aiMode = result.result || result;- const text = aiMode.markdown || aiMode.text || '';+ const text = stripEntityMarkup(aiMode.markdown || aiMode.text || '');const citations = (aiMode.sources || []).map((src, idx) => ({url: src.url || '',title: src.label || '',@@ -204,7 +211,7 @@ export function parseScraperResponse(result, scraperId) {};}- const text = result.markdown || result.text || '';+ const text = stripEntityMarkup(result.markdown || result.text || '');const model = result.model || scraperId;const citations = (result.sources || []).map((src, idx) => ({url: src.url || '',diff --git a/server/src/lib/cloro-scraper.test.js b/server/src/lib/cloro-scraper.test.jsindex b10b19780..ef04f2f44 100644--- a/server/src/lib/cloro-scraper.test.js+++ b/server/src/lib/cloro-scraper.test.js@@ -170,6 +170,21 @@ describe('cloro-scraper – parseScraperResponse', () => {});});+ describe('ChatGPT entity markup', () => {+ const markdown =+ 'Top tools: **<Entity category="software" value="Tesseract" disambig="Adlift AI search optimization platform"/>** and <Entity category="software" value="Ansvisor" />.';++ it('stores the names instead of the entity tags', () => {+ const parsed = parseScraperResponse({ markdown, sources: [] }, 'chatgpt-web');+ expect(parsed.text).toBe('Top tools: **Tesseract** and Ansvisor.');+ });++ it('cleans the shopping answer too', () => {+ const parsed = parseScraperResponse({ markdown, sources: [] }, 'chatgpt-shopping');+ expect(parsed.text).toBe('Top tools: **Tesseract** and Ansvisor.');+ });+ });+describe('search_queries (observed query fan-out, #332)', () => {it('maps Copilot string[] searchQueries to rich items tagged with the source platform', () => {const result = {diff --git a/server/src/lib/entity-markup.js b/server/src/lib/entity-markup.jsnew file mode 100644index 000000000..6c34df9e9--- /dev/null+++ b/server/src/lib/entity-markup.js@@ -0,0 +1,53 @@+/**+ * ChatGPT's entity markup in answer text.+ *+ * ChatGPT tags the products, companies and places it names, and the markdown+ * Cloro returns for it can keep the tag where the name should be:+ * `<Entity category="software" value="Tesseract" disambig="Adlift AI search+ * optimization platform"/>` for "Tesseract". The tag showed as literal text on+ * the answer, and its attributes counted as mentions: a competitor named only+ * in `disambig` (or one called "Software") was counted as mentioned, and the+ * extra names shifted every mention position.+ *+ * New answers are cleaned before they're saved. The response parser and the+ * web app clean again, so answers stored before that read and count properly+ * too. Mirrored by web/src/lib/entity-markup.ts — keep the two in sync.+ */++/** A self-closing `<Entity …/>` tag; group 1 is its attribute list. */+const ENTITY_TAG = /<Entity((?:\s+[\w:-]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*\/>/g;++const ATTRIBUTE = /([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;++const NAMED_ESCAPES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };++/** Attribute values are XML-escaped ("AT&T"); undo the common escapes. */+function unescapeXml(value) {+ return value.replace(+ /&(?:(amp|lt|gt|quot|apos)|#(\d+)|#x([\da-f]+));/gi,+ (match, name, dec, hex) => {+ if (name) return NAMED_ESCAPES[name.toLowerCase()];+ const code = dec ? Number(dec) : parseInt(hex, 16);+ // NUL and lone surrogates would not survive a Postgres text column.+ const valid = code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff);+ return valid ? String.fromCodePoint(code) : match;+ },+ );+}++/**+ * Replaces each `<Entity … value="X" …/>` tag with X, the name the answer+ * reads. A tag without a `value` is left as it is.+ *+ * @param {string} text+ * @returns {string}+ */+export function stripEntityMarkup(text) {+ if (!text || !text.includes('<Entity')) return text;+ return text.replace(ENTITY_TAG, (tag, attributes) => {+ for (const [, name, doubleQuoted, singleQuoted] of attributes.matchAll(ATTRIBUTE)) {+ if (name === 'value') return unescapeXml(doubleQuoted ?? singleQuoted);+ }+ return tag;+ });+}diff --git a/server/src/lib/entity-markup.test.js b/server/src/lib/entity-markup.test.jsnew file mode 100644index 000000000..86851269b--- /dev/null+++ b/server/src/lib/entity-markup.test.js@@ -0,0 +1,48 @@+import { describe, it, expect } from 'vitest';+import { stripEntityMarkup } from './entity-markup.js';++describe('stripEntityMarkup', () => {+ it('replaces each entity tag with its value', () => {+ const text =+ '1. **<Entity category="software" value="Tesseract" disambig="Adlift AI search optimization platform"/>** — tracks answers.\n' ++ '2. **<Entity category="software" value="Ansvisor" />** — open source.';+ expect(stripEntityMarkup(text)).toBe(+ '1. **Tesseract** — tracks answers.\n2. **Ansvisor** — open source.',+ );+ });++ it('reads the value whatever the attribute order, quoting and spacing', () => {+ expect(stripEntityMarkup('<Entity value="Acme" category="company"/>')).toBe('Acme');+ expect(stripEntityMarkup("<Entity category='company' value='Acme'/>")).toBe('Acme');+ expect(stripEntityMarkup('<Entity\n category = "company"\n value = "Acme"\n/>')).toBe('Acme');+ });++ it('unescapes XML escapes in the value', () => {+ expect(stripEntityMarkup('<Entity category="company" value="AT&T"/>')).toBe('AT&T');+ expect(stripEntityMarkup('<Entity value=""Q" 'R'"/>')).toBe('"Q" \'R\'');+ });++ it('keeps escapes for characters a text column cannot hold', () => {+ expect(stripEntityMarkup('<Entity value="a�b�c"/>')).toBe('a�b�c');+ });++ it('keeps an attribute containing ">" or "/>" inside its value', () => {+ expect(stripEntityMarkup('<Entity value="A/B" disambig="x > y />"/> ok')).toBe('A/B ok');+ });++ it('leaves a tag without a value as it is', () => {+ const text = 'See <Entity category="software"/> here.';+ expect(stripEntityMarkup(text)).toBe(text);+ });++ it('leaves other tags and look-alikes alone', () => {+ const text = '<entity class="User"/> <EntityList value="x"/> <Entity value="open">';+ expect(stripEntityMarkup(text)).toBe(text);+ });++ it('returns empty and plain text unchanged', () => {+ expect(stripEntityMarkup('')).toBe('');+ expect(stripEntityMarkup(null)).toBe(null);+ expect(stripEntityMarkup('No tags here.')).toBe('No tags here.');+ });+});diff --git a/server/src/lib/response-parser.js b/server/src/lib/response-parser.jsindex 095b59648..6680cef01 100644--- a/server/src/lib/response-parser.js+++ b/server/src/lib/response-parser.js@@ -3,6 +3,8 @@* Mentions and citations are computed locally; sentiment comes from AI.*/+import { stripEntityMarkup } from './entity-markup.js';+/*** Strip markdown link URLs so only display text remains.* "[label](https://example.com/path)" → "label"@@ -14,6 +16,15 @@ function stripUrls(text) {return cleaned;}+/**+ * The answer as its reader sees it, for matching names: ChatGPT's entity tags+ * replaced by the names they stand for (their other attributes would count as+ * mentions), URLs stripped.+ */+function matchableText(text) {+ return stripUrls(stripEntityMarkup(text));+}+const WORD_CHAR = /[\p{L}\p{N}_]/u;// Scripts written without spaces between words (Chinese, Japanese, Thai…) and// Korean, whose particles attach to the noun: a name there is always flanked@@ -100,7 +111,7 @@ export function countOwnDomainCitations(citations, brandDomains) {* Used to short-circuit sentiment analysis when the brand isn't mentioned.*/export function countBrandMentions(text, brand) {- const cleanText = stripUrls(text);+ const cleanText = matchableText(text);let count = countOccurrences(cleanText, brand.brandName);for (const domain of brand.domains) {count += countOccurrences(cleanText, domain);@@ -153,7 +164,7 @@ function firstOccurrenceIndex(text, term) {* competitors the answer actually mentions.*/export function computeMentionPosition(text, brand, competitors = []) {- const cleanText = stripUrls(text);+ const cleanText = matchableText(text);const entityFirstIndex = (names) => {let best = -1;@@ -196,7 +207,7 @@ export function computeMentionPosition(text, brand, competitors = []) {*/export function parseResponse(response, brand, sentiment = 'neutral', competitors = []) {const { text, citations } = response;- const cleanText = stripUrls(text);+ const cleanText = matchableText(text);// --- Brand mention count (on URL-stripped text to avoid double-counting) ---let mentionCount = countOccurrences(cleanText, brand.brandName);diff --git a/server/src/lib/response-parser.test.js b/server/src/lib/response-parser.test.jsindex a37d1cbc4..43599bdee 100644--- a/server/src/lib/response-parser.test.js+++ b/server/src/lib/response-parser.test.js@@ -421,6 +421,46 @@ describe('response-parser', () => {expect(result.competitorMentions[0].mention_position).toBe(1);});});++ describe('ChatGPT entity markup', () => {+ const text = [+ '1. <Entity category="software" value="Tesseract" disambig="Adlift AI search optimization platform"/> — tracks answers.',+ '2. <Entity category="software" value="Ansvisor" /> — open source.',+ '3. Adlift — agency platform.',+ ].join('\n');+ const brand = { brandName: 'Ansvisor', domains: ['ansvisor.com'] };+ const competitors = [+ { id: 'tesseract', name: 'Tesseract', domain: null },+ { id: 'adlift', name: 'Adlift', domain: null },+ { id: 'software', name: 'Software', domain: null },+ ];++ it('counts only the names the answer reads, not the tag attributes', () => {+ const result = parseResponse({ text, citations: [] }, brand, 'neutral', competitors);+ const byId = new Map(result.competitorMentions.map((c) => [c.competitor_id, c]));+ expect(result.mentionCount).toBe(1);+ expect(byId.get('tesseract').mention_count).toBe(1);+ // Named in Tesseract's disambig and once in the answer itself.+ expect(byId.get('adlift').mention_count).toBe(1);+ // Only ever the tag's category.+ expect(byId.get('software').mention_count).toBe(0);+ });++ it('ranks entities by where the answer names them', () => {+ const result = computeMentionPosition(text, brand, competitors);+ expect(result.mentionPosition).toBe(2);+ expect(result.mentionedEntityCount).toBe(3);+ expect(result.competitorPositions.get('tesseract')).toBe(1);+ expect(result.competitorPositions.get('adlift')).toBe(3);+ expect(result.competitorPositions.has('software')).toBe(false);+ });++ it('counts a brand named only inside a tag', () => {+ expect(countBrandMentions('Try <Entity category="software" value="Ansvisor"/>.', brand)).toBe(+ 1,+ );+ });+ });});describe('brandMentionRanges', () => {diff --git a/web/src/components/results/response-detail.tsx b/web/src/components/results/response-detail.tsxindex f92d81307..d0c5237a3 100644--- a/web/src/components/results/response-detail.tsx+++ b/web/src/components/results/response-detail.tsx@@ -8,6 +8,7 @@ import { Badge } from '@/components/ui/badge';import { Button } from '@/components/ui/button';import { Card, CardContent } from '@/components/ui/card';import { cn } from '@/lib/utils';+import { stripEntityMarkup } from '@/lib/entity-markup';import { formatSearchQuerySource, visibleSearchQueries } from './query-fanout';/** Past this length the compact view folds the answer behind "Show full response". */@@ -29,8 +30,9 @@ export function ResponseDetail({variant: 'page' | 'compact';}) {const compact = variant === 'compact';+ const response = stripEntityMarkup(result.response);const searchQueries = visibleSearchQueries(result.searchQueries);- const foldable = compact && result.response.length > FOLD_AT_CHARS;+ const foldable = compact && response.length > FOLD_AT_CHARS;const [unfolded, setUnfolded] = useState(false);const folded = foldable && !unfolded;@@ -39,7 +41,7 @@ export function ResponseDetail({<Section title="AI Response" compact={compact}><div className={cn('relative', folded && 'max-h-72 overflow-hidden')}><div className="prose prose-sm dark:prose-invert max-w-none">- <Markdown>{result.response}</Markdown>+ <Markdown>{response}</Markdown></div>{folded && (<div className="pointer-events-none absolute inset-x-0 bottom-0 h-16 bg-gradient-to-t from-background to-transparent" />diff --git a/web/src/lib/actions/reports.ts b/web/src/lib/actions/reports.tsindex f8a53d62b..69a646409 100644--- a/web/src/lib/actions/reports.ts+++ b/web/src/lib/actions/reports.ts@@ -20,6 +20,7 @@ import {import { getCitationsOverview, type CitationsSourceBreakdown } from '@/lib/actions/citations';import { getShoppingKpis } from '@/lib/actions/shopping';import { type SourceCategory } from '@/lib/citations/classify';+import { stripEntityMarkup } from '@/lib/entity-markup';import {getReportTemplate,ALL_REPORT_SECTIONS,@@ -624,9 +625,9 @@ const REPORT_EVIDENCE_COUNT = 10;/** How many sourcing prompts each cited URL lists. */const EVIDENCE_PROMPTS_PER_URL = 3;-/** Light markdown/URL strip so excerpts read as prose. */+/** Light markdown/URL/entity-tag strip so excerpts read as prose. */function stripMarkdown(text: string): string {- return text+ return stripEntityMarkup(text).replace(/\[([^\]]*)\]\([^)]+\)/g, '$1').replace(/https?:\/\/[^\s)>\]]+/g, '').replace(/[*_#`]/g, '')diff --git a/web/src/lib/entity-markup.test.ts b/web/src/lib/entity-markup.test.tsnew file mode 100644index 000000000..6c4f484b5--- /dev/null+++ b/web/src/lib/entity-markup.test.ts@@ -0,0 +1,48 @@+import { describe, expect, it } from 'vitest';++import { stripEntityMarkup } from './entity-markup';++describe('stripEntityMarkup', () => {+ it('replaces each entity tag with its value', () => {+ const text =+ '1. **<Entity category="software" value="Tesseract" disambig="Adlift AI search optimization platform"/>** — tracks answers.\n' ++ '2. **<Entity category="software" value="Ansvisor" />** — open source.';+ expect(stripEntityMarkup(text)).toBe(+ '1. **Tesseract** — tracks answers.\n2. **Ansvisor** — open source.',+ );+ });++ it('reads the value whatever the attribute order, quoting and spacing', () => {+ expect(stripEntityMarkup('<Entity value="Acme" category="company"/>')).toBe('Acme');+ expect(stripEntityMarkup("<Entity category='company' value='Acme'/>")).toBe('Acme');+ expect(stripEntityMarkup('<Entity\n category = "company"\n value = "Acme"\n/>')).toBe('Acme');+ });++ it('unescapes XML escapes in the value', () => {+ expect(stripEntityMarkup('<Entity category="company" value="AT&T"/>')).toBe('AT&T');+ expect(stripEntityMarkup('<Entity value=""Q" 'R'"/>')).toBe('"Q" \'R\'');+ });++ it('keeps escapes for characters a text column cannot hold', () => {+ expect(stripEntityMarkup('<Entity value="a�b�c"/>')).toBe('a�b�c');+ });++ it('keeps an attribute containing ">" or "/>" inside its value', () => {+ expect(stripEntityMarkup('<Entity value="A/B" disambig="x > y />"/> ok')).toBe('A/B ok');+ });++ it('leaves a tag without a value as it is', () => {+ const text = 'See <Entity category="software"/> here.';+ expect(stripEntityMarkup(text)).toBe(text);+ });++ it('leaves other tags and look-alikes alone', () => {+ const text = '<entity class="User"/> <EntityList value="x"/> <Entity value="open">';+ expect(stripEntityMarkup(text)).toBe(text);+ });++ it('returns empty and plain text unchanged', () => {+ expect(stripEntityMarkup('')).toBe('');+ expect(stripEntityMarkup('No tags here.')).toBe('No tags here.');+ });+});diff --git a/web/src/lib/entity-markup.ts b/web/src/lib/entity-markup.tsnew file mode 100644index 000000000..b9d243f75--- /dev/null+++ b/web/src/lib/entity-markup.ts@@ -0,0 +1,50 @@+/**+ * ChatGPT's entity markup in stored answer text.+ *+ * ChatGPT answers could be saved with a tag where a name should be:+ * `<Entity category="software" value="Tesseract" disambig="…"/>` for+ * "Tesseract". The answer view showed the tag as literal text. New answers are+ * cleaned before they're saved; this cleans the ones stored before that when+ * they're read. Mirrors server/src/lib/entity-markup.js — keep the two in sync.+ */++/** A self-closing `<Entity …/>` tag; group 1 is its attribute list. */+const ENTITY_TAG = /<Entity((?:\s+[\w:-]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*\/>/g;++const ATTRIBUTE = /([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;++const NAMED_ESCAPES: Record<string, string> = {+ amp: '&',+ lt: '<',+ gt: '>',+ quot: '"',+ apos: "'",+};++/** Attribute values are XML-escaped ("AT&T"); undo the common escapes. */+function unescapeXml(value: string): string {+ return value.replace(+ /&(?:(amp|lt|gt|quot|apos)|#(\d+)|#x([\da-f]+));/gi,+ (match, name?: string, dec?: string, hex?: string) => {+ if (name) return NAMED_ESCAPES[name.toLowerCase()];+ const code = dec ? Number(dec) : parseInt(hex ?? '', 16);+ // NUL and lone surrogates would not survive a Postgres text column.+ const valid = code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff);+ return valid ? String.fromCodePoint(code) : match;+ },+ );+}++/**+ * Replaces each `<Entity … value="X" …/>` tag with X, the name the answer+ * reads. A tag without a `value` is left as it is.+ */+export function stripEntityMarkup(text: string): string {+ if (!text || !text.includes('<Entity')) return text;+ return text.replace(ENTITY_TAG, (tag, attributes: string) => {+ for (const [, name, doubleQuoted, singleQuoted] of attributes.matchAll(ATTRIBUTE)) {+ if (name === 'value') return unescapeXml(doubleQuoted ?? singleQuoted);+ }+ return tag;+ });+}