Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(content): read no tool's report, report acronym or maker's German as a source, read the brand once for a send, and read long input in linear time (#1544 review follow-ups) (#1563)
Cihan7605ce97df3e
diff --git a/server/src/lib/content-agent.js b/server/src/lib/content-agent.jsindex 78620c19a..901938cca 100644--- a/server/src/lib/content-agent.js+++ b/server/src/lib/content-agent.js@@ -44,6 +44,7 @@ import {todayText,} from './content-sources.js';import { draftLanguage } from './content-language.js';+import { editSpans } from './edit-spans.js';import { getLanguageName } from './languages.js';import { PlanLimitError, reserveAiBudget } from './plan-guard.js';@@ -374,50 +375,92 @@ const TR_SENTENCE_WORDS = new Set('ancak ama fakat ayrıca bu şu bir ve ile son yeni özellikle'.split(' '),);const LEAD = String.raw`(?:(?:the|a|an)\s+)?(?:\d{4}\s+)?`;+/**+ * A pattern that starts with a name, run only on a text with its trigger+ * word: tried at each capital, it would read a long line of capitalised+ * words in time adding up across the patterns (#1544 review).+ */+const when = (trigger, re) => Object.assign(re, { trigger });+/** Up to three small words before a report's noun, none a product's or the reader's ("this", "your"). */+const SMALL_WORDS = String.raw`(?:(?!(?:your|this|our|each|these|those|that|my|their)\b)[\p{Ll}\p{N}]+\s+){0,3}`;+/** "report by Gartner", "survey from a 2025 Statista": a source's report and its name. */+const REPORT_FROM = String.raw`(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s+(?:by|from)\s+${LEAD}${SOURCE_NAME}`;+/**+ * Words that speak to the reader ("you can see…"), or tell them what to do+ * in a tool ("filter by channel", "open the report"): a how-to, not a claim.+ * A verb only as one, before a small word: never a noun's ("open rates",+ * "use cases", "add-on sales", "download numbers").+ */+const TO_READER = String.raw`(?:(?:you|your|yours|you['’](?:re|ll|ve|d))\b|(?:open|click|select|filter|go|choose|use|look|see|check|find|navigate|scroll|set|add|export|download|sort|compare)(?![\p{L}'’-])\s+(?:the|a|an|by|to|on|in|at|for|your|this|it|out)\b)`;+/** Kinds of report an acronym names, never a source ("SEO raporuna göre", "PDF raporuna göre"). */+const TR_REPORT_KINDS = String.raw`(?!(?:SEO|PDF|KPI|CRM|ERP)(?![\p{L}\p{N}]))`;const ATTRIBUTIONS = [new RegExp(String.raw`\b[Aa]ccording to\s+${LEAD}${SOURCE_NAME}`, 'gu'),new RegExp(String.raw`\b[Aa]s\s+(?:reported|noted|stated|found)\s+by\s+${LEAD}${SOURCE_NAME}`,'gu',),- new RegExp(String.raw`${SOURCE_NAME}\s+(?:says|said)\b`, 'gu'),+ when(/\b(?:says|said)\b/, new RegExp(String.raw`${SOURCE_NAME}\s+(?:says|said)\b`, 'gu')),// "Claims" and "states" are nouns too ("Warranty claims are…", "all 50// US states"): those, as "reports" and the rest, only before "that".- new RegExp(- String.raw`${SOURCE_NAME}\s+(?:reports|reported|notes|noted|finds|found|estimates|estimated)\s+that\b`,- 'gu',+ when(+ /\b(?:reports|reported|notes|noted|finds|found|estimates|estimated)\s+that\b/,+ new RegExp(+ String.raw`${SOURCE_NAME}\s+(?:reports|reported|notes|noted|finds|found|estimates|estimated)\s+that\b`,+ 'gu',+ ),),// …and never a run of names counted ("all 50 US states that allow…", "most- // US states that", "Most US states that", "the 48 contiguous US states- // that"), or one read from the middle of a name ("New England states+ // US states that", "Most US states that", "In most of the US states that",+ // "the 48 contiguous US states that"), or one read from the middle of a name ("New England states// that"); small words between only after a number or "all", so "In some// cases Gartner states that" is read. A capital first, as below: a// lookbehind ending in a run of spaces isn't tried at each of them.- new RegExp(- String.raw`(?=\p{Lu})(?<!(?:\b(?:[Aa]ll|\d{1,3})(?:\s+\p{Ll}+){0,2}|\b(?:[Mm]ost|[Mm]any|[Ss]ome|[Ss]everal|[Bb]oth|[Ee]very)|\p{Lu}[\p{L}\p{N}&.'’-]*)\s+)(?!(?:All|Most|Many|Some|Several|Both|Every)\b)${SOURCE_NAME}\s+(?:claims|claimed|states|stated)\s+that\b`,- 'gu',+ when(+ /\b(?:claims|claimed|states|stated)\s+that\b/,+ new RegExp(+ String.raw`(?=\p{Lu})(?<!(?:\b(?:[Aa]ll|\d{1,3})(?:\s+\p{Ll}+){0,2}|\b(?:[Mm]ost|[Mm]any|[Ss]ome|[Ss]everal|[Bb]oth|[Ee]very)(?:\s+of\s+the)?|\p{Lu}[\p{L}\p{N}&.'’-]*)\s+)(?!(?:All|Most|Many|Some|Several|Both|Every)\b)${SOURCE_NAME}\s+(?:claims|claimed|states|stated)\s+that\b`,+ 'gu',+ ),+ ),+ // "shows" only before "that" or a figure: a tool's report shows things+ // too ("The Search Console report shows clicks", "GA4's Acquisition report+ // shows where…").+ when(+ /\b(?:report|study|survey|research|data|analysis)\s+(?:shows|showed|finds|found|says|said|suggests)\b/,+ new RegExp(+ String.raw`${SOURCE_NAME}(?:['’]s)?\s+(?:report|study|survey|research|data|analysis)\s+(?:(?:shows|showed)(?=\s+(?:that\b|\p{N}))|(?:finds|found|says|said|suggests)\b)`,+ 'gu',+ ),),+ // Never the maker's own words, a manual or the page's own, as product+ // pages give them ("laut Hersteller", "laut Herstellerangaben", "laut+ // Angaben des Herstellers", "laut Datenblättern", "laut Anleitung"), in a+ // compound too ("laut Produktdatenblatt", "laut Packungsbeilage", "laut+ // Kundenbewertungen").new RegExp(- String.raw`${SOURCE_NAME}(?:['’]s)?\s+(?:report|study|survey|research|data|analysis)\s+(?:shows|showed|finds|found|says|said|suggests)\b`,+ String.raw`\b[Ll]aut\s+${LEAD}(?![\p{L}-]*?(?:[Hh]ersteller|[Dd]atenbl|[Aa]nleitung|[Bb]eschreibung|[Gg]esetz|[Aa]ngaben|[Bb]eilage|[Bb]ewertungen))${SOURCE_NAME}`,'gu',),- // Never the maker's own words or a manual, as product pages give them- // ("laut Hersteller", "laut Herstellerangaben", "laut Datenblättern",- // "laut Anleitung").+ // "nach Angaben von Gartner", "laut Angaben des Statistischen+ // Bundesamtes", never the maker's ("des Herstellers").new RegExp(- String.raw`\b[Ll]aut\s+${LEAD}(?!Hersteller|Datenbl|Anleitung|Bedienungsanleitung|Produktbeschreibung|Gesetz)${SOURCE_NAME}`,+ String.raw`\b(?:[Nn]ach|[Ll]aut)\s+Angaben\s+(?:von|des|der)\s+${LEAD}(?!Hersteller)${SOURCE_NAME}`,'gu',),- // "nach Angaben von Gartner", never the maker's ("des Herstellers").- new RegExp(String.raw`\b[Nn]ach\s+Angaben\s+von\s+${LEAD}(?!Hersteller)${SOURCE_NAME}`, 'gu'),- // "Gartner zufolge"; never a noun German capitalises: after an article- // ("Dem Bericht zufolge", "Einer Studie zufolge") or a plural ("Berichten",+ // "Gartner zufolge"; never a noun German capitalises: after an article,+ // with an adjective or not ("Dem Bericht zufolge", "Einer Studie zufolge",+ // "Einer neuen Studie zufolge"; a verb's ending is no adjective's: "Die+ // Ausgaben sollen Gartner zufolge steigen") or a plural ("Berichten",// "Herstellerangaben", "Insidern zufolge"), the cost a name like Nielsen// pays. A capital first: the lookbehind ends in a run of spaces, and tried// at each of a long run's, it would read it in time growing with its square.- new RegExp(- String.raw`(?=\p{Lu})(?<!\b${DE_DET}\s+)(?!${DE_DET}\b|Hersteller)${SOURCE_NAME}(?<!\p{Ll}(?:en|ern))\s+zufolge\b`,- 'gu',+ when(+ /\bzufolge\b/,+ new RegExp(+ String.raw`(?=\p{Lu})(?<!\b${DE_DET}(?:\s+\p{Ll}+(?:en|em|er|es))?\s+)(?!${DE_DET}\b|Hersteller)${SOURCE_NAME}(?<!\p{Ll}(?:en|ern))\s+zufolge\b`,+ 'gu',+ ),),// "As Zapier notes, …", "As Zapier puts it, …": the verb then a comma or// "that", never an event ("As Google adds AI Overviews to more queries").@@ -441,7 +484,15 @@ const ATTRIBUTIONS = [// lookbehind ending in a run of spaces, tried at each of them, would read// a long run in time growing with its square.new RegExp(- String.raw`(?=(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s)(?<=(?:\b[Aa]ccording\s+to|\b[Pp]er|(?:^|[.!?]\s+)In)\s+(?:(?!(?:your|this|our|each|these|those|that|my|their)\b)[\p{Ll}\p{N}]+\s+){0,3})(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s+(?:by|from)\s+${LEAD}${SOURCE_NAME}(?=\s*,)`,+ String.raw`(?=(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s)(?<=(?:\b[Aa]ccording\s+to|\b[Pp]er)\s+${SMALL_WORDS})${REPORT_FROM}(?=\s*,)`,+ 'gu',+ ),+ // A sentence's "In …," only when the reader isn't spoken to after the+ // comma: then it's a tool's report ("In the report from Google Search+ // Console, you can see…", "In a report from Google Analytics, filter by+ // channel").+ new RegExp(+ String.raw`(?=(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s)(?<=(?:^|[.!?]\s+)In\s+${SMALL_WORDS})${REPORT_FROM}(?=\s*,(?!\s*${TO_READER}))`,'gu',),// "data from Statista shows that …" or a figure, never a tool's ("Data@@ -459,16 +510,24 @@ const ATTRIBUTIONS = [// "Gartner'ın raporuna göre": a source's report, study, data or survey.// A year or a word before the noun ("Gartner'ın 2025 raporuna", "son// raporuna"), a name of more words than one ("Google Analytics").- new RegExp(String.raw`${TR_NAME}['’]\p{L}{1,3}\s+${TR_REPORT}`, 'gu'),+ when(/göre/, new RegExp(String.raw`${TR_NAME}['’]\p{L}{1,3}\s+${TR_REPORT}`, 'gu')),// Without its ending ("Gartner raporuna", "TÜİK verilerine"): a name in// the sentence, or one shaped as a name (TÜİK, GA4, HubSpot), never a// sentence's first noun ("Ürün açıklamasına göre", "Sipariş verilerine").// A capital first, so the lookbehind isn't tried in a run of spaces.- new RegExp(String.raw`(?=\p{Lu})(?<=[^\s.!?:]\s+)${TR_NAME}\s+${TR_REPORT}`, 'gu'),- new RegExp(String.raw`${TR_SHAPED}\s+${TR_REPORT}`, 'gu'),+ // A list's number isn't the sentence's ("1) Ürün açıklamasına göre").+ // Never a kind of report's acronym, here or below ("Aylık SEO raporuna göre").+ when(+ /göre/,+ new RegExp(+ String.raw`(?=\p{Lu})(?<=[^\s.!?:)]\s+)${TR_REPORT_KINDS}${TR_NAME}\s+${TR_REPORT}`,+ 'gu',+ ),+ ),+ when(/göre/, new RegExp(String.raw`${TR_REPORT_KINDS}${TR_SHAPED}\s+${TR_REPORT}`, 'gu')),// "Gartner'a göre", never a comparison ("ChatGPT'ye göre daha hızlı");// "en" after it is the source's claim ("Gartner'a göre en hızlı büyüyen").- new RegExp(String.raw`${TR_NAME}['’]\p{L}{1,3}\s+göre\b(?!\s+daha\b)`, 'gu'),+ when(/göre/, new RegExp(String.raw`${TR_NAME}['’]\p{L}{1,3}\s+göre\b(?!\s+daha\b)`, 'gu')),];/**@@ -510,8 +569,10 @@ export function attributableSources(pageHtml, numbered, authors = [], language =/** Each attribution a text writes: the name it gives a claim to, and where. */function attributionsIn(text, language = null) {const found = [];+ const s = String(text);for (const re of attributionsFor(language)) {- for (const match of String(text).matchAll(re)) {+ if (re.trigger && !re.trigger.test(s)) continue;+ for (const match of s.matchAll(re)) {const name = match.groups?.name ?? '';// A sentence's words alone ("Who says", "Son raporuna göre") name no one.if (name && nameStart(name) < name.length) {@@ -605,16 +666,21 @@ export function placeholderAttributions(text, known = null) {* wordings read ("As Zapier says") goes once.*/function attributionEdits(text, known) {- const edits = [];+ const { spans: edits, free, add } = editSpans();for (const { name, index } of attributionsIn(text, known?.language)) {if (known && knownName(name, known)) continue;const start = index + nameStart(name);- // The possessive stays: "According to [source]'s latest survey".- const end = index + name.replace(/['’]\p{Ll}{1,3}$/u, '').length;- if (start >= end || edits.some((e) => start < e.end && end > e.start)) continue;- edits.push({ start, end, text: '[source]' });+ // The possessive stays when a word follows it ("According to [source]'s+ // latest survey"); a name that is one goes whole ("According to+ // McDonald's, sales rose": "[source], sales rose").+ const owner = /^\s+[\p{L}\p{N}]/u.test(+ text.slice(index + name.length, index + name.length + 3),+ );+ const end = index + (owner ? name.replace(/['’]\p{Ll}{1,3}$/u, '') : name).length;+ if (start >= end || !free(start, end)) continue;+ add({ start, end, text: '[source]' });}- return edits.sort((a, b) => a.start - b.start);+ return edits;}/**diff --git a/server/src/lib/content-agent.test.js b/server/src/lib/content-agent.test.jsindex fd8c5c09e..83bd937cf 100644--- a/server/src/lib/content-agent.test.js+++ b/server/src/lib/content-agent.test.js@@ -390,7 +390,15 @@ describe('keepSuggestions', () => {// More ways to give a claim to a source.expect(addsAttribution('', 'According to the Zapier report, teams forecast.')).toBe(true);expect(addsAttribution('', 'According to a 2025 Gartner survey, teams forecast.')).toBe(true);- expect(addsAttribution('', "Gartner's research shows teams forecast.")).toBe(true);+ expect(addsAttribution('', "Gartner's research shows that teams forecast.")).toBe(true);+ // A tool's report shows things too: "shows" only before "that" or a figure (#1544 review).+ expect(addsAttribution('', 'The Search Console report shows clicks and impressions.')).toBe(+ false,+ );+ expect(addsAttribution('', "GA4's Acquisition report shows where visitors come from.")).toBe(+ false,+ );+ expect(addsAttribution('', 'The Gartner report shows 40% of teams forecast.')).toBe(true);});// A replace of the whole passage: its words before and after.@@ -907,7 +915,7 @@ describe('blocks, names and languages as the review read them (#1520)', () => {htmlLines('<h2>Forecasting Basics</h2><p>Per <a href="#">Gartner</a>’s survey, most do.</p>'),).toEqual(['Forecasting Basics', 'Per Gartner’s survey, most do.']);const cites = attributableSources(- '<p><a href="https://gartner.example">Gartner</a>’s report shows teams forecast.</p>',+ '<p><a href="https://gartner.example">Gartner</a>’s report shows that teams forecast.</p>',[],[],);@@ -1005,8 +1013,8 @@ describe('attributions as the review of #1537 read them', () => {expect(placeholderAttributions("According to Gartner's latest survey, most do.", none)).toBe("According to [source]'s latest survey, most do.",);- expect(placeholderAttributions('Gartner’s research shows teams forecast.', none)).toBe(- '[source]’s research shows teams forecast.',+ expect(placeholderAttributions('Gartner’s research shows that teams forecast.', none)).toBe(+ '[source]’s research shows that teams forecast.',);});@@ -1196,6 +1204,128 @@ describe('no invented people or quotes (CE-044)', () => {});});+describe('attributions as the review of #1544 read them', () => {+ const none = knownPeople('');+ const reads = (texts, expected) => {+ for (const text of texts) expect(addsAttribution('', text), text).toBe(expected);+ };++ it('reads no tool’s report when the reader is spoken to after the comma', () => {+ reads(+ [+ 'In the report from Google Search Console, you can see clicks.',+ 'In a report from Google Analytics, filter by channel.',+ 'In a report from Google Analytics, your top pages are listed.',+ ],+ false,+ );+ // A claim's "In …," still is, and "according to" whoever it speaks to;+ // a verb's word that is a noun there is no instruction.+ reads(+ [+ 'In a 2025 report by Gartner, most teams forecast weekly.',+ 'According to a survey from Statista, your customers want speed.',+ 'In a 2025 study from Stanford, open rates fell 8%.',+ 'In a report by the WTO, export volumes rose 3%.',+ 'In a report by Gartner, use cases for AI tripled.',+ 'In a report by Gartner, go-to-market costs rose 30%.',+ 'In a report by Gartner, add-on sales grew 12%.',+ 'In a report by Gartner, download numbers doubled.',+ ],+ true,+ );+ reads(['In a report from Google Analytics, open the Acquisition tab.'], false);+ });++ it('reads no report’s acronym, or a list’s first noun, as a Turkish source', () => {+ reads(+ [+ 'SEO raporuna göre sıralama yükseldi.',+ 'PDF raporuna göre teslimat iki gün.',+ 'Aylık SEO raporuna göre sıralama yükseldi.',+ 'Bu PDF raporuna göre teslimat iki gün.',+ 'KPI raporuna göre satışlar arttı.',+ '1) Ürün açıklamasına göre ölçüler değişti.',+ ],+ false,+ );+ reads(+ ['TÜİK raporuna göre enflasyon düştü.', 'Bu yıl Gartner raporuna göre pazar büyüyor.'],+ true,+ );+ });++ it('reads no maker’s or page’s German words, in a compound or after an adjective', () => {+ reads(+ [+ 'Laut Angaben des Herstellers hält der Akku zehn Stunden.',+ 'Laut Produktdatenblatt wiegt es zwei Kilo.',+ 'Laut Packungsbeilage zweimal täglich.',+ 'Laut Kundenbewertungen ist er leise.',+ 'Einer neuen Studie zufolge wächst der Markt.',+ 'Einer repräsentativen Umfrage zufolge wächst der Markt.',+ ],+ false,+ );+ reads(+ [+ 'Laut Gartner wächst der Markt.',+ 'Gartner zufolge wächst der Markt.',+ // The giver of the figures, by "von", "des" or "der".+ 'Laut Angaben von Statista wuchs der Markt.',+ 'Laut Angaben des Statistischen Bundesamtes stiegen die Preise.',+ 'Laut Bewertungsportal Trustpilot ist er beliebt.',+ // A verb's ending before the name is no adjective's.+ 'Die Ausgaben sollen Gartner zufolge steigen.',+ 'Die Kosten liegen Gartner zufolge höher.',+ 'Weltweit nutzen Statista zufolge viele Teams KI.',+ 'Die Preise werden IDC zufolge steigen.',+ 'Das gilt immer Gartner zufolge.',+ ],+ true,+ );+ });++ it('reads no count of names ("most of the US states") as a source', () => {+ reads(['In most of the US states that allow it, the fee is lower.'], false);+ reads(['Gartner states that most teams forecast.'], true);+ });++ it('puts thousands of sources on one line, and skips what a line has no trigger for, quickly', () => {+ const time = (read) => {+ const started = Date.now();+ read();+ return Date.now() - started;+ };+ // A character past Latin-1 (Ü) in each: V8 skips no position then.+ // 20,000 sources rebuilt per edit took 3 s (main, 2026-10-26); read once, under 100 ms.+ const line = 'According to Gartner, most do. '.repeat(20_000);+ placeholderAttributionsInHtml('<p>According to Gartner, most do.</p>');+ let html;+ expect(+ time(() => (html = placeholderAttributionsInHtml(`<p>Ürün <b>x</b> ${line}</p>`))),+ ).toBeLessThan(400);+ expect(html).toBe(`<p>Ürün <b>x</b> ${'According to [source], most do. '.repeat(20_000)}</p>`);+ // A long line of capitalised words, none of the name-first patterns' words+ // in it: they aren't tried at each capital (160 ms on main, 4 ms now).+ const caps = `Ürün ${'Acme Beta Corp Delta '.repeat(20_000)}.`;+ attributedNames('Ürün Acme.');+ expect(time(() => attributedNames(caps))).toBeLessThan(60);+ });++ it('puts a name that is itself possessive whole, and keeps an owner’s ending', () => {+ expect(placeholderAttributions("According to McDonald's, sales rose.", none)).toBe(+ 'According to [source], sales rose.',+ );+ expect(placeholderAttributions("According to Gartner's latest survey, most do.", none)).toBe(+ "According to [source]'s latest survey, most do.",+ );+ expect(placeholderAttributions("According to Gartner's 2025 survey, most do.", none)).toBe(+ "According to [source]'s 2025 survey, most do.",+ );+ });+});+describe('a replace on a heading', () => {const page = draftPassages('<h2>What is LLM SEO</h2><p>It is new.</p><h2>FAQ</h2><p>Ask us.</p><p>Use good coffee.</p>',diff --git a/server/src/lib/content-delivery.js b/server/src/lib/content-delivery.jsindex b103a7b7a..71a81ec54 100644--- a/server/src/lib/content-delivery.js+++ b/server/src/lib/content-delivery.js@@ -262,7 +262,11 @@ async function webflowFor(brandId) {async function brandFor(brandId) {const [{ data: brand }, { data: domains }] = await Promise.all([- supabaseAdmin.from('brands').select('id, name, industry, language').eq('id', brandId).single(),+ supabaseAdmin+ .from('brands')+ .select('id, name, industry, language, site_type, shopping_mode_enabled')+ .eq('id', brandId)+ .single(),supabaseAdmin.from('brand_domains').select('domain')@@ -276,6 +280,8 @@ async function brandFor(brandId) {domain: domains?.[0]?.domain ?? '',industry: brand?.industry ?? '',language: brand?.language ?? null,+ // Its kind of site, as the editor keys My rules by it; null when unread.+ siteType: brand ? brandSiteType(brand) : null,};}@@ -439,21 +445,17 @@ const untouched = untouchedRules;/*** The workspace's My rules (00214) for these drafts' types and the brand's* kind of site (00227 keys a set by both, as the editor reads it), by type;- * none when unread.+ * none when unread, the brand's kind of site too (brandFor): never another+ * kind's set.*/-async function myRuleSets(orgId, brandId, drafts) {+async function myRuleSets(orgId, siteType, drafts) {const types = [...new Set(drafts.map((d) => d.rules?.set ?? d.page_type).filter(Boolean))];- if (!types.length) return new Map();- const { data: brand } = await supabaseAdmin- .from('brands')- .select('site_type, industry, shopping_mode_enabled')- .eq('id', brandId)- .single();+ if (!types.length || !siteType) return new Map();const { data, error } = await supabaseAdmin.from('content_rule_sets').select('page_type, overrides').eq('organization_id', orgId)- .eq('site_type', brandSiteType(brand))+ .eq('site_type', siteType).in('page_type', types);if (error) logger.warn({ err: error }, '[content-delivery] My rules not read');return new Map((data ?? []).map((set) => [set.page_type, set]));@@ -555,14 +557,13 @@ export async function sendContentPages({// rules for a draft no one has set rules on, as the editor reads them),// else the brand's.const rowOf = new Map((drafts ?? []).map((d) => [d.id, d]));- const [brand, mine] = await Promise.all([- brandFor(brandId),- myRuleSets(- orgId,- brandId,- going.map((item) => rowOf.get(item.page.draftId)).filter((d) => d && untouched(d.rules)),- ),- ]);+ // The brand once: its kind of site keys My rules.+ const brand = await brandFor(brandId);+ const mine = await myRuleSets(+ orgId,+ brand.siteType,+ going.map((item) => rowOf.get(item.page.draftId)).filter((d) => d && untouched(d.rules)),+ );for (const item of going) {const draft = rowOf.get(item.page.draftId) ?? {};const rules = untouched(draft.rules)diff --git a/server/src/lib/content-delivery.test.js b/server/src/lib/content-delivery.test.jsindex 6af289115..8faecf5d7 100644--- a/server/src/lib/content-delivery.test.js+++ b/server/src/lib/content-delivery.test.js@@ -49,7 +49,10 @@ function from(table) {}if (table === 'webhook_configs') return { data: db.configs, error: null };if (table === 'brand_cms_connections') return { data: db.webflow, error: null };- if (table === 'brands') return { data: db.brand, error: null };+ if (table === 'brands') {+ db.brandReads += 1;+ return { data: db.brand, error: null };+ }if (table === 'brand_domains') return { data: [{ domain: 'acme.example' }], error: null };return { data: null, error: null };};@@ -123,6 +126,7 @@ const draft = (id, over = {}) => ({beforeEach(() => {db.brand = { id: 'b1', name: 'Acme Coffee', industry: 'Coffee', language: null };+ db.brandReads = 0;db.drafts = [draft(page().draftId)];db.ruleSets = [];db.told = null;@@ -337,11 +341,24 @@ describe('sendContentPages', () => {];await send();expect(JSON.parse(postJson.mock.calls[0][1]).page.language).toBe('fr');+ // The brand read once, for the package and its kind of site (#1544 review).+ expect(db.brandReads).toBe(1);++ // A shop (Shopping mode on): the e-commerce set's.+ db.brand = { ...db.brand, shopping_mode_enabled: true };+ await send();+ expect(JSON.parse(postJson.mock.calls[1][1]).page.language).toBe('de');++ // The brand unread: no kind of site's set, never the B2B one.+ db.brand = null;+ await send();+ expect(JSON.parse(postJson.mock.calls[2][1]).page.language).toBe('en');// The draft's own choices win once it has any.+ db.brand = { id: 'b1', name: 'Acme Coffee', industry: 'Coffee', language: null };db.drafts[0].rules = { source: 'default', overrides: {}, textRules: [] };await send();- expect(JSON.parse(postJson.mock.calls[1][1]).page.language).toBe('en');+ expect(JSON.parse(postJson.mock.calls[3][1]).page.language).toBe('en');});it('takes JSON-LD however its script type is written, as the clean keeps it', async () => {diff --git a/server/src/lib/content-people.js b/server/src/lib/content-people.jsindex 5129550f0..fb7e0e496 100644--- a/server/src/lib/content-people.js+++ b/server/src/lib/content-people.js@@ -1,4 +1,5 @@import * as cheerio from 'cheerio';+import { editSpans } from './edit-spans.js';import { logger } from './logger.js';/**@@ -55,14 +56,18 @@ export const isKnown = (words, known) => {const wordCount = (text) => (String(text).match(/\p{L}[\p{L}\p{N}'’-]*/gu) ?? []).length;// A capitalised word of a name ("Ada", "McDonald", "O'Neil", "Jean-Luc"): one-// way to read it only (the lookahead checks it has a small letter), so a-// long word can't make the reading slow; an apostrophe only before a-// capital ("O'Neil", not "Yılmaz'a" or "Doe's"). Initials ("J. K.-// Rowling"), and particles between the words ("Ursula von der Leyen").-const WORD = String.raw`\p{Lu}(?=[\p{L}'’]*\p{Ll})(?:\p{L}|['’](?=\p{Lu}))+(?:-\p{L}+)?`;+// way to read it only (the lookahead checks it has a small letter, reading+// only what the word can hold), so a long word can't make the reading slow;+// an apostrophe only before a capital ("O'Neil", not "Yılmaz'a" or+// "Doe's"). Initials ("J. K. Rowling"), and particles between the words+// ("Ursula von der Leyen").+const WORD = String.raw`\p{Lu}(?=(?:\p{L}|['’](?=\p{Lu}))*?\p{Ll})(?:\p{L}|['’](?=\p{Lu}))+(?:-\p{L}+)?`;const PARTICLE = String.raw`(?:von|van|der|den|de|del|della|di|da|du|la|le|al|bin|ibn|ten|ter)`;const STARTS_PARTICLE = new RegExp(`^${PARTICLE}$`);-export const NAME = String.raw`(?<!\p{L})(?:(?:\p{Lu}\.\s?){1,3}\s?${WORD}|${WORD}(?:\s+(?:${PARTICLE}\s+){0,3}(?:\p{Lu}\.\s+)?${WORD}){1,2})(?!\p{L})`;+// Never started inside a word, an apostrophe's either: a run of letters+// joined by apostrophes ("Ab’Ab’Ab’…") would be tried at each capital in it,+// and read in time growing with its square (#1544 review).+export const NAME = String.raw`(?<!\p{L}|\p{L}['’])(?:(?:\p{Lu}\.\s?){1,3}\s?${WORD}|${WORD}(?:\s+(?:${PARTICLE}\s+){0,3}(?:\p{Lu}\.\s+)?${WORD}){1,2})(?!\p{L})`;const ROLE_WORDS = ['ceo',@@ -479,8 +484,12 @@ const SPOKEN_AFTER = new RegExp();const SPOKEN_BEFORE = new RegExp(String.raw`${SPEECH}[,:]?\s*$`, 'u');-/** A placeholder the agent left, or one put in: never read again. */-const BRACKETS = /\[[^\]]*\]/g;+/**+ * A placeholder the agent left, or one put in: never read again. Never one+ * with another "[" in it, so an opening bracket with no closing one isn't+ * read to the text's end from each "[" after it.+ */+const BRACKETS = /\[[^[\]]*\]/g;/*** Whether guillemets enclose the words between two quotes («A» ve «B»,@@ -500,8 +509,7 @@ export function peopleEdits(text, known, { heading = false } = {}) {const s = String(text ?? '');// Placeholders aren't read: blanked out, with their places kept.const read = s.replace(BRACKETS, (m) => ' '.repeat(m.length));- const edits = [];- const free = (start, end) => edits.every((e) => end <= e.start || start >= e.end);+ const { spans: edits, free, add } = editSpans();for (const pattern of QUOTES) {for (const m of read.matchAll(pattern)) {@@ -516,7 +524,7 @@ export function peopleEdits(text, known, { heading = false } = {}) {SPOKEN_AFTER.test(read.slice(end, end + 80)) ||SPOKEN_BEFORE.test(read.slice(Math.max(0, m.index - 40), m.index));if ((words < MIN_QUOTE_WORDS && !spoken) || isKnown(inner, known)) continue;- if (free(m.index, end)) edits.push({ start: m.index, end, text: QUOTE_PLACEHOLDER });+ if (free(m.index, end)) add({ start: m.index, end, text: QUOTE_PLACEHOLDER });}}for (const cue of NAME_CUES) {@@ -527,10 +535,10 @@ export function peopleEdits(text, known, { heading = false } = {}) {if (!person) continue;const start = cue.honorific ? m.index : at + person.start;const end = at + person.start + person.name.length;- if (free(start, end)) edits.push({ start, end, text: NAME_PLACEHOLDER });+ if (free(start, end)) add({ start, end, text: NAME_PLACEHOLDER });}}- return edits.sort((a, b) => a.start - b.start);+ return edits;}/** A text with each unknown name and quote put as a placeholder, and the edits made. */@@ -647,18 +655,26 @@ export function editLines($, editsFor) {starts.push(at);at += n.data.length;}- // From the end, so the places before stay put. The placeholder goes in- // the node the edit starts in; the rest of its words are cut.- for (const e of [...edits].reverse()) {- nodes.forEach((n, i) => {- const from = starts[i];- const to = from + n.data.length;- if (to <= e.start || from >= e.end) return;- const head = n.data.slice(0, Math.max(0, e.start - from));- const tail = n.data.slice(Math.min(n.data.length, e.end - from));- n.data = head + (e.start >= from ? e.text : '') + tail;- });- }+ // Each node's text built once from its pieces between the edits (in+ // order and apart): the placeholder goes in the node the edit starts in;+ // the rest of its words are cut. Rebuilt per edit, a line with thousands+ // would take their count times its length (#1544 review).+ let first = 0;+ nodes.forEach((n, i) => {+ const from = starts[i];+ const to = from + n.data.length;+ while (first < edits.length && edits[first].end <= from) first++;+ let out = '';+ let at = 0;+ let touched = false;+ for (let j = first; j < edits.length && edits[j].start < to; j++) {+ const e = edits[j];+ touched = true;+ out += n.data.slice(at, Math.max(at, e.start - from)) + (e.start >= from ? e.text : '');+ at = Math.min(n.data.length, e.end - from);+ }+ if (touched) n.data = out + n.data.slice(at);+ });};// An inline tag's text, with any line inside it read on its own: a line// break or a block in a bold or a link ends the line there too (BREAK).diff --git a/server/src/lib/content-people.test.js b/server/src/lib/content-people.test.jsindex 0c53d7c26..e57176f39 100644--- a/server/src/lib/content-people.test.js+++ b/server/src/lib/content-people.test.js@@ -625,3 +625,50 @@ describe('authorWords', () => {expect(log.warn.mock.calls[0][0]).toMatchObject({ organizationId: 'o1' });});});++describe('long input read quickly (#1544 review)', () => {+ const none = knownPeople('');+ const time = (read) => {+ const started = Date.now();+ read();+ return Date.now() - started;+ };++ it('reads letters joined by apostrophes, and an opening bracket with no closing one', () => {+ // A character past Latin-1 (Ü, ’) in each: V8 skips no position then.+ const texts = (n) => [+ `Ürün ${'Ab’'.repeat(n)} said it.`,+ `Ürün ${'A’'.repeat(n)} said it.`,+ `Ürün ${'a’Aa’A'.repeat(Math.ceil(n / 2))} said it.`,+ `Ürün ${"''A".repeat(n)} said it.`,+ `Ürün ${'[a '.repeat(n)}Marco Rossi said it.`,+ ];+ // Compiled once, apart from the timing.+ texts(2).forEach((text) => guardPeople(text, none));+ for (const text of texts(20_000)) {+ expect(+ time(() => guardPeople(text, none)),+ text.slice(0, 12),+ ).toBeLessThan(150);+ }+ // A name after an opening quote mark is still read, and an apostrophe's own name.+ expect(guardPeople("'Jane Doe said it twice.", none).text).toBe(`'${NAME} said it twice.`);+ expect(guardPeople("Bob O'Neil said it twice.", none).text).toBe(`${NAME} said it twice.`);+ });++ it('puts thousands of names on one line in time near its length', () => {+ // 20,000 names: checked against every edit kept, or rebuilt per edit,+ // they took over a second (main, 2026-10-26); read once, about 30 ms.+ const line = 'Marco Rossi said it. '.repeat(20_000);+ guardPeople('Marco Rossi said it.', none);+ let out;+ expect(time(() => (out = guardPeople(`Ürün: ${line}`, none)))).toBeLessThan(150);+ expect(out.edits).toHaveLength(20_000);+ let html;+ expect(+ time(() => (html = guardPeopleInHtml(`<p>Ürün: <b>x</b> ${line}</p>`, none))),+ ).toBeLessThan(300);+ expect(html.count).toBe(20_000);+ expect(html.html).toBe(`<p>Ürün: <b>x</b> ${`${NAME} said it. `.repeat(20_000)}</p>`);+ });+});diff --git a/server/src/lib/content-sources.js b/server/src/lib/content-sources.jsindex aed7938a6..467c35143 100644--- a/server/src/lib/content-sources.js+++ b/server/src/lib/content-sources.js@@ -123,6 +123,8 @@ const CURRENCY = /[$€£¥₺]/;/** What makes a lone digit a claim: a percent, a times, a scale. */const UNIT_AFTER =/^\s?(?:%|×|[xX]\b|[kKMB]\b|bn\b|mn\b|[Mm]illions?\b|[Bb]illions?\b|[Tt]housands?\b|[Tt]rillions?\b)/;+/** UNIT_AFTER read at a place in the text, without slicing it off. */+const UNIT_AT = new RegExp(UNIT_AFTER.source.slice(1), 'y');/** A percent written before its number, as Turkish does: "%40", "yüzde 40". */const PERCENT_BEFORE = /(?:%\s?|(?<!\p{L})yüzde\s)$/iu;@@ -141,8 +143,12 @@ function unitOf(text, start, end) {if (unit === 'b' || unit === 'bn' || unit.startsWith('billion')) return 'billion';return 'trillion';}-/** A placeholder the agent leaves ("[figure, source, 2025]"): it holds no claim. */-const BRACKETS = /\[[^\]]*\]/g;+/**+ * A placeholder the agent leaves ("[figure, source, 2025]"): it holds no+ * claim. Never one with another "[" in it, so an opening bracket with no+ * closing one isn't read to the text's end from each "[" after it.+ */+const BRACKETS = /\[[^[\]]*\]/g;/*** A number's value as compared: thousands separators dropped ("1,299" and@@ -330,20 +336,30 @@ export function sourceSuggestion({ named, marks, pieces, numbered, known }) {/** A text with each number it adds put as a placeholder: for a page written whole, with no marks. */export function placeholderNumbers(text, known) {- const added = addedNumbers(text, known);- let out = String(text ?? '');- for (const n of added.reverse()) {- // The sign and unit go with it: "$40", "40%" and "%40" become the placeholder.+ const s = String(text ?? '');+ // The text built once from its pieces, left to right: rebuilt per number,+ // a line with thousands would take their count times its length (#1544 review).+ // The sign and unit go with it: "$40", "40%" and "%40" become the placeholder.+ const found = addedNumbers(s, known).map((n) => {let start = n.start;- let end = n.end;- if (CURRENCY.test(out[start - 1] ?? '')) start -= 1;- const percent = out.slice(Math.max(0, start - 2), start).match(/%\s?$/);+ if (CURRENCY.test(s[start - 1] ?? '')) start -= 1;+ const percent = s.slice(Math.max(0, start - 2), start).match(/%\s?$/);if (percent) start -= percent[0].length;- const unit = out.slice(end).match(UNIT_AFTER);- if (unit) end += unit[0].length;- out = `${out.slice(0, start)}${NUMBER_PLACEHOLDER}${out.slice(end)}`;- }- return out;+ return { start, end: n.end };+ });+ let out = '';+ let at = 0;+ found.forEach((n, i) => {+ let end = n.end;+ UNIT_AT.lastIndex = end;+ const unit = UNIT_AT.exec(s);+ // A unit that runs into the next number's sign is that one's ("40 %50").+ if (unit && end + unit[0].length <= (found[i + 1]?.start ?? Infinity)) end += unit[0].length;+ const start = Math.max(n.start, at);+ out += s.slice(at, start) + NUMBER_PLACEHOLDER;+ at = Math.max(at, end);+ });+ return out + s.slice(at);}/** Tags whose text a page's placeholders never touch: code and what isn't words. */diff --git a/server/src/lib/content-sources.test.js b/server/src/lib/content-sources.test.jsindex f2fb3b6ff..b2a2b8214 100644--- a/server/src/lib/content-sources.test.js+++ b/server/src/lib/content-sources.test.js@@ -291,3 +291,34 @@ describe('no number without a source', () => {expect([...knownNumbers(htmlText('<td>1:16</td><td>2026</td>'))]).toEqual(['1', '16', '2026']);});});++describe('long input read quickly (#1544 review)', () => {+ const known = knownNumbers('Trackers in 2026 cost $1,299 a year.');+ const time = (read) => {+ const started = Date.now();+ read();+ return Date.now() - started;+ };++ it('puts thousands of numbers on one line, and reads an opening bracket with no closing one, in time near its length', () => {+ // A character past Latin-1 (Ü) in each: V8 skips no position then.+ // 20,000 numbers: the text rebuilt per number took over a second (main,+ // 2026-10-26); built once, under 10 ms.+ const line = `Ürün: ${'Teams see 40% more. '.repeat(20_000)}`;+ placeholderNumbers('Teams see 40% more.', known);+ let out;+ expect(time(() => (out = placeholderNumbers(line, known)))).toBeLessThan(400);+ expect(out).toBe(`Ürün: ${`Teams see ${NUMBER_PLACEHOLDER} more. `.repeat(20_000)}`);+ const open = `Ürün ${'[a '.repeat(20_000)}40% more.`;+ addedNumbers(open.slice(0, 40), known);+ expect(time(() => addedNumbers(open, known))).toBeLessThan(150);+ expect(addedNumbers(open, known).map((n) => n.value)).toEqual(['40']);+ // A unit that runs into the next number's sign is that one's: the space stays.+ expect(placeholderNumbers('Teams 40 %50 more.', known)).toBe(+ `Teams ${NUMBER_PLACEHOLDER} ${NUMBER_PLACEHOLDER} more.`,+ );+ expect(placeholderNumbers('%40 %50 more.', known)).toBe(+ `${NUMBER_PLACEHOLDER} ${NUMBER_PLACEHOLDER} more.`,+ );+ });+});diff --git a/server/src/lib/edit-spans.js b/server/src/lib/edit-spans.jsnew file mode 100644index 000000000..0e1714660--- /dev/null+++ b/server/src/lib/edit-spans.js@@ -0,0 +1,53 @@+/**+ * Edits to a text ({ start, end, …}) kept apart and in order of start, as+ * the guards put their placeholders: whether a new one is free is read from+ * its two neighbours, found by a binary search, so a line with thousands of+ * names, sources or numbers to replace takes time near its length, not its+ * edits times them (#1544 review).+ */+export function editSpans() {+ const spans = [];+ // The first span that starts at `start` or after it.+ const at = (start) => {+ let lo = 0;+ let hi = spans.length;+ while (lo < hi) {+ const mid = (lo + hi) >> 1;+ if (spans[mid].start < start) lo = mid + 1;+ else hi = mid;+ }+ return lo;+ };+ return {+ /** The spans, in order of start. */+ spans,+ /** Whether [start, end) overlaps none of the spans kept. */+ free(start, end) {+ const i = at(start);+ if (spans[i - 1] && spans[i - 1].end > start) return false;+ // The spans that start inside it: one at most, past an empty one at+ // its start (an empty span may share a start with another).+ for (let j = i; j < spans.length && spans[j].start < end; j++) {+ if (start < spans[j].end) return false;+ }+ return true;+ },+ /**+ * Keep an edit, in its place: after the spans that start before it, or+ * at its start and end no later (an empty span first), so the span+ * before a place is always the one that may run past it.+ */+ add(edit) {+ let lo = 0;+ let hi = spans.length;+ while (lo < hi) {+ const mid = (lo + hi) >> 1;+ const span = spans[mid];+ if (span.start < edit.start || (span.start === edit.start && span.end <= edit.end)) {+ lo = mid + 1;+ } else hi = mid;+ }+ spans.splice(lo, 0, edit);+ },+ };+}diff --git a/server/src/lib/edit-spans.test.js b/server/src/lib/edit-spans.test.jsnew file mode 100644index 000000000..b11c7b741--- /dev/null+++ b/server/src/lib/edit-spans.test.js@@ -0,0 +1,58 @@+import { describe, expect, it } from 'vitest';+import { editSpans } from './edit-spans.js';++/** Edits kept apart and in order, checked against their neighbours (#1544 review). */++/** What a list of edits says is free, read the slow way: against every one. */+const freeByAll = (spans, start, end) => spans.every((e) => end <= e.start || start >= e.end);++describe('editSpans', () => {+ it('keeps edits in order of start and says what is free, as against every edit', () => {+ const { spans, free, add } = editSpans();+ for (const [start, end] of [+ [20, 25],+ [5, 10],+ [30, 31],+ ]) {+ expect(free(start, end)).toBe(true);+ add({ start, end });+ }+ expect(spans.map((e) => e.start)).toEqual([5, 20, 30]);+ for (const [start, end] of [+ [0, 5],+ [0, 6],+ [9, 11],+ [10, 20],+ [10, 21],+ [24, 30],+ [26, 29],+ [31, 40],+ [3, 40],+ ]) {+ expect(free(start, end), `${start}-${end}`).toBe(freeByAll(spans, start, end));+ }+ });++ it('reads an empty span at another one’s start as the slow check does', () => {+ const { spans, free, add } = editSpans();+ add({ start: 5, end: 10 });+ expect(free(5, 5)).toBe(true);+ add({ start: 5, end: 5 });+ expect(free(5, 7)).toBe(false);+ expect(free(5, 7)).toBe(freeByAll(spans, 5, 7));+ expect(free(10, 12)).toBe(true);+ });++ it('agrees with the slow check on random edits', () => {+ const { spans, free, add } = editSpans();+ let seed = 7;+ const next = (n) => (seed = (seed * 1103515245 + 12345) % 2 ** 31) % n;+ for (let i = 0; i < 2000; i++) {+ const start = next(500);+ const end = start + next(6);+ const fast = free(start, end);+ expect(fast, `${start}-${end}`).toBe(freeByAll(spans, start, end));+ if (fast) add({ start, end });+ }+ });+});