Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(content): read no product sentence as a claim, know the brand's own name, keep the possessive out of the placeholder, and read more Turkish and German forms (review follow-ups of #1537) (#1544)

Cihan3b650fb62c80
diff --git a/server/src/lib/content-agent.js b/server/src/lib/content-agent.js
index fbd62ff85..78620c19a 100644
--- a/server/src/lib/content-agent.js
+++ b/server/src/lib/content-agent.js
@@ -356,6 +356,23 @@ const NAME_WORD = String.raw`(?<![\p{L}\p{N}&'’.-])\p{Lu}(?:[\p{L}\p{N}&'’-]
// A name as an attribution gives it: capitalised words, after an article
// or a year ("the Zapier report", "a 2025 Gartner survey").
const SOURCE_NAME = String.raw`(?<name>(?!(?:The|This|That|These|Those|Our|Their|Its|Your|My|His|Her|We|They|It|A|An)\b)${NAME_WORD}(?:\s+${NAME_WORD}){0,${MAX_NAME_WORDS - 1}})`;
+/** A name before a Turkish ending ("Google Analytics'e göre"): capitalised words, a few at most. */
+const TR_WORD = String.raw`(?<![\p{L}\p{N}.-])\p{Lu}(?:[\p{L}\p{N}-]|${NAME_DOT})*`;
+const TR_NAME = String.raw`(?<name>${TR_WORD}(?:\s+${TR_WORD}){0,2})`;
+/** A name shaped as one, sentence's start or not: an acronym (TÜİK, GA4) or a capital after a small letter (HubSpot). */
+const TR_SHAPED = String.raw`(?<name>(?<![\p{L}\p{N}.-])\p{Lu}(?:[\p{Lu}\p{N}]+(?![\p{Ll}])|(?=[\p{L}\p{N}]*?(?:\p{Ll}\p{Lu}|\p{L}\p{N}))[\p{L}\p{N}]+))`;
+/** A source's report, study, data or survey "göre": a year or a small word before it. */
+const TR_REPORT = String.raw`(?:(?:\d{4}|\p{Ll}+)\s+)?(?:raporuna|araştırmasına|verilerine|anketine|çalışmasına|açıklamasına)\s+göre\b`;
+/** German articles and possessives before a noun ("dem Bericht", "unseren Messungen"). */
+const DE_DET = String.raw`(?:[Dd](?:em|er|en|ie|as|es)|[Ee]in(?:e|er|em|en|es)?|[Uu]nser(?:e|er|em|en|es)?|[Ii]hr(?:e|er|em|en|es)?|[Nn]euesten|[Aa]ktuellen|[Jj]üngsten)`;
+/**
+ * Turkish words a sentence starts with before a source's name ("Ancak
+ * Gartner'a göre", "Son raporuna göre"): read past by the attributions only,
+ * never by the people guard ("Son Heung-min").
+ */
+const TR_SENTENCE_WORDS = new Set(
+ 'ancak ama fakat ayrıca bu şu bir ve ile son yeni özellikle'.split(' '),
+);
const LEAD = String.raw`(?:(?:the|a|an)\s+)?(?:\d{4}\s+)?`;
const ATTRIBUTIONS = [
new RegExp(String.raw`\b[Aa]ccording to\s+${LEAD}${SOURCE_NAME}`, 'gu'),
@@ -371,24 +388,60 @@ const ATTRIBUTIONS = [
'gu',
),
// …and never a run of names counted ("all 50 US states that allow…", "most
- // US states that", "the 48 contiguous US states that"), or one read from
- // the middle of a name ("New England states that").
+ // US states that", "Most US states that", "the 48 contiguous US states
+ // that"), or one read from the middle of a name ("New England states
+ // that"); small words between only after a number or "all", so "In some
+ // cases Gartner states that" is read. A capital first, as below: a
+ // lookbehind ending in a run of spaces isn't tried at each of them.
new RegExp(
- String.raw`(?<!(?:\b(?:[Aa]ll|[Mm]ost|[Mm]any|[Ss]ome|[Ss]everal|[Bb]oth|[Ee]very|\d{1,3})(?:\s+\p{Ll}+){0,2}|\p{Lu}[\p{L}\p{N}&.'’-]*)\s+)${SOURCE_NAME}\s+(?:claims|claimed|states|stated)\s+that\b`,
+ String.raw`(?=\p{Lu})(?<!(?:\b(?:[Aa]ll|\d{1,3})(?:\s+\p{Ll}+){0,2}|\b(?:[Mm]ost|[Mm]any|[Ss]ome|[Ss]everal|[Bb]oth|[Ee]very)|\p{Lu}[\p{L}\p{N}&.'’-]*)\s+)(?!(?:All|Most|Many|Some|Several|Both|Every)\b)${SOURCE_NAME}\s+(?:claims|claimed|states|stated)\s+that\b`,
'gu',
),
new RegExp(
String.raw`${SOURCE_NAME}(?:['’]s)?\s+(?:report|study|survey|research|data|analysis)\s+(?:shows|showed|finds|found|says|said|suggests)\b`,
'gu',
),
- new RegExp(String.raw`\b[Ll]aut\s+${LEAD}${SOURCE_NAME}`, 'gu'),
+ // Never the maker's own words or a manual, as product pages give them
+ // ("laut Hersteller", "laut Herstellerangaben", "laut Datenblättern",
+ // "laut Anleitung").
+ new RegExp(
+ String.raw`\b[Ll]aut\s+${LEAD}(?!Hersteller|Datenbl|Anleitung|Bedienungsanleitung|Produktbeschreibung|Gesetz)${SOURCE_NAME}`,
+ 'gu',
+ ),
+ // "nach Angaben von Gartner", never the maker's ("des Herstellers").
+ new RegExp(String.raw`\b[Nn]ach\s+Angaben\s+von\s+${LEAD}(?!Hersteller)${SOURCE_NAME}`, 'gu'),
+ // "Gartner zufolge"; never a noun German capitalises: after an article
+ // ("Dem Bericht zufolge", "Einer Studie zufolge") or a plural ("Berichten",
+ // "Herstellerangaben", "Insidern zufolge"), the cost a name like Nielsen
+ // pays. A capital first: the lookbehind ends in a run of spaces, and tried
+ // at each of a long run's, it would read it in time growing with its square.
+ new RegExp(
+ String.raw`(?=\p{Lu})(?<!\b${DE_DET}\s+)(?!${DE_DET}\b|Hersteller)${SOURCE_NAME}(?<!\p{Ll}(?:en|ern))\s+zufolge\b`,
+ 'gu',
+ ),
// "As Zapier notes, …", "As Zapier puts it, …": the verb then a comma or
// "that", never an event ("As Google adds AI Overviews to more queries").
new RegExp(String.raw`\b[Aa]s\s+${LEAD}${SOURCE_NAME}\s+${SPEECH}(?=\s*[,:]|\s+that\b)`, 'gu'),
- // "a 2025 report by Gartner, …", "a survey from Statista finds": as a
- // claim's source, never a thing to get ("Download the report from Gartner.").
+ // "a survey from Statista finds": as a claim's source, never a thing to get
+ // ("Download the report from Gartner."); "shows" only before "that" or a
+ // figure, as a tool's report shows things too ("The Performance report
+ // from Google Search Console shows clicks"). One short aside may come
+ // before the verb ("A 2025 report by Gartner, published in May, found…"),
+ // bounded so the read stays linear.
new RegExp(
- String.raw`\b(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s+(?:by|from)\s+${LEAD}${SOURCE_NAME}(?=\s*,|\s+(?:shows?|showed|finds|found|says|said|suggests?|suggested|reveals?|revealed|estimates?|estimated)\b)`,
+ String.raw`\b(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s+(?:by|from)\s+${LEAD}${SOURCE_NAME}(?=(?:\s*,[^,.!?;]{1,80},)?\s+(?:(?:shows?|showed)\s+(?:that\b|\p{N})|(?:finds|found|says|said|suggests?|suggested|reveals?|revealed|estimates?|estimated)\b))`,
+ 'gu',
+ ),
+ // "In a 2025 report by Gartner, …", "According to a survey from Statista,
+ // …": the comma only after "according to", "per" or a sentence's "In",
+ // with small words between that aren't a product's or the reader's ("In
+ // this report from Google Search Console, you can…", "In the Performance
+ // report from…"), never a product's sentence ("Send a survey from
+ // Typeform, Jotform or Google Forms"). The noun is looked for first: a
+ // lookbehind ending in a run of spaces, tried at each of them, would read
+ // a long run in time growing with its square.
+ new RegExp(
+ String.raw`(?=(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s)(?<=(?:\b[Aa]ccording\s+to|\b[Pp]er|(?:^|[.!?]\s+)In)\s+(?:(?!(?:your|this|our|each|these|those|that|my|their)\b)[\p{Ll}\p{N}]+\s+){0,3})(?:[Rr]eport|[Ss]tudy|[Ss]urvey|[Pp]aper|[Pp]oll)\s+(?:by|from)\s+${LEAD}${SOURCE_NAME}(?=\s*,)`,
'gu',
),
// "data from Statista shows that …" or a figure, never a tool's ("Data
@@ -404,16 +457,18 @@ const ATTRIBUTIONS = [
'gu',
),
// "Gartner'ın raporuna göre": a source's report, study, data or survey.
- new RegExp(
- String.raw`(?<name>(?<![\p{L}\p{N}.-])\p{Lu}(?:[\p{L}\p{N}-]|${NAME_DOT})*)['’]\p{L}{1,3}\s+(?:raporuna|araştırmasına|verilerine|anketine|çalışmasına|açıklamasına)\s+göre\b`,
- 'gu',
- ),
+ // A year or a word before the noun ("Gartner'ın 2025 raporuna", "son
+ // raporuna"), a name of more words than one ("Google Analytics").
+ new RegExp(String.raw`${TR_NAME}['’]\p{L}{1,3}\s+${TR_REPORT}`, 'gu'),
+ // Without its ending ("Gartner raporuna", "TÜİK verilerine"): a name in
+ // the sentence, or one shaped as a name (TÜİK, GA4, HubSpot), never a
+ // sentence's first noun ("Ürün açıklamasına göre", "Sipariş verilerine").
+ // A capital first, so the lookbehind isn't tried in a run of spaces.
+ new RegExp(String.raw`(?=\p{Lu})(?<=[^\s.!?:]\s+)${TR_NAME}\s+${TR_REPORT}`, 'gu'),
+ new RegExp(String.raw`${TR_SHAPED}\s+${TR_REPORT}`, 'gu'),
// "Gartner'a göre", never a comparison ("ChatGPT'ye göre daha hızlı");
// "en" after it is the source's claim ("Gartner'a göre en hızlı büyüyen").
- new RegExp(
- String.raw`(?<name>(?<![\p{L}\p{N}.-])\p{Lu}(?:[\p{L}\p{N}-]|${NAME_DOT})*)['’]\p{L}{1,3}\s+göre\b(?!\s+daha\b)`,
- 'gu',
- ),
+ new RegExp(String.raw`${TR_NAME}['’]\p{L}{1,3}\s+göre\b(?!\s+daha\b)`, 'gu'),
];
/**
@@ -437,7 +492,7 @@ const attributionsFor = (language) =>
* to already, Brief › Your input's (an expert, the person's own data) and
* the workspace's authors.
*/
-export function attributableSources(pageHtml, numbered, authors = [], language = null) {
+export function attributableSources(pageHtml, numbered, authors = [], language = null, own = []) {
const inputs = numbered.filter((s) => s.kind === 'input').map(sourceWords);
return {
// Read line by line, as a change's blocks are (htmlLines).
@@ -445,6 +500,8 @@ export function attributableSources(pageHtml, numbered, authors = [], language =
...htmlLines(pageHtml).flatMap((line) => attributedNames(line, language)),
...inputs,
...authors,
+ // The brand's own: its report or data says what it says ("A weekly report from Acme").
+ ...own,
),
language,
};
@@ -456,7 +513,10 @@ function attributionsIn(text, language = null) {
for (const re of attributionsFor(language)) {
for (const match of String(text).matchAll(re)) {
const name = match.groups?.name ?? '';
- if (name) found.push({ name, index: match.index + match[0].indexOf(name) });
+ // A sentence's words alone ("Who says", "Son raporuna göre") name no one.
+ if (name && nameStart(name) < name.length) {
+ found.push({ name, index: match.index + match[0].indexOf(name) });
+ }
}
}
return found;
@@ -478,7 +538,10 @@ function nameStart(name) {
const words = [...name.matchAll(/\S+/g)];
for (const [i, word] of words.entries()) {
const lower = word[0].toLowerCase();
- if (NOT_NAMES.has(lower)) continue;
+ // A sentence's word, never an acronym written as one ("According to AMA").
+ if ((NOT_NAMES.has(lower) || TR_SENTENCE_WORDS.has(lower)) && !/^\p{Lu}{2,}$/u.test(word[0])) {
+ continue;
+ }
const ly = /^\p{L}+ly$/u.test(lower);
if (!ly && !isRole(lower)) return word.index;
if (i === words.length - 1 && (ly || /^\p{Lu}\p{Ll}+$/u.test(word[0]))) return word.index;
@@ -546,7 +609,8 @@ function attributionEdits(text, known) {
for (const { name, index } of attributionsIn(text, known?.language)) {
if (known && knownName(name, known)) continue;
const start = index + nameStart(name);
- const end = index + name.length;
+ // The possessive stays: "According to [source]'s latest survey".
+ const end = index + name.replace(/['’]\p{Ll}{1,3}$/u, '').length;
if (start >= end || edits.some((e) => start < e.end && end > e.start)) continue;
edits.push({ start, end, text: '[source]' });
}
@@ -1264,7 +1328,8 @@ async function fixWithRun({
draft.body_html,
numbered,
authors,
- await draftLanguage(draft, brand.organization_id, brand.language),
+ await draftLanguage(draft, brand),
+ [brand.name],
);
// Where it may link: the brand's pages and the draft's sources (A); a
// hand-off's source checks link the draft's sources.
diff --git a/server/src/lib/content-agent.test.js b/server/src/lib/content-agent.test.js
index 9267e8573..fd8c5c09e 100644
--- a/server/src/lib/content-agent.test.js
+++ b/server/src/lib/content-agent.test.js
@@ -97,7 +97,8 @@ const {
} = await import('./content-agent.js');
const { linkKey, NO_LINK_TARGETS } = await import('./content-links.js');
const { knownNumbers, numberSources } = await import('./content-sources.js');
-const { NAME_PLACEHOLDER, QUOTE_PLACEHOLDER, knownPeople } = await import('./content-people.js');
+const { NAME_PLACEHOLDER, QUOTE_PLACEHOLDER, guardPeople, knownPeople } =
+ await import('./content-people.js');
const { PlanLimitError } = await import('./plan-guard.js');
// The real hold: self-hosted here, so it reserves nothing and writes each charge.
const realGuard = await vi.importActual('./plan-guard.js');
@@ -651,6 +652,44 @@ describe('keepSuggestions', () => {
expect(Date.now() - started).toBeLessThan(500);
});
+ it('reads a long run of spaces quickly: no lookbehind is tried at each of them', () => {
+ // A character past Latin-1 (’, ı) in each: V8 skips no position then.
+ const reads = (run) => [
+ () => attributableSources(`<p>Ürün${run}açıklamasına göre</p>`, [], []),
+ () => placeholderAttributions(`In a’s${run}b report from Gartner, most do.`),
+ () => addsAttribution('', `x${run}Gartner raporuna göre en hızlı.`),
+ () => attributedNames(`Dem’${run}Bericht zufolge.`),
+ () => attributedNames(`Most’${run}US states that allow it.`),
+ ];
+ // Compiled once, apart from the timing.
+ reads(' ').forEach((read) => read());
+ for (const space of [' ', '\u00a0', '\n']) {
+ for (const read of reads(space.repeat(30_000))) {
+ const started = Date.now();
+ read();
+ expect(Date.now() - started).toBeLessThan(150);
+ }
+ }
+ });
+
+ it('reads a claim with a short aside after "a report by X" (review, #1544)', () => {
+ expect(
+ attributedNames('A 2025 report by Gartner, published in May, found that 40% of teams plan.'),
+ ).toEqual(['Gartner']);
+ expect(
+ attributedNames(
+ 'A recent study from Stanford, which surveyed 500 teams, shows that most plan.',
+ ),
+ ).toEqual(['Stanford']);
+ // Still never a product's sentence, nor an aside past its bound.
+ expect(
+ attributedNames('The Performance report from Google Search Console shows clicks.'),
+ ).toEqual([]);
+ expect(
+ attributedNames(`A report by Gartner, ${'and so on '.repeat(10)}, found that most plan.`),
+ ).toEqual([]);
+ });
+
it('reads a long run of initials quickly: each dot is read one way', () => {
const initials = 'U.S.'.repeat(10_000);
const started = Date.now();
@@ -910,6 +949,188 @@ describe('blocks, names and languages as the review read them (#1520)', () => {
});
});
+describe('attributions as the review of #1537 read them', () => {
+ const none = knownPeople('');
+ const reads = (texts, expected) => {
+ for (const text of texts) expect(addsAttribution('', text), text).toBe(expected);
+ };
+
+ it('reads no product sentence as a claim: a comma after "from X", a tool’s report that "shows"', () => {
+ reads(
+ [
+ 'Send a survey from Typeform, Jotform or Google Forms.',
+ 'Download the 2025 report from Gartner, then share it with your team.',
+ 'The Performance report from Google Search Console shows clicks and impressions.',
+ 'Who says B2B content has to be boring?',
+ 'In this report from Google Search Console, you can see clicks.',
+ 'In the Performance report from Google Search Console, clicks are listed.',
+ 'Built in a study from Figma, the kit has 40 parts.',
+ ],
+ false,
+ );
+ // An acronym is a name, even one a sentence could start with ("ama").
+ reads(
+ ['According to AMA, most marketers plan weekly.', 'AMA says that most plan weekly.'],
+ true,
+ );
+ // Turkish sentence words are the attributions' only: the people guard still reads a person.
+ expect(guardPeople('Son Heung-min said the team won.', none).text).toContain(NAME_PLACEHOLDER);
+ reads(
+ [
+ 'In a 2025 report by Gartner, most teams forecast weekly.',
+ 'According to a survey from Statista, most teams forecast weekly.',
+ 'A survey from Statista shows that most teams forecast weekly.',
+ 'A report from Gartner shows 40 teams forecast weekly.',
+ 'A survey from Statista finds most teams forecast weekly.',
+ ],
+ true,
+ );
+ });
+
+ it('knows the brand’s own name: its report or data says what it says', () => {
+ const known = attributableSources('<p>Acme tracks AI answers.</p>', [], [], 'en', ['Acme']);
+ expect(addsAttribution('', 'Acme’s data shows that most pages gain citations.', known)).toBe(
+ false,
+ );
+ expect(
+ addsAttribution('', 'A weekly report from Acme shows that your top pages gain.', known),
+ ).toBe(false);
+ expect(addsAttribution('', 'Gartner’s data shows that most pages gain.', known)).toBe(true);
+ });
+
+ it('keeps the possessive out of the placeholder, never a name’s own apostrophe', () => {
+ expect(placeholderAttributions("According to O'Day, most do.", none)).toBe(
+ 'According to [source], most do.',
+ );
+ expect(placeholderAttributions("According to Gartner's latest survey, most do.", none)).toBe(
+ "According to [source]'s latest survey, most do.",
+ );
+ expect(placeholderAttributions('Gartner’s research shows teams forecast.', none)).toBe(
+ '[source]’s research shows teams forecast.',
+ );
+ });
+
+ it('reads Turkish reports with or without the ending, a word before the noun, a name of more words', () => {
+ reads(
+ [
+ "Gartner'ın 2025 raporuna göre pazar büyüyor.",
+ "Gartner'ın son raporuna göre pazar büyüyor.",
+ 'Pazar, Gartner raporuna göre büyüyor.',
+ 'TÜİK verilerine göre enflasyon düştü.',
+ 'HubSpot verilerine göre e-posta açılma oranı düştü.',
+ ],
+ true,
+ );
+ // Without the ending, a sentence's first noun isn't a name.
+ reads(
+ [
+ 'Son raporuna göre pazar büyüyor.',
+ 'Ürün açıklamasına göre kahve makinesi on bardak yapar.',
+ 'Sipariş verilerine göre en çok satan model bu.',
+ 'Haziran verilerine göre satışlar arttı.',
+ ],
+ false,
+ );
+ // The brand's own: "Panelinizdeki Ansvisor verilerine göre" reads Ansvisor, known.
+ const known = attributableSources('<p>Ansvisor</p>', [], [], 'tr', ['Ansvisor']);
+ expect(addsAttribution('', 'Panelinizdeki Ansvisor verilerine göre trafik arttı.', known)).toBe(
+ false,
+ );
+ expect(placeholderAttributions("Google Analytics'e göre trafik arttı.", none)).toBe(
+ "[source]'e göre trafik arttı.",
+ );
+ expect(placeholderAttributions("Ancak Google Analytics'e göre trafik arttı.", none)).toBe(
+ "Ancak [source]'e göre trafik arttı.",
+ );
+ });
+
+ it('reads German "zufolge" and "nach Angaben von", never the maker’s own words', () => {
+ reads(
+ ['Gartner zufolge wächst der Markt.', 'Nach Angaben von Gartner wächst der Markt.'],
+ true,
+ );
+ reads(
+ [
+ 'Berichten zufolge wächst der Markt.',
+ 'Den Angaben zufolge wächst der Markt.',
+ 'Laut Hersteller hält der Akku zehn Stunden.',
+ 'Laut Herstellerangaben hält der Akku zehn Stunden.',
+ 'Laut Datenblatt wiegt es zwei Kilo.',
+ 'Laut Datenblättern wiegt es zwei Kilo.',
+ 'Nach Angaben des Herstellers hält der Akku zehn Stunden.',
+ // A noun German capitalises, after an article or as a plural.
+ 'Dem Bericht zufolge wächst der Markt.',
+ 'Einer Studie zufolge wächst der Markt.',
+ 'Herstellerangaben zufolge hält der Akku zehn Stunden.',
+ 'Kundenbewertungen zufolge ist er leise.',
+ 'Unseren Messungen zufolge ist er leise.',
+ 'Insidern zufolge wächst der Markt.',
+ 'Neuesten Zahlen zufolge wächst der Markt.',
+ ],
+ false,
+ );
+ reads(['Laut Gartner wächst der Markt.', 'IDC zufolge wächst der Markt.'], true);
+ });
+});
+
+describe('the review of #1537, round 2 (low follow-ups)', () => {
+ it('ends a line at a line break inside a bold or a link too', () => {
+ expect(
+ htmlLines(
+ '<p><strong>Forecasting basics<br>According to Zapier, teams forecast.</strong></p>',
+ ),
+ ).toEqual(['Forecasting basics', 'According to Zapier, teams forecast.']);
+ expect(
+ htmlLines(
+ '<p><em>Teams plan weekly, according to Gartner.<br>Most teams plan monthly.</em></p>',
+ ),
+ ).toEqual(['Teams plan weekly, according to Gartner.', 'Most teams plan monthly.']);
+ expect(htmlLines('<p><strong>ChatGPT<br>Gemini<br>Claude</strong></p>')).toEqual([
+ 'ChatGPT',
+ 'Gemini',
+ 'Claude',
+ ]);
+ // A claim a bold's line break hid is read; one on a page that cites the source isn't new.
+ const cites = attributableSources('<p>According to Gartner, teams forecast.</p>', [], []);
+ expect(
+ keepSuggestions(
+ [
+ suggestion({
+ kind: 'insert',
+ passage: 1,
+ before: '',
+ after:
+ '<p><strong>Forecasting basics<br>According to Zapier, teams forecast.</strong></p>',
+ sources: [],
+ marks: [],
+ }),
+ ],
+ [{ tag: 'p', text: 'Track ChatGPT, Gemini and Claude.' }],
+ new Set(),
+ new Set(['dated']),
+ null,
+ { ...sourcing, attributable: cites },
+ ),
+ ).toEqual([]);
+ });
+
+ it('reads "states" and "claims" as nouns after a count at a sentence’s start, and a claim after small words', () => {
+ for (const text of [
+ 'Most US states that allow it ship wine.',
+ 'Every FDA states that list it are fine.',
+ ]) {
+ expect(addsAttribution('', text), text).toBe(false);
+ expect(placeholderAttributions(text, knownPeople('')), text).toBe(text);
+ }
+ for (const text of [
+ 'In some cases Gartner states that most teams forecast weekly.',
+ 'For many years Gartner claims that most teams forecast weekly.',
+ ]) {
+ expect(addsAttribution('', text), text).toBe(true);
+ }
+ });
+});
+
describe('no invented people or quotes (CE-044)', () => {
const passages = draftPassages(BODY);
const ruleKeys = new Set(['dated', 'faq']);
@@ -1384,6 +1605,23 @@ describe('fixDraftRules', () => {
expect(await fix()).toBe(1);
});
+ it('gives a claim to the brand’s own data: its name is known', async () => {
+ generateObject.mockResolvedValue({
+ object: {
+ suggestions: [
+ suggestion({
+ after: 'good coffee and hot water, as Acme Coffee’s data shows that brews improve',
+ sources: [],
+ marks: [],
+ }),
+ ],
+ },
+ usage: { inputTokens: 900, outputTokens: 200 },
+ });
+ const { count } = await fixDraftRules({ draftId: DRAFT, brandId: BRAND, input: input() });
+ expect(count).toBe(1);
+ });
+
it('charges the budget, asks the model and keeps one batch for review', async () => {
generateObject.mockResolvedValue({
object: {
diff --git a/server/src/lib/content-delivery.js b/server/src/lib/content-delivery.js
index c50f8ff78..b103a7b7a 100644
--- a/server/src/lib/content-delivery.js
+++ b/server/src/lib/content-delivery.js
@@ -4,7 +4,7 @@ import { z } from 'zod';
import supabaseAdmin from '../config/supabase.js';
import { logger } from './logger.js';
import { PageFetchError } from './safe-fetch.js';
-import { rulesLanguage, untouchedRules } from './content-language.js';
+import { brandSiteType, rulesLanguage, untouchedRules } from './content-language.js';
import { postJson } from './safe-post.js';
import { isWebflowConfigured } from './composio.js';
import { sendToWebflow } from './cms/webflow-delivery.js';
@@ -436,14 +436,24 @@ async function deliverBatch(going, { brand, configs, webflow, batchId }) {
/** A draft no one has set rules on: the workspace's My rules apply (rules.ts effectiveRules). */
const untouched = untouchedRules;
-/** The workspace's My rules (00214) for these drafts' types, by type; none when unread. */
-async function myRuleSets(orgId, drafts) {
+/**
+ * The workspace's My rules (00214) for these drafts' types and the brand's
+ * kind of site (00227 keys a set by both, as the editor reads it), by type;
+ * none when unread.
+ */
+async function myRuleSets(orgId, brandId, drafts) {
const types = [...new Set(drafts.map((d) => d.rules?.set ?? d.page_type).filter(Boolean))];
if (!types.length) return new Map();
+ const { data: brand } = await supabaseAdmin
+ .from('brands')
+ .select('site_type, industry, shopping_mode_enabled')
+ .eq('id', brandId)
+ .single();
const { data, error } = await supabaseAdmin
.from('content_rule_sets')
.select('page_type, overrides')
.eq('organization_id', orgId)
+ .eq('site_type', brandSiteType(brand))
.in('page_type', types);
if (error) logger.warn({ err: error }, '[content-delivery] My rules not read');
return new Map((data ?? []).map((set) => [set.page_type, set]));
@@ -549,6 +559,7 @@ export async function sendContentPages({
brandFor(brandId),
myRuleSets(
orgId,
+ brandId,
going.map((item) => rowOf.get(item.page.draftId)).filter((d) => d && untouched(d.rules)),
),
]);
diff --git a/server/src/lib/content-delivery.test.js b/server/src/lib/content-delivery.test.js
index 055fafffb..6af289115 100644
--- a/server/src/lib/content-delivery.test.js
+++ b/server/src/lib/content-delivery.test.js
@@ -35,7 +35,13 @@ function from(table) {
}
if (table === 'content_rule_sets') {
const types = q.filters.find(([op]) => op === 'in')?.[2] ?? [];
- return { data: db.ruleSets.filter((r) => types.includes(r.page_type)), error: null };
+ const site = q.filters.find(([op, col]) => op === 'eq' && col === 'site_type')?.[2];
+ return {
+ data: db.ruleSets.filter(
+ (r) => types.includes(r.page_type) && (!r.site_type || r.site_type === site),
+ ),
+ error: null,
+ };
}
if (table === 'content_drafts') {
const ids = q.filters.find(([op]) => op === 'in')?.[2];
@@ -320,7 +326,15 @@ describe('sendContentPages', () => {
});
it('reads the language from My rules for a draft no one has set rules on, as the editor does', async () => {
- db.ruleSets = [{ page_type: 'blog_post', overrides: { language: { value: 'French' } } }];
+ // A set for each kind of site (00227): the brand's kind's is read.
+ db.ruleSets = [
+ {
+ page_type: 'blog_post',
+ site_type: 'ecommerce',
+ overrides: { language: { value: 'German' } },
+ },
+ { page_type: 'blog_post', site_type: 'b2b', overrides: { language: { value: 'French' } } },
+ ];
await send();
expect(JSON.parse(postJson.mock.calls[0][1]).page.language).toBe('fr');
diff --git a/server/src/lib/content-language.js b/server/src/lib/content-language.js
index 3bc373cda..a8cc86b07 100644
--- a/server/src/lib/content-language.js
+++ b/server/src/lib/content-language.js
@@ -1,6 +1,7 @@
import supabaseAdmin from '../config/supabase.js';
import { logger } from './logger.js';
import { languageCodeOf } from './languages.js';
+import { siteTypeOf } from './content-page-types.js';
/**
* A page's language, as the editor shows it and the CMS send reads it: its
@@ -28,22 +29,33 @@ export function rulesLanguage(rules, brandLanguage) {
);
}
+/** The brand's kind of site, as My rules (00227) and the editor key a set by it. */
+export function brandSiteType(brand) {
+ return siteTypeOf({
+ siteType: brand?.site_type,
+ industry: brand?.industry,
+ shoppingModeEnabled: brand?.shopping_mode_enabled,
+ });
+}
+
/**
- * A draft's language, reading the workspace's My rules for its page type
- * when it has no rules of its own. A failed read leaves the brand's.
+ * A draft's language, reading the workspace's My rules for its page type and
+ * the brand's kind of site (as the editor's getMyRuleSets does) when it has
+ * no rules of its own. A failed read leaves the brand's.
*/
-export async function draftLanguage(draft, organizationId, brandLanguage) {
+export async function draftLanguage(draft, brand) {
let rules = draft.rules;
const type = rules?.set ?? draft.page_type;
- if (untouchedRules(rules) && type && organizationId) {
+ if (untouchedRules(rules) && type && brand?.organization_id) {
const { data, error } = await supabaseAdmin
.from('content_rule_sets')
.select('overrides')
- .eq('organization_id', organizationId)
+ .eq('organization_id', brand.organization_id)
+ .eq('site_type', brandSiteType(brand))
.eq('page_type', type)
.maybeSingle();
if (error) logger.warn({ err: error }, '[content-language] My rules not read');
rules = error ? null : (data ?? null);
}
- return rulesLanguage(rules, brandLanguage);
+ return rulesLanguage(rules, brand?.language);
}
diff --git a/server/src/lib/content-language.test.js b/server/src/lib/content-language.test.js
index a2c8674a7..9c61748b2 100644
--- a/server/src/lib/content-language.test.js
+++ b/server/src/lib/content-language.test.js
@@ -20,7 +20,9 @@ vi.mock('../config/supabase.js', () => ({
}));
vi.mock('./logger.js', () => ({ logger: { warn: vi.fn() } }));
-const { draftLanguage, rulesLanguage, untouchedRules } = await import('./content-language.js');
+const { brandSiteType, draftLanguage, rulesLanguage, untouchedRules } =
+ await import('./content-language.js');
+const brand = (over = {}) => ({ organization_id: 'org-1', language: 'en', ...over });
beforeEach(() => {
rows = {};
@@ -51,7 +53,7 @@ describe('untouchedRules', () => {
describe('draftLanguage', () => {
it('reads the draft’s own rules, without My rules', async () => {
const draft = { page_type: 'guide', rules: { overrides: { language: { value: 'German' } } } };
- expect(await draftLanguage(draft, 'org-1', 'en')).toBe('de');
+ expect(await draftLanguage(draft, brand())).toBe('de');
expect(reads).toEqual([]);
});
@@ -60,21 +62,38 @@ describe('draftLanguage', () => {
data: { overrides: { language: { value: 'German' } } },
error: null,
};
- expect(await draftLanguage({ page_type: 'guide', rules: null }, 'org-1', 'en')).toBe('de');
+ expect(
+ await draftLanguage(
+ { page_type: 'guide', rules: null },
+ brand({ shopping_mode_enabled: true }),
+ ),
+ ).toBe('de');
+ // Keyed by the brand's kind of site too (00227), as the editor reads them.
expect(reads[0]).toEqual({
table: 'content_rule_sets',
filters: [
['organization_id', 'org-1'],
+ ['site_type', 'ecommerce'],
['page_type', 'guide'],
],
});
// The set a draft's rules name, over its page type.
- await draftLanguage({ page_type: 'guide', rules: { set: 'best_of' } }, 'org-1', 'en');
- expect(reads[1].filters[1]).toEqual(['page_type', 'best_of']);
+ await draftLanguage({ page_type: 'guide', rules: { set: 'best_of' } }, brand());
+ expect(reads[1].filters[2]).toEqual(['page_type', 'best_of']);
});
it('keeps the brand’s when My rules can’t be read', async () => {
rows.content_rule_sets = { data: null, error: new Error('down') };
- expect(await draftLanguage({ page_type: 'guide', rules: null }, 'org-1', 'tr')).toBe('tr');
+ expect(
+ await draftLanguage({ page_type: 'guide', rules: null }, brand({ language: 'tr' })),
+ ).toBe('tr');
+ });
+});
+
+describe('brandSiteType', () => {
+ it('reads the brand’s kind of site as the editor does', () => {
+ expect(brandSiteType({ site_type: 'b2c' })).toBe('b2c');
+ expect(brandSiteType({ shopping_mode_enabled: true })).toBe('ecommerce');
+ expect(brandSiteType({})).toBe('b2b');
});
});
diff --git a/server/src/lib/content-people.js b/server/src/lib/content-people.js
index b9a7714bc..5129550f0 100644
--- a/server/src/lib/content-people.js
+++ b/server/src/lib/content-people.js
@@ -327,27 +327,31 @@ const NAME_CUES = [
words: 2,
role: true,
},
- // "Sarah, our head of growth", at a sentence's start: a single name.
+ // "Sarah, our head of growth", at a sentence's start: a single name. A
+ // capital first, so the lookbehind (ending in spaces) isn't tried in a
+ // run of them: tried at each, a long run reads in time growing with its square.
{
re: new RegExp(
- String.raw`(?<=^|[.!?]\s+|[“"]\s*)(?<name>${WORD}),\s+(?:our|the|a|an|their|its)\s+(?:[\p{L}-]+\s+){0,3}${ROLE_ANY}(?!\p{L})`,
+ String.raw`(?=\p{Lu})(?<=^|[.!?]\s+|[“"]\s*)(?<name>${WORD}),\s+(?:our|the|a|an|their|its)\s+(?:[\p{L}-]+\s+){0,3}${ROLE_ANY}(?!\p{L})`,
'gu',
),
words: 1,
role: true,
},
// Whoever says a quote, right next to it: "“…,” says Jane Doe", "„…“,
- // sagt Anna Schmidt", "“…” diyor Ayşe Demir", "“…,” Jane Doe says".
+ // sagt Anna Schmidt", "“…” diyor Ayşe Demir", "“…,” Jane Doe says". The
+ // spaces read one way only (a mark between, or none), so a quote mark
+ // before a long run of them reads in time growing with the run, not its square.
{
re: new RegExp(
- String.raw`${QUOTE_END}\s*[,.!?]?\s*${QUOTE_SAYS}(?!\p{L})\s+(?<name>${NAME})`,
+ String.raw`${QUOTE_END}\s*(?:[,.!?]\s*)?${QUOTE_SAYS}(?!\p{L})\s+(?<name>${NAME})`,
'gu',
),
words: 2,
},
{
re: new RegExp(
- String.raw`${QUOTE_END}\s*[,.!?]?\s*(?<name>${NAME})\s+${QUOTE_SAYS}(?!\p{L})`,
+ String.raw`${QUOTE_END}\s*(?:[,.!?]\s*)?(?<name>${NAME})\s+${QUOTE_SAYS}(?!\p{L})`,
'gu',
),
words: 2,
@@ -629,6 +633,7 @@ export function guardPeopleInHtml(html, known) {
*/
export function editLines($, editsFor) {
let count = 0;
+ const BREAK = Symbol('line end');
// A line: its own text nodes and its inline tags', in order, read as one.
const line = (nodes, heading) => {
if (!nodes.length) return;
@@ -655,14 +660,18 @@ export function editLines($, editsFor) {
});
}
};
- // An inline tag's text, with any line inside it read on its own.
+ // An inline tag's text, with any line inside it read on its own: a line
+ // break or a block in a bold or a link ends the line there too (BREAK).
const inlineText = (el) => {
const out = [];
for (const node of $(el).contents().get()) {
if (node.type === 'text') out.push(node);
else if (node.type !== 'tag' || KEPT_AS_IS.has(node.name)) continue;
- else if (BLOCKS.has(node.name)) walk(node, HEADINGS.has(node.name));
- else out.push(...inlineText(node));
+ else if (node.name === 'br') out.push(BREAK);
+ else if (BLOCKS.has(node.name)) {
+ out.push(BREAK);
+ walk(node, HEADINGS.has(node.name));
+ } else out.push(...inlineText(node));
}
return out;
};
@@ -679,7 +688,15 @@ export function editLines($, editsFor) {
line(run, heading);
run = [];
walk(node, HEADINGS.has(node.name));
- } else run.push(...inlineText(node));
+ } else {
+ for (const piece of inlineText(node)) {
+ if (piece !== BREAK) run.push(piece);
+ else {
+ line(run, heading);
+ run = [];
+ }
+ }
+ }
}
line(run, heading);
};
diff --git a/server/src/lib/content-people.test.js b/server/src/lib/content-people.test.js
index 54860be15..0c53d7c26 100644
--- a/server/src/lib/content-people.test.js
+++ b/server/src/lib/content-people.test.js
@@ -265,6 +265,23 @@ describe('guardPeople: what the review found', () => {
expect(fix('It was explained by Marco Rossi.')).toBe(`It was explained by ${NAME}.`);
});
+ it('reads a long run of spaces quickly, after a quote mark too (review, #1544)', () => {
+ // A character past Latin-1 (’, “): V8 skips no position then.
+ const texts = (run) => [
+ `Ürün’${run}sarah, our head of growth.`,
+ `“Pour slowly”${run}says it.`,
+ `"Pour slowly"${run}it says.`,
+ ];
+ texts(' ').forEach((text) => guardPeople(text, people));
+ for (const space of [' ', '\u00a0', '\n']) {
+ for (const text of texts(space.repeat(30_000))) {
+ const started = Date.now();
+ guardPeople(text, people);
+ expect(Date.now() - started).toBeLessThan(150);
+ }
+ }
+ });
+
it('reads long text quickly, whatever its words', () => {
const word = `A${'a'.repeat(400)}`;
const long = `${word} ${word} ${word} said. `.repeat(10);
diff --git a/server/src/lib/content-writer.js b/server/src/lib/content-writer.js
index f16470ad4..47e5f69f8 100644
--- a/server/src/lib/content-writer.js
+++ b/server/src/lib/content-writer.js
@@ -906,7 +906,8 @@ async function writeWithRun({ draft, brand, model, input, budget, rewrite, run }
draft.body_html,
numbered,
authors,
- await draftLanguage(draft, brand.organization_id, brand.language),
+ await draftLanguage(draft, brand),
+ [brand.name],
);
if (rewrite) {