Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(content): read links and page length in time near their size (#1700)
Gökhan Geyikf9dac2bbf62f
diff --git a/server/src/lib/content-links.js b/server/src/lib/content-links.jsindex a79430fc0..f5160a308 100644--- a/server/src/lib/content-links.js+++ b/server/src/lib/content-links.js@@ -20,6 +20,13 @@ export const MAX_SOURCE_PAGES = 20;/** Query parameters that only track a visit. */const TRACKING = /^(utm_\w+|gclid|fbclid|msclkid|mc_cid|mc_eid|ref)$/i;+/** A path without the slashes it ends in, read back from its end. */+function withoutTrailingSlashes(path) {+ let end = path.length;+ while (end > 0 && path[end - 1] === '/') end -= 1;+ return path.slice(0, end);+}+/*** A URL as links compare: an http(s) address, its host lower-cased without* www, no hash, no tracking parameters, no trailing slash. Null for anything else.@@ -35,8 +42,7 @@ export function linkKey(url) {const host = parsed.hostname.toLowerCase().replace(/^www\./, '');const params = [...parsed.searchParams].filter(([name]) => !TRACKING.test(name));const search = params.length ? `?${new URLSearchParams(params).toString()}` : '';- const path = parsed.pathname.replace(/\/+$/, '');- return `${host}${path}${search}`;+ return `${host}${withoutTrailingSlashes(parsed.pathname)}${search}`;}/** A URL as a link keeps it: without tracking parameters or a hash; null when not http(s). */diff --git a/server/src/lib/content-links.test.js b/server/src/lib/content-links.test.jsindex 85cecc9b9..f8164a886 100644--- a/server/src/lib/content-links.test.js+++ b/server/src/lib/content-links.test.js@@ -86,6 +86,19 @@ describe('linkKey', () => {'https://www.acme.example/a/?plan=pro',);});++ it('reads long links in time near their size', () => {+ expect(linkKey('https://acme.example/a//b///')).toBe('acme.example/a//b');+ const slashes = `https://acme.example/${'/'.repeat(100_000)}a`;+ const page = Array.from(+ { length: 500 },+ (_, i) => `https://acme.example/${'/'.repeat(2_000)}${i}`,+ );+ const started = Date.now();+ expect(linkKey(slashes)).toBe(`acme.example/${'/'.repeat(100_000)}a`);+ for (const url of page) linkKey(url);+ expect(Date.now() - started).toBeLessThan(2000);+ });});describe('readLinkTargets', () => {diff --git a/server/src/lib/content-rewrite.test.js b/server/src/lib/content-rewrite.test.jsindex fe739578c..8ce7b6fab 100644--- a/server/src/lib/content-rewrite.test.js+++ b/server/src/lib/content-rewrite.test.js@@ -179,6 +179,18 @@ describe('the rewrite', () => {expect(textLength('<h1>Ab</h1><p>c d & e</p>')).toBe('Ab c d x e'.length);});+ it('counts a long page with tags left open in time near its size', () => {+ // As the regexes read them: a tag never closed is text, a script never closed too.+ expect(textLength('<p>a</p><script>b')).toBe('a b'.length);+ expect(textLength('a <b c')).toBe('a <b c'.length);+ for (const tag of ['<p', '<script', '<style ', '<', '</div']) {+ const html = `<p>Lead</p>${tag.repeat(Math.ceil(100_000 / tag.length))}`;+ const started = Date.now();+ expect(textLength(html)).toBeGreaterThan(4);+ expect(Date.now() - started).toBeLessThan(2000);+ }+ });+it('tells the model to keep every kept line, the facts and one H1, in HTML', () => {expect(REWRITE_SYSTEM_PROMPT).toContain('never change, split, merge or drop one');expect(REWRITE_SYSTEM_PROMPT).toContain('Never invent numbers');diff --git a/server/src/lib/page-content.js b/server/src/lib/page-content.jsindex 939fe9627..226859138 100644--- a/server/src/lib/page-content.js+++ b/server/src/lib/page-content.js@@ -479,16 +479,108 @@ function commonAncestor($, a, b) {return el;}+/*+ * The tag passes below read each '<' once, left to right, and give what the+ * regex each names would, in time near the text's length however its tags+ * are left open.+ */++/** ASCII letters lower-cased, nothing else: every index stays where it was. */+const asciiLower = (text) => text.replace(/[A-Z]+/g, (letters) => letters.toLowerCase());++/** A letter, digit or _, as \b reads a word. */+const isWordChar = (code) =>+ (code >= 48 && code <= 57) ||+ (code >= 65 && code <= 90) ||+ (code >= 97 && code <= 122) ||+ code === 95;++/** Each script and style element as a space: /<(script|style)\b[\s\S]*?<\/\1>/gi. */+function withoutScripts(text) {+ const lower = asciiLower(text);+ // A name with no closing tag left closes nowhere after it either.+ const open = new Set(['script', 'style']);+ let out = '';+ let kept = 0;+ for (let at = lower.indexOf('<'); at !== -1 && open.size; ) {+ const name = [...open].find((tag) => lower.startsWith(tag, at + 1));+ const after = name ? at + 1 + name.length : 0;+ if (name && !isWordChar(lower.charCodeAt(after))) {+ const close = lower.indexOf(`</${name}>`, after);+ if (close !== -1) {+ out += `${text.slice(kept, at)} `;+ kept = close + name.length + 3;+ at = lower.indexOf('<', kept);+ continue;+ }+ open.delete(name);+ }+ at = lower.indexOf('<', at + 1);+ }+ return out + text.slice(kept);+}++const SPACED_TAGS = new Set([+ 'p',+ 'h1',+ 'h2',+ 'h3',+ 'h4',+ 'h5',+ 'h6',+ 'li',+ 'td',+ 'th',+ 'blockquote',+ 'br',+ 'div',+]);++/** Each block tag, opening or closing, as a space: /<\/?(p|h[1-6]|li|td|th|blockquote|br|div)\b[^>]*>/gi. */+function withoutBlockTags(text) {+ const lower = asciiLower(text);+ let out = '';+ let kept = 0;+ for (let at = lower.indexOf('<'); at !== -1; ) {+ const from = lower[at + 1] === '/' ? at + 2 : at + 1;+ let end = from;+ while (isWordChar(lower.charCodeAt(end))) end += 1;+ if (SPACED_TAGS.has(lower.slice(from, end))) {+ const close = lower.indexOf('>', end);+ // No '>' left: no tag after this one ends either.+ if (close === -1) break;+ out += `${text.slice(kept, at)} `;+ kept = close + 1;+ at = lower.indexOf('<', kept);+ continue;+ }+ // The word holds no '<': the next tag starts after it.+ at = lower.indexOf('<', Math.max(at + 1, end));+ }+ return out + text.slice(kept);+}++/** Every other tag dropped: /<[^>]*>/g. */+function withoutTags(text) {+ let out = '';+ let kept = 0;+ for (let at = text.indexOf('<'); at !== -1; ) {+ const close = text.indexOf('>', at + 1);+ if (close === -1) break;+ out += text.slice(kept, at);+ kept = close + 1;+ at = text.indexOf('<', kept);+ }+ return out + text.slice(kept);+}+/*** The page's text length as the Rules tab counts it (pageTextLength in* web/src/lib/content/rules.ts): a space between blocks, an entity as one* character, runs of spaces as one.*/export function textLength(html) {- return String(html ?? '')- .replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ' ')- .replace(/<\/?(p|h[1-6]|li|td|th|blockquote|br|div)\b[^>]*>/gi, ' ')- .replace(/<[^>]*>/g, '')+ return withoutTags(withoutBlockTags(withoutScripts(String(html ?? '')))).replace(/ /g, ' ').replace(/&[#a-z0-9]+;/gi, 'x').replace(/\s+/g, ' ')diff --git a/web/src/lib/content/link-targets.test.ts b/web/src/lib/content/link-targets.test.tsindex e5bd30377..e61861229 100644--- a/web/src/lib/content/link-targets.test.ts+++ b/web/src/lib/content/link-targets.test.ts@@ -13,6 +13,19 @@ describe('linkKey and linkHref', () => {'https://www.acme.example/a/?plan=pro',);});++ it('read long links in time near their size', () => {+ expect(linkKey('https://acme.example/a//b///')).toBe('acme.example/a//b');+ const slashes = `https://acme.example/${'/'.repeat(100_000)}a`;+ const page = Array.from(+ { length: 500 },+ (_, i) => `https://acme.example/${'/'.repeat(2_000)}${i}`,+ );+ const started = performance.now();+ expect(linkKey(slashes)).toBe(`acme.example/${'/'.repeat(100_000)}a`);+ for (const url of page) linkKey(url);+ expect(performance.now() - started).toBeLessThan(2000);+ });});describe('pickLinkTargets', () => {diff --git a/web/src/lib/content/link-targets.ts b/web/src/lib/content/link-targets.tsindex 4a1f2488b..ea9ad8068 100644--- a/web/src/lib/content/link-targets.ts+++ b/web/src/lib/content/link-targets.ts@@ -28,6 +28,13 @@ export interface LinkTargets {/** Query parameters that only track a visit. */const TRACKING = /^(utm_\w+|gclid|fbclid|msclkid|mc_cid|mc_eid|ref)$/i;+/** A path without the slashes it ends in, read back from its end. */+function withoutTrailingSlashes(path: string): string {+ let end = path.length;+ while (end > 0 && path[end - 1] === '/') end -= 1;+ return path.slice(0, end);+}+/*** A URL as links compare: an http(s) address, its host lower-cased without* www, no hash, no tracking parameters, no trailing slash. Null for anything else.@@ -43,7 +50,7 @@ export function linkKey(url: string): string | null {const host = parsed.hostname.toLowerCase().replace(/^www\./, '');const params = [...parsed.searchParams].filter(([name]) => !TRACKING.test(name));const search = params.length ? `?${new URLSearchParams(params).toString()}` : '';- return `${host}${parsed.pathname.replace(/\/+$/, '')}${search}`;+ return `${host}${withoutTrailingSlashes(parsed.pathname)}${search}`;}/** A URL as a link keeps it: without tracking parameters or a hash; null when not http(s). */diff --git a/web/src/lib/content/rules.test.ts b/web/src/lib/content/rules.test.tsindex 1205905af..3c3726458 100644--- a/web/src/lib/content/rules.test.ts+++ b/web/src/lib/content/rules.test.ts@@ -1231,6 +1231,14 @@ describe('My rules and text rules', () => {it('starts only a page brought in to optimize at its own length (F)', () => {expect(pageTextLength('<h1>Pricing</h1><p>Plans & prices</p>')).toBe(22);expect(pageTextLength('<p>a<script>x()</script> b</p><style>p{}</style>')).toBe(3);+ // Tags left open are read in time near the page's size.+ expect(pageTextLength('<p>a</p><script>b')).toBe('a b'.length);+ for (const tag of ['<p', '<script', '<style ', '<', '</div']) {+ const html = `<p>Lead</p>${tag.repeat(Math.ceil(100_000 / tag.length))}`;+ const started = performance.now();+ expect(pageTextLength(html)).toBeGreaterThan(4);+ expect(performance.now() - started).toBeLessThan(2000);+ }const page = `<h1>Pricing</h1><p>${'word '.repeat(40)}</p>`;expect(startingRules('optimize', page)).toEqual({ ...NO_DRAFT_RULES, importLength: 207 });// Within the range the Length rule takes.diff --git a/web/src/lib/content/rules.ts b/web/src/lib/content/rules.tsindex 1cc9e77c5..dacc6bb39 100644--- a/web/src/lib/content/rules.ts+++ b/web/src/lib/content/rules.ts@@ -454,6 +454,102 @@ export function setToSave(rules: DraftRules, mine: SavedRuleSet | undefined): Sa};}+/*+ * The tag passes below read each '<' once, left to right, and give what the+ * regex each names would, in time near the text's length however its tags+ * are left open. The same as textLength's on the server+ * (server/src/lib/page-content.js).+ */++/** ASCII letters lower-cased, nothing else: every index stays where it was. */+const asciiLower = (text: string) => text.replace(/[A-Z]+/g, (letters) => letters.toLowerCase());++/** A letter, digit or _, as \b reads a word. */+const isWordChar = (code: number) =>+ (code >= 48 && code <= 57) ||+ (code >= 65 && code <= 90) ||+ (code >= 97 && code <= 122) ||+ code === 95;++/** Each script and style element as a space: /<(script|style)\b[\s\S]*?<\/\1>/gi. */+function withoutScripts(text: string): string {+ const lower = asciiLower(text);+ // A name with no closing tag left closes nowhere after it either.+ const open = new Set(['script', 'style']);+ let out = '';+ let kept = 0;+ for (let at = lower.indexOf('<'); at !== -1 && open.size; ) {+ const name = [...open].find((tag) => lower.startsWith(tag, at + 1));+ const after = name ? at + 1 + name.length : 0;+ if (name && !isWordChar(lower.charCodeAt(after))) {+ const close = lower.indexOf(`</${name}>`, after);+ if (close !== -1) {+ out += `${text.slice(kept, at)} `;+ kept = close + name.length + 3;+ at = lower.indexOf('<', kept);+ continue;+ }+ open.delete(name);+ }+ at = lower.indexOf('<', at + 1);+ }+ return out + text.slice(kept);+}++const SPACED_TAGS = new Set([+ 'p',+ 'h1',+ 'h2',+ 'h3',+ 'h4',+ 'h5',+ 'h6',+ 'li',+ 'td',+ 'th',+ 'blockquote',+ 'br',+ 'div',+]);++/** Each block tag, opening or closing, as a space: /<\/?(p|h[1-6]|li|td|th|blockquote|br|div)\b[^>]*>/gi. */+function withoutBlockTags(text: string): string {+ const lower = asciiLower(text);+ let out = '';+ let kept = 0;+ for (let at = lower.indexOf('<'); at !== -1; ) {+ const from = lower[at + 1] === '/' ? at + 2 : at + 1;+ let end = from;+ while (isWordChar(lower.charCodeAt(end))) end += 1;+ if (SPACED_TAGS.has(lower.slice(from, end))) {+ const close = lower.indexOf('>', end);+ // No '>' left: no tag after this one ends either.+ if (close === -1) break;+ out += `${text.slice(kept, at)} `;+ kept = close + 1;+ at = lower.indexOf('<', kept);+ continue;+ }+ // The word holds no '<': the next tag starts after it.+ at = lower.indexOf('<', Math.max(at + 1, end));+ }+ return out + text.slice(kept);+}++/** Every other tag dropped: /<[^>]*>/g. */+function withoutTags(text: string): string {+ let out = '';+ let kept = 0;+ for (let at = text.indexOf('<'); at !== -1; ) {+ const close = text.indexOf('>', at + 1);+ if (close === -1) break;+ out += text.slice(kept, at);+ kept = close + 1;+ at = text.indexOf('<', kept);+ }+ return out + text.slice(kept);+}+/*** A page's text length in characters, about as the editor counts it: the* words without tags, runs of spaces as one (a space between blocks, where@@ -461,10 +557,7 @@ export function setToSave(rules: DraftRules, mine: SavedRuleSet | undefined): Sa* to count: on the server, and before a draft opens.*/export function pageTextLength(html: string): number {- return html- .replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ' ')- .replace(/<\/?(p|h[1-6]|li|td|th|blockquote|br|div)\b[^>]*>/gi, ' ')- .replace(/<[^>]*>/g, '')+ return withoutTags(withoutBlockTags(withoutScripts(html))).replace(/ /g, ' ').replace(/&[#a-z0-9]+;/gi, 'x').replace(/\s+/g, ' ')