Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(content): import a page's own text, not its author box, icons or related posts (#1322)
Cihanf10d200f1a54
diff --git a/server/src/lib/page-content.js b/server/src/lib/page-content.jsindex 038d6b988..813c89643 100644--- a/server/src/lib/page-content.js+++ b/server/src/lib/page-content.js@@ -3,6 +3,15 @@* content › Page URL): its main text without the site's frame, its title,* and its schema JSON-LD. The editor cleans it again, as it does a paste;* this only picks what belongs to the page.+ *+ * Many pages have no <main> or <article> (Webflow, page builders), and the+ * ones that do often keep an author box, share buttons or related posts in+ * it. So the text is found where the page's paragraphs are, the root is the+ * part of the page holding it and its H1, and the furniture around the text+ * is taken out: blocks named for it (author, share, related…), blocks that+ * are mostly links (breadcrumbs, a table of contents, icon rows, post+ * lists), bylines ("8 min read"), and avatars, logos, icons and cover+ * images.*/import * as cheerio from 'cheerio';@@ -17,14 +26,229 @@ const MAX_JSON_LD_LENGTH = 50_000;/** Where a page's own content usually is, the most telling first. */const MAIN_SELECTORS = ['main', 'article', '[role="main"]', '#content', '#main'];-/** The site's frame and what can't be content. */+/** The site's frame, and what can't be content. */const FRAME =- 'script, style, noscript, template, iframe, object, embed, form, button, input, select, textarea, svg, canvas, nav, header, footer, aside, [role="navigation"], [role="banner"], [role="contentinfo"], [aria-hidden="true"]';+ 'script, style, noscript, template, iframe, object, embed, form, button, input, select, textarea, svg, canvas, nav, header, footer, aside, [role="navigation"], [role="banner"], [role="contentinfo"], [aria-hidden="true"], .w-condition-invisible';++/**+ * What the page hides, taken out around the text only: a collapsed FAQ+ * answer ships hidden, and it is just the text an AEO page needs. A block+ * hidden until found (the browser's search opens it) is never taken out.+ */+const HIDDEN =+ '[hidden]:not([hidden="until-found"]), [style*="display:none"], [style*="display: none"]';++/**+ * Words in a block's class or id that name furniture: anywhere, even in the+ * text (an author box or share buttons in the article)…+ */+const FURNITURE_WORDS = new Set([+ 'ad',+ 'ads',+ 'advert',+ 'author',+ 'authors',+ 'avatar',+ 'bio',+ 'breadcrumb',+ 'breadcrumbs',+ 'byline',+ 'comment',+ 'comments',+ 'consent',+ 'cookie',+ 'modal',+ 'newsletter',+ 'popup',+ 'recommended',+ 'related',+ 'share',+ 'sharing',+ 'social',+ 'sponsor',+ 'sponsored',+ 'subscribe',+ 'toc',+]);++/** …and outside the text only, as the article's own call to action is its text. */+const PAGE_FURNITURE_WORDS = new Set([+ ...FURNITURE_WORDS,+ 'announcement',+ 'cta',+ 'footer',+ 'header',+ 'menu',+ 'meta',+ 'nav',+ 'navbar',+ 'pager',+ 'pagination',+ 'promo',+ 'sidebar',+ 'signup',+ 'tag',+ 'tags',+ 'widget',+]);++/**+ * Words in an image's (or its block's) class or id that mark it as no part+ * of the text: anywhere…+ */+const IMAGE_FURNITURE_WORDS = new Set([+ 'author',+ 'avatar',+ 'badge',+ 'emoji',+ 'gravatar',+ 'icon',+ 'logo',+ 'profile',+ 'sprite',+]);++/**+ * …and outside the text, the post's cover. ("cover" itself is no sign:+ * Tailwind's object-cover is on any picture.)+ */+const PAGE_IMAGE_FURNITURE_WORDS = new Set([+ ...IMAGE_FURNITURE_WORDS,+ 'featured',+ 'hero',+ 'thumb',+ 'thumbnail',+]);++/** A line about the post rather than of it. */+const BYLINE =+ /\b\d+\s*(?:min|mins|minute|minutes)\s+read\b|\breading time\b|\blast updated\b|\bupdated on\b|\bpublished (?:on|by)\b|\bposted (?:on|by)\b|\bwritten by\b/i;++/** Paragraphs shorter than this don't say where the text is. */+const MIN_PARAGRAPH = 25;+/** The text area holds at least this share of the page's paragraph text. */+const TEXT_AREA_SHARE = 0.6;+/** A furniture block is never this big a share of the root's text. */+const MAX_FURNITURE_SHARE = 0.5;+/**+ * Outside the text, a block with at most this much text besides its links+ * is furniture (breadcrumbs, icon rows, a few related posts), as is a+ * table of contents (links within the page) …+ */+const MAX_LINK_LIST_TEXT = 50;+/**+ * … but a list going to this many pages, or a long block mostly of links,+ * is the page's own (a category's products) and is kept whole.+ */+const MIN_LIST_TARGETS = 8;+const MAX_LINK_LIST = 600;+const LIST_LINK_SHARE = 0.3;+/** A byline is a short line. */+const MAX_BYLINE = 200;+/** Images this small (width or height, in pixels) are icons. */+const MAX_ICON_SIZE = 64;function clean(text) {return (text ?? '').replace(/\s+/g, ' ').trim();}+/** A block's class and id, as lower-case words: "authorBox ans-trk__i" → author, box, ans, trk, i. */+function wordsOf($el) {+ const names = `${$el.attr('class') ?? ''} ${$el.attr('id') ?? ''}`;+ return names+ .replace(/([a-z0-9])([A-Z])/g, '$1 $2')+ .toLowerCase()+ .split(/[^a-z0-9]+/)+ .filter(Boolean);+}++function named($el, words) {+ return wordsOf($el).some((word) => words.has(word));+}++/** An element's text, with a space where its blocks meet ("Ada" "8 min read" → "Ada 8 min read"). */+function textOf($, el) {+ return clean(+ $(el)+ .find('*')+ .addBack()+ .contents()+ .filter((_, node) => node.type === 'text')+ .map((_, node) => node.data)+ .get()+ .join(' '),+ );+}++/**+ * Where the page's text is: paragraphs score their block and, half, the one+ * above it (as reader modes do), and the text area is the best block,+ * widened until it holds most of the page's paragraph text. Null when the+ * page has no paragraphs to go by.+ */+function findTextArea($, body) {+ const scores = new Map();+ let total = 0;+ // An author's bio or a comment is no sign of where the text is, nor is+ // what the page hides: the text is where its visible paragraphs are, and+ // a hidden answer inside it stays (HIDDEN).+ const paragraphs = body+ .find('p, pre, blockquote, li')+ .toArray()+ .filter(+ (el) =>+ clean($(el).text()).length >= MIN_PARAGRAPH &&+ $(el).closest(HIDDEN).length === 0 &&+ !$(el)+ .parents()+ .toArray()+ .some((block) => named($(block), PAGE_FURNITURE_WORDS)),+ );+ const counted = new Set(paragraphs);+ paragraphs.forEach((el) => {+ const length = clean($(el).text()).length;+ total += length;+ const score =+ 1 + (clean($(el).text()).match(/,/g)?.length ?? 0) + Math.min(3, Math.floor(length / 100));+ const parent = el.parent;+ if (!parent || parent.type !== 'tag') return;+ scores.set(parent, (scores.get(parent) ?? 0) + score);+ const grandparent = parent.parent;+ if (grandparent && grandparent.type === 'tag') {+ scores.set(grandparent, (scores.get(grandparent) ?? 0) + score / 2);+ }+ });+ let best = null;+ for (const [el, score] of scores) {+ if (!best || score > best.score) best = { el, score };+ }+ if (!best) return null;++ const paragraphText = (el) =>+ $(el)+ .find('p, pre, blockquote, li')+ .toArray()+ .reduce((sum, p) => (counted.has(p) ? sum + clean($(p).text()).length : sum), 0);+ let area = best.el;+ while (+ paragraphText(area) < total * TEXT_AREA_SHARE &&+ area.parent &&+ area.parent.type === 'tag' &&+ area.parent !== body[0]+ ) {+ area = area.parent;+ }+ return area;+}++/** The closest block holding both elements. */+function commonAncestor($, a, b) {+ const above = new Set([a, ...$(a).parents().toArray()]);+ let el = b;+ while (el && !above.has(el)) el = el.parent;+ return el;+}+/*** @param {string} html the page as fetched* @param {string} pageUrl the page's address after redirects, for its links@@ -43,9 +267,115 @@ export function extractPageContent(html, pageUrl) {const title =h1 || clean($('meta[property="og:title"]').attr('content')) || clean($('title').first().text());- const main = MAIN_SELECTORS.map((selector) => $(selector).first()).find((el) => el.length);- const root = main ?? $('body');- root.find(FRAME).remove();+ const body = $('body');+ body.find(FRAME).remove();++ // The root: the page's main block when it holds the text; else the part+ // of the page holding the text and its H1; else the text area alone.+ const textArea = findTextArea($, body);+ body+ .find(HIDDEN)+ .filter(+ (_, el) =>+ !textArea || ($(el).closest(textArea).length === 0 && !$(el).find(textArea).length),+ )+ .remove();+ const heading = body.find('h1').first()[0] ?? null;+ const main = MAIN_SELECTORS.map((selector) => $(selector).toArray())+ .flat()+ .find((el) => !textArea || el === textArea || $(el).find(textArea).length > 0);+ const root = main+ ? $(main)+ : textArea+ ? $(heading ? commonAncestor($, heading, textArea) : textArea)+ : body;++ // What stays whatever its name: the text area, the blocks around it and+ // the H1's.+ const kept = new Set();+ for (const el of [textArea, heading]) {+ if (!el) continue;+ kept.add(el);+ for (const parent of $(el).parents().toArray()) kept.add(parent);+ }+ const rootLength = clean(root.text()).length;++ // 'remove' for furniture; 'list' for a long list of links outside the+ // text (a category's products), kept and read as the text is.+ const judge = (el, inContent) => {+ const $el = $(el);+ const text = textOf($, el);+ const toc = `${$el.attr('class') ?? ''} ${$el.attr('id') ?? ''}`+ .toLowerCase()+ .includes('table-of-contents');+ if (+ (named($el, inContent ? FURNITURE_WORDS : PAGE_FURNITURE_WORDS) || toc) &&+ text.length < rootLength * MAX_FURNITURE_SHARE+ ) {+ return 'remove';+ }+ // In the text, lists of links and short lines are its own; a page with+ // no paragraphs to go by keeps them too.+ if (!textArea || inContent) return null;+ const links = ($el.is('a') ? $el : $el.find('a')).toArray();+ if (links.length >= 2) {+ const linkText = links.reduce((sum, a) => sum + textOf($, a).length, 0);+ const hrefs = links.map((a) => $(a).attr('href') ?? '');+ // A table of contents: links within the page.+ if (hrefs.filter((href) => href.startsWith('#')).length * 2 > links.length) return 'remove';+ if (+ new Set(hrefs).size >= MIN_LIST_TARGETS ||+ (text.length > MAX_LINK_LIST && linkText >= text.length * LIST_LINK_SHARE)+ ) {+ return 'list';+ }+ if (text.length - linkText <= MAX_LINK_LIST_TEXT) return 'remove';+ }+ return text.length <= MAX_BYLINE && BYLINE.test(text) ? 'remove' : null;+ };+ const lists = new Set();+ const strip = (parent, inContent) => {+ $(parent)+ .children()+ .each((_, el) => {+ const content = inContent || el === textArea;+ if (kept.has(el)) {+ strip(el, content);+ return;+ }+ const verdict = judge(el, content);+ if (verdict === 'remove') {+ $(el).remove();+ } else if (verdict === 'list') {+ lists.add(el);+ strip(el, true);+ } else {+ strip(el, content);+ }+ });+ };+ strip(root[0], root[0] === textArea);++ // Avatars, logos, icons and cover images aren't the text's; a list's+ // pictures (its products) are its own.+ root.find('img').each((_, el) => {+ const $el = $(el);+ const small = ['width', 'height'].some((side) => {+ const size = Number($el.attr(side));+ return Number.isFinite(size) && size > 0 && size <= MAX_ICON_SIZE;+ });+ const above = $el.parentsUntil(root).toArray();+ // In the text when the root is the text area itself, as strip has it.+ const words =+ textArea && (root[0] === textArea || above.includes(textArea))+ ? IMAGE_FURNITURE_WORDS+ : PAGE_IMAGE_FURNITURE_WORDS;+ // Its own name or its block's, not the text's or the H1's wrappers.+ const marked =+ !above.some((node) => lists.has(node)) &&+ [el, ...above].some((node) => !kept.has(node) && named($(node), words));+ if (small || marked) $el.remove();+ });// Links and images keep working outside the page: relative addresses// become absolute; anything else the editor's cleaning decides on.diff --git a/server/src/lib/page-content.test.js b/server/src/lib/page-content.test.jsindex 09d7382e9..391bbb0b5 100644--- a/server/src/lib/page-content.test.js+++ b/server/src/lib/page-content.test.js@@ -65,4 +65,176 @@ describe('extractPageContent', () => {expect(title).toBe('Lisbon & Porto');expect(html).toBe('<h1>Lisbon & Porto</h1><p>Days out.</p>');});++ it('takes an article’s author box, byline, share buttons, related posts and comments out', () => {+ const { html } = extractPageContent(+ `<body><main><article>+ <header class="entry-header"><h1>Cold brew at home</h1></header>+ <div class="entry-meta">By Ada · 5 min read</div>+ <div class="entry-content">+ <p>Cold brew takes time, coarse grounds and cold water, nothing more.</p>+ <img src="/img/jar.jpg" alt="A jar of cold brew">+ <p>Steep it for twelve to eighteen hours, then strain it twice.</p>+ <div class="sharedaddy sd-sharing"><a href="https://x.com/share">Share on X</a><a href="https://facebook.com/share">Share on Facebook</a></div>+ </div>+ <div class="author-box"><img class="avatar" src="/ada.jpg" alt="Ada" width="96"><p>Ada writes about coffee, slowly and with care.</p></div>+ </article>+ <section class="related-posts"><h3>Related</h3><a href="/espresso">Espresso basics</a><a href="/latte">Latte art</a></section>+ <div id="comments"><p>Great post, thanks for sharing the recipe with us!</p></div>+ </main></body>`,+ 'https://coffee.example/cold-brew',+ );+ expect(html).toContain('<h1>Cold brew at home</h1>');+ expect(html).toContain('coarse grounds and cold water');+ expect(html).toContain('src="https://coffee.example/img/jar.jpg"');+ for (const gone of [+ '5 min read',+ 'Share on',+ 'Ada writes',+ 'ada.jpg',+ 'Espresso basics',+ 'Great post',+ ]) {+ expect(html).not.toContain(gone);+ }+ });++ it('finds the text on a page with no <main>, and keeps its H1 and lead but not the furniture', () => {+ const { html } = extractPageContent(+ `<body>+ <div class="announcement-bar"><a href="/trial">Stop measuring. Start improving.</a></div>+ <div class="page-wrapper"><section class="blog-hero"><div class="container">+ <div class="hero-content">+ <div class="columns"><a href="/blog">← Explore all</a><a href="/blog/ai">AI Search</a></div>+ <div class="blog-tag">AI Search</div>+ <h1>AI search monitoring</h1>+ <div class="blog-intro-text">Run one shared prompt set on every engine each week, and report each one apart.</div>+ <div class="div-block-7"><div>Cihan Geyik</div><img src="/eye.png" alt=""><div>8 min read</div><div>Last Updated:</div><div>October 21, 2026</div></div>+ <div class="blog-single-hero-img"><img class="all-img" src="/cihan.jpg" alt="Cihan"></div>+ </div>+ <div class="blog-details">+ <div class="blog-single-right">Explore with AI<a href="https://chatgpt.com/?q=x"><img src="/chatgpt.svg" alt="ChatGPT"></a><a href="https://claude.ai/new?q=x"><img src="/claude.svg" alt="Claude"></a>In this article<a href="#step-1">Step 1</a><a href="#step-2">Step 2</a></div>+ <div class="rich-text">+ <p>Many teams check ChatGPT one day, Google the next and Gemini when someone asks.</p>+ <figure><img src="/prompts.png" alt="The prompt list"></figure>+ <p>Record mentions, cited addresses and competitors, the same way every week.</p>+ <div class="ai-monitor-cta"><h2>How Ansvisor helps</h2><p>Our platform runs one prompt set across every engine and keeps every answer.</p></div>+ <div class="qoute-box"><img class="qoute-icon" src="/quote.svg" alt=""><p>Monitoring should explain why visibility changes, not only that it did.</p></div>+ </div>+ </div>+ <div class="author-box"><img src="/cihan-small.jpg" alt="Cihan"><h3>Cihan Geyik</h3><p>Cihan is the co-founder of Ansvisor, an open-source platform.</p></div>+ <div><a href="/blog">← Explore all</a><h3>Related Blog Posts</h3><a href="/blog/crawlers">How to check AI crawler access</a><a href="/blog/refresh">How to refresh content for AI search</a></div>+ </div></section>+ <section class="preferred-source"><p>Get more Ansvisor insights in Google, add us as a preferred source.</p><a href="https://google.com/preferences">Add Ansvisor</a></section>+ <div class="cta-footer"><p>Make your brand visible in AI answers, start a trial today.</p></div>+ </div>+ </body>`,+ 'https://www.ansvisor.com/blog/ai-search-monitoring',+ );+ for (const kept of [+ '<h1>AI search monitoring</h1>',+ 'Run one shared prompt set',+ 'Many teams check ChatGPT',+ 'src="https://www.ansvisor.com/prompts.png"',+ // The article's own call to action is its text.+ 'How Ansvisor helps',+ 'Monitoring should explain why',+ ]) {+ expect(html).toContain(kept);+ }+ for (const gone of [+ 'Stop measuring',+ 'Explore all',+ 'blog-tag',+ '8 min read',+ 'eye.png',+ 'cihan.jpg',+ 'Explore with AI',+ 'chatgpt.svg',+ 'Step 1',+ 'quote.svg',+ 'co-founder',+ 'Related Blog Posts',+ 'preferred source',+ 'start a trial today',+ ]) {+ expect(html).not.toContain(gone);+ }+ });++ it('keeps a category’s long list of products, with their pictures', () => {+ const products = Array.from(+ { length: 12 },+ (_, i) =>+ `<div class="product-card"><a href="/p/${i}"><img class="product-thumb" src="/p${i}.jpg" alt="Runner ${i}"></a><a href="/p/${i}">Wool Runner number ${i}</a><span>$98</span></div>`,+ ).join('');+ const { html } = extractPageContent(+ `<body><main><h1>Men’s shoes</h1>+ <div class="collection-description"><p>Sustainable, supportive and wildly comfortable sneakers, ready when you are.</p></div>+ <div class="product-grid">${products}</div>+ </main></body>`,+ 'https://shoes.example/collections/men',+ );+ expect(html).toContain('Wool Runner number 11');+ expect(html).toContain('src="https://shoes.example/p11.jpg"');+ expect(html).toContain('wildly comfortable sneakers');+ });++ it('drops what the page hides and tiny icons, but never the text for its name', () => {+ const { html } = extractPageContent(+ `<body><main><h1>Dinner menu</h1>+ <div class="menu-items">+ <p>Grilled sea bass with lemon, capers and new potatoes, our house dish.</p>+ <p>Slow roasted lamb shoulder with rosemary, garlic and white beans.</p>+ <img src="/star.png" width="16" height="16" alt="">+ <img class="h-full w-full object-cover" src="/bass.jpg" alt="Sea bass">+ </div>+ <div class="w-condition-invisible"><p>No dishes are on the menu this evening, sorry.</p></div>+ <p hidden>Hidden note for the kitchen staff only.</p>+ </main></body>`,+ 'https://bistro.example/menu',+ );+ expect(html).toContain('Grilled sea bass');+ expect(html).toContain('src="https://bistro.example/bass.jpg"');+ for (const gone of ['star.png', 'No dishes', 'Hidden note']) {+ expect(html).not.toContain(gone);+ }+ });++ it('keeps a collapsed FAQ answer in the text, and hides what is hidden around it', () => {+ const { html } = extractPageContent(+ `<body><div hidden class="mobile-menu"><p>Shop all our coffee subscriptions and gifts today.</p></div>+ <main><h1>Coffee subscription FAQ</h1>+ <div class="faq">+ <p>Answers to what customers ask most about deliveries, plans and pausing.</p>+ <h3>Can I pause my plan?</h3>+ <div hidden><p>Yes, pause or skip any delivery from your account, up to two days before it ships.</p></div>+ <h3>Do you ship abroad?</h3>+ <div hidden="until-found"><p>We ship to the EU and the UK, with tracked delivery in three to five days.</p></div>+ <h3>Is grinding free?</h3>+ <div style="display: none"><p>Grinding is free: pick whole bean, espresso, filter or French press.</p></div>+ </div>+ </main></body>`,+ 'https://acme-coffee.example/faq',+ );+ for (const kept of ['pause or skip', 'We ship to the EU', 'Grinding is free']) {+ expect(html).toContain(kept);+ }+ expect(html).toContain('hidden="until-found"');+ expect(html).not.toContain('Shop all our coffee');+ });++ it('judges images in the text as the text’s when the text area is the root', () => {+ const { html } = extractPageContent(+ `<body><article>+ <p>Our guide to brewing a better cup at home, with the gear you already own.</p>+ <p>Start with fresh beans, a scale and water just off the boil, then time it.</p>+ <img class="size-thumbnail" src="/pour-over.jpg" alt="Pour-over">+ <img class="avatar" src="/ada.jpg" alt="Ada">+ </article></body>`,+ 'https://acme-coffee.example/guides/brew',+ );+ expect(html).toContain('src="https://acme-coffee.example/pour-over.jpg"');+ expect(html).not.toContain('ada.jpg');+ });});