Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: make HTML-to-text and feed tag extraction linear on hostile input
Adam Eivyf6908b249a4c
diff --git a/server/lib/htmlToText.js b/server/lib/htmlToText.jsindex 1681756841..cecc3dc48d 100644--- a/server/lib/htmlToText.js+++ b/server/lib/htmlToText.js@@ -35,10 +35,41 @@ import { decodeXmlEntities } from './xmlEntities.js';* @param {boolean} [options.collapseSpaces=false] - Collapse space/tab runs.* @returns {string} Readable plain text, trimmed.*/+const CONTAINER_OPEN_RE = /<(script|style|head|noscript)\b/gi;++// Remove each container element with its content. A lazy `[\s\S]*?` between+// the open and close tags rescans to the end of the document for every open+// tag that has no close, which is quadratic on hostile HTML (inbound mail from+// any sender reaches this): 200 KB of bare `<head>` stalled the loop for+// seconds. Index scanning keeps the old semantics — an unclosed container is+// left in place — in linear time.+function stripContainerElements(html) {+ const lower = html.toLowerCase();+ // Next close position per tag, so a run of unclosed opens does not rescan+ // to the end for each one (-1 = no close anywhere after the cached search).+ const nextClose = new Map();+ let out = '';+ let cursor = 0;+ CONTAINER_OPEN_RE.lastIndex = 0;+ for (let m = CONTAINER_OPEN_RE.exec(html); m !== null; m = CONTAINER_OPEN_RE.exec(html)) {+ const tag = m[1].toLowerCase();+ const from = m.index + m[0].length;+ let close = nextClose.get(tag);+ if (close === undefined || (close !== -1 && close < from)) {+ close = lower.indexOf(`</${tag}>`, from);+ nextClose.set(tag, close);+ }+ if (close === -1) continue;+ out += html.slice(cursor, m.index);+ cursor = close + tag.length + 3;+ CONTAINER_OPEN_RE.lastIndex = cursor;+ }+ return out + html.slice(cursor);+}+export function htmlToText(html, { extraEntities, paragraphBreak = '\n', collapseSpaces = false } = {}) {if (typeof html !== 'string') return '';- const stripped = html- .replace(/<(script|style|head|noscript)\b[\s\S]*?<\/\1>/gi, '')+ const stripped = stripContainerElements(html).replace(/<br\s*\/?>/gi, '\n').replace(/<\/p>/gi, paragraphBreak).replace(/<\/(div|li|tr|h[1-6])>/gi, '\n')diff --git a/server/lib/htmlToText.test.js b/server/lib/htmlToText.test.jsindex cd602476ea..4b657591d5 100644--- a/server/lib/htmlToText.test.js+++ b/server/lib/htmlToText.test.js@@ -50,3 +50,18 @@ describe('htmlToText', () => {expect(htmlToText(42)).toBe('');});});++describe('htmlToText on hostile container tags', () => {+ it('keeps the old semantics for unclosed and nested containers', () => {+ expect(htmlToText('<head>a<head>b</head>c')).toBe('c');+ expect(htmlToText('<p>before</p><style>unclosed body text')).toBe('before\nunclosed body text');+ expect(htmlToText('<SCRIPT>x</script>kept<Style>y</STYLE>')).toBe('kept');+ });++ it('stays linear on a document made of unclosed container tags', () => {+ const html = '<head>'.repeat(200_000) + '<p>tail</p>';+ const started = performance.now();+ expect(htmlToText(html)).toBe('tail');+ expect(performance.now() - started).toBeLessThan(1_000);+ });+});diff --git a/server/services/feeds.js b/server/services/feeds.jsindex 69d3cc4d72..172b73dc02 100644--- a/server/services/feeds.js+++ b/server/services/feeds.js@@ -69,12 +69,33 @@ async function ensureItemsSorted() {/*** Extract text content from an XML tag (non-greedy, handles CDATA).*/+// Index scanning rather than a regex: the old `\s*(?:CDATA)?([\s\S]*?)\s*</tag>`+// pattern backtracked catastrophically when a fetched feed opened a tag and+// never closed it (a 2 KB `<title>` plus spaces took 13 s), and feeds refresh+// on a schedule from servers PortOS does not control.const extractTag = (xml, tag) => {- const re = new RegExp(`<${tag}[^>]*>\\s*(?:<!\\[CDATA\\[)?([\\s\\S]*?)(?:\\]\\]>)?\\s*</${tag}>`, 'i');- const match = xml.match(re);- return match ? match[1].trim() : '';+ const lower = xml.toLowerCase();+ const open = `<${tag.toLowerCase()}`;+ const close = `</${tag.toLowerCase()}>`;+ for (let from = 0; ; ) {+ const start = lower.indexOf(open, from);+ if (start === -1) return '';+ const after = start + open.length;+ const boundary = xml[after];+ if (boundary !== undefined && boundary !== '>' && boundary !== '/' && !/\s/.test(boundary)) { from = after; continue; }+ const gt = xml.indexOf('>', after);+ if (gt === -1) return '';+ const end = lower.indexOf(close, gt + 1);+ if (end === -1) return '';+ let inner = xml.slice(gt + 1, end).trim();+ if (inner.startsWith('<![CDATA[')) inner = inner.slice('<![CDATA['.length);+ if (inner.endsWith(']]>')) inner = inner.slice(0, -']]>'.length);+ return inner.trim();+ }};+export const __testing = { extractTag };+/*** Extract href from an Atom <link> tag.*/diff --git a/server/services/feeds.test.js b/server/services/feeds.test.jsindex fce8116847..ee4169e774 100644--- a/server/services/feeds.test.js+++ b/server/services/feeds.test.js@@ -504,3 +504,21 @@ describe('getFeedStats', () => {expect(stats.topUnread[0].unread).toBe(2);});});++describe('extractTag', () => {+ it('reads attributed, CDATA-wrapped and case-folded tags like the regex did', () => {+ const { extractTag } = feeds.__testing;+ expect(extractTag('<title type="html"> Hello </title>', 'title')).toBe('Hello');+ expect(extractTag('<title>\n<![CDATA[ A & B ]]>\n</TITLE>', 'title')).toBe('A & B');+ expect(extractTag('<titles>no</titles><title>yes</title>', 'title')).toBe('yes');+ expect(extractTag('<dc:creator>Ann</dc:creator>', 'dc:creator')).toBe('Ann');+ expect(extractTag('<title>never closed', 'title')).toBe('');+ });++ it('does not backtrack on an unclosed tag padded with whitespace', () => {+ const { extractTag } = feeds.__testing;+ const started = performance.now();+ expect(extractTag('<title>' + ' '.repeat(200_000), 'title')).toBe('');+ expect(performance.now() - started).toBeLessThan(500);+ });+});