Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: make HTML-to-text and feed tag extraction linear on hostile input

Adam Eivyf6908b249a4c
diff --git a/server/lib/htmlToText.js b/server/lib/htmlToText.js
index 1681756841..cecc3dc48d 100644
--- a/server/lib/htmlToText.js
+++ b/server/lib/htmlToText.js
@@ -35,10 +35,41 @@ import { decodeXmlEntities } from './xmlEntities.js';
* @param {boolean} [options.collapseSpaces=false] - Collapse space/tab runs.
* @returns {string} Readable plain text, trimmed.
*/
+const CONTAINER_OPEN_RE = /<(script|style|head|noscript)\b/gi;
+
+// Remove each container element with its content. A lazy `[\s\S]*?` between
+// the open and close tags rescans to the end of the document for every open
+// tag that has no close, which is quadratic on hostile HTML (inbound mail from
+// any sender reaches this): 200 KB of bare `<head>` stalled the loop for
+// seconds. Index scanning keeps the old semantics — an unclosed container is
+// left in place — in linear time.
+function stripContainerElements(html) {
+ const lower = html.toLowerCase();
+ // Next close position per tag, so a run of unclosed opens does not rescan
+ // to the end for each one (-1 = no close anywhere after the cached search).
+ const nextClose = new Map();
+ let out = '';
+ let cursor = 0;
+ CONTAINER_OPEN_RE.lastIndex = 0;
+ for (let m = CONTAINER_OPEN_RE.exec(html); m !== null; m = CONTAINER_OPEN_RE.exec(html)) {
+ const tag = m[1].toLowerCase();
+ const from = m.index + m[0].length;
+ let close = nextClose.get(tag);
+ if (close === undefined || (close !== -1 && close < from)) {
+ close = lower.indexOf(`</${tag}>`, from);
+ nextClose.set(tag, close);
+ }
+ if (close === -1) continue;
+ out += html.slice(cursor, m.index);
+ cursor = close + tag.length + 3;
+ CONTAINER_OPEN_RE.lastIndex = cursor;
+ }
+ return out + html.slice(cursor);
+}
+
export function htmlToText(html, { extraEntities, paragraphBreak = '\n', collapseSpaces = false } = {}) {
if (typeof html !== 'string') return '';
- const stripped = html
- .replace(/<(script|style|head|noscript)\b[\s\S]*?<\/\1>/gi, '')
+ const stripped = stripContainerElements(html)
.replace(/<br\s*\/?>/gi, '\n')
.replace(/<\/p>/gi, paragraphBreak)
.replace(/<\/(div|li|tr|h[1-6])>/gi, '\n')
diff --git a/server/lib/htmlToText.test.js b/server/lib/htmlToText.test.js
index cd602476ea..4b657591d5 100644
--- a/server/lib/htmlToText.test.js
+++ b/server/lib/htmlToText.test.js
@@ -50,3 +50,18 @@ describe('htmlToText', () => {
expect(htmlToText(42)).toBe('');
});
});
+
+describe('htmlToText on hostile container tags', () => {
+ it('keeps the old semantics for unclosed and nested containers', () => {
+ expect(htmlToText('<head>a<head>b</head>c')).toBe('c');
+ expect(htmlToText('<p>before</p><style>unclosed body text')).toBe('before\nunclosed body text');
+ expect(htmlToText('<SCRIPT>x</script>kept<Style>y</STYLE>')).toBe('kept');
+ });
+
+ it('stays linear on a document made of unclosed container tags', () => {
+ const html = '<head>'.repeat(200_000) + '<p>tail</p>';
+ const started = performance.now();
+ expect(htmlToText(html)).toBe('tail');
+ expect(performance.now() - started).toBeLessThan(1_000);
+ });
+});
diff --git a/server/services/feeds.js b/server/services/feeds.js
index 69d3cc4d72..172b73dc02 100644
--- a/server/services/feeds.js
+++ b/server/services/feeds.js
@@ -69,12 +69,33 @@ async function ensureItemsSorted() {
/**
* Extract text content from an XML tag (non-greedy, handles CDATA).
*/
+// Index scanning rather than a regex: the old `\s*(?:CDATA)?([\s\S]*?)\s*</tag>`
+// pattern backtracked catastrophically when a fetched feed opened a tag and
+// never closed it (a 2 KB `<title>` plus spaces took 13 s), and feeds refresh
+// on a schedule from servers PortOS does not control.
const extractTag = (xml, tag) => {
- const re = new RegExp(`<${tag}[^>]*>\\s*(?:<!\\[CDATA\\[)?([\\s\\S]*?)(?:\\]\\]>)?\\s*</${tag}>`, 'i');
- const match = xml.match(re);
- return match ? match[1].trim() : '';
+ const lower = xml.toLowerCase();
+ const open = `<${tag.toLowerCase()}`;
+ const close = `</${tag.toLowerCase()}>`;
+ for (let from = 0; ; ) {
+ const start = lower.indexOf(open, from);
+ if (start === -1) return '';
+ const after = start + open.length;
+ const boundary = xml[after];
+ if (boundary !== undefined && boundary !== '>' && boundary !== '/' && !/\s/.test(boundary)) { from = after; continue; }
+ const gt = xml.indexOf('>', after);
+ if (gt === -1) return '';
+ const end = lower.indexOf(close, gt + 1);
+ if (end === -1) return '';
+ let inner = xml.slice(gt + 1, end).trim();
+ if (inner.startsWith('<![CDATA[')) inner = inner.slice('<![CDATA['.length);
+ if (inner.endsWith(']]>')) inner = inner.slice(0, -']]>'.length);
+ return inner.trim();
+ }
};
+export const __testing = { extractTag };
+
/**
* Extract href from an Atom <link> tag.
*/
diff --git a/server/services/feeds.test.js b/server/services/feeds.test.js
index fce8116847..ee4169e774 100644
--- a/server/services/feeds.test.js
+++ b/server/services/feeds.test.js
@@ -504,3 +504,21 @@ describe('getFeedStats', () => {
expect(stats.topUnread[0].unread).toBe(2);
});
});
+
+describe('extractTag', () => {
+ it('reads attributed, CDATA-wrapped and case-folded tags like the regex did', () => {
+ const { extractTag } = feeds.__testing;
+ expect(extractTag('<title type="html"> Hello </title>', 'title')).toBe('Hello');
+ expect(extractTag('<title>\n<![CDATA[ A & B ]]>\n</TITLE>', 'title')).toBe('A & B');
+ expect(extractTag('<titles>no</titles><title>yes</title>', 'title')).toBe('yes');
+ expect(extractTag('<dc:creator>Ann</dc:creator>', 'dc:creator')).toBe('Ann');
+ expect(extractTag('<title>never closed', 'title')).toBe('');
+ });
+
+ it('does not backtrack on an unclosed tag padded with whitespace', () => {
+ const { extractTag } = feeds.__testing;
+ const started = performance.now();
+ expect(extractTag('<title>' + ' '.repeat(200_000), 'title')).toBe('');
+ expect(performance.now() - started).toBeLessThan(500);
+ });
+});