Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(news): decode gzip at the SSRF guard, drop rung-comment dashes, and cover the lane ladder order so archive and EDGAR lanes read real bodies

nirholase1de07deae6e
diff --git a/api/_lib/article-extract.js b/api/_lib/article-extract.js
index c1c59a256..729cb4c50 100644
--- a/api/_lib/article-extract.js
+++ b/api/_lib/article-extract.js
@@ -239,11 +239,65 @@ export async function extractArticle(url, { feedContentText = null, title = null
readerError = err;
}
- // A partial direct page (2 paragraphs) still beats the feed teaser.
+ // Rung 3: Wayback. The same article as it was archived: strictly better than
+ // a partial page, so it runs for any page thinner than a full read.
+ let laneError = null;
+ try {
+ const { paragraphs: archived, snapshot, html: archivedHtml } = await fetchViaWayback(url, extractParagraphs);
+ if (archived.length >= 3 && archived.length > paragraphs.length) {
+ return {
+ paragraphs: archived,
+ extraction: 'wayback',
+ blocked_reason: null,
+ // The snapshot's own metadata (headline, image, byline) stands in
+ // when the live page was blocked.
+ html: html || archivedHtml,
+ lane_sources: [{ label: 'Internet Archive snapshot', url: snapshot }],
+ };
+ }
+ } catch (err) {
+ laneError = err;
+ }
+
+ // A partial direct page (2 paragraphs) still beats the substitutes below.
if (paragraphs.length >= 2) {
return { paragraphs, extraction: 'page', blocked_reason: null, html, lane_sources: [] };
}
+ // Rung 4: primary document: the SEC filing the headline's company made.
+ if (title) {
+ try {
+ const { paragraphs: filed, filing } = await fetchPrimaryFiling(title, publishedAt, extractParagraphs);
+ if (filed.length >= 2) {
+ return {
+ paragraphs: filed,
+ extraction: 'primary',
+ blocked_reason: null,
+ html,
+ lane_sources: [{ label: `${filing.filer} ${filing.form} filed ${filing.filed}`, url: filing.url }],
+ };
+ }
+ } catch (err) {
+ laneError = laneError || err;
+ }
+
+ // Rung 5: sibling coverage from other outlets.
+ try {
+ const { paragraphs: sibling, siblings } = await fetchSiblings(title, url);
+ if (siblings.length >= 2) {
+ return {
+ paragraphs: sibling,
+ extraction: 'siblings',
+ blocked_reason: null,
+ html,
+ lane_sources: siblings.map((s) => ({ label: s.source, url: s.link })),
+ };
+ }
+ } catch (err) {
+ laneError = laneError || err;
+ }
+ }
+
// Rung 6, publisher feed body, when it is more than a teaser.
if (feedContentText) {
const feedParas = paragraphsFromFeed(feedContentText);
diff --git a/api/_lib/article-lanes.js b/api/_lib/article-lanes.js
index d93ebe46a..3e6b9f375 100644
--- a/api/_lib/article-lanes.js
+++ b/api/_lib/article-lanes.js
@@ -90,13 +90,13 @@ export async function fetchViaWayback(url, extractParagraphs) {
['json'],
);
const raw = waybackRawUrl(JSON.parse(lookup)?.archived_snapshots?.closest, url);
- if (!raw) return { paragraphs: [], snapshot: null };
+ if (!raw) return { paragraphs: [], snapshot: null, html: null };
const html = await fetchText(
raw,
{ accept: 'text/html,application/xhtml+xml', 'user-agent': LANE_USER_AGENT },
['text/html', 'application/xhtml+xml'],
);
- return { paragraphs: extractParagraphs(html), snapshot: raw };
+ return { paragraphs: extractParagraphs(html), snapshot: raw, html };
}
// ── Primary documents: SEC EDGAR ───────────────────────────────────────────
@@ -115,6 +115,9 @@ function secSlot() {
return turn;
}
+// SEC cover-page and signature furniture that leads every filing.
+const FILING_FURNITURE = /securities and exchange commission|^form (8-k|10-k|10-q|s-1)\b|check the appropriate box|commission file number|state or other jurisdiction|\(address of principal|emerging growth company|indicate by check mark|pursuant to section 13 or 15\(d\)|date of report|written communications|soliciting material|pre-commencement|rule 1[34][ade]-|pursuant to the requirements of the securities exchange act/i;
+
const filingCache = new Map(); // accession -> paragraphs
const FILING_CACHE_MAX = 128;
@@ -192,7 +195,7 @@ export async function fetchPrimaryFiling(title, publishedAt, extractParagraphs,
if (!paragraphs) {
await secSlot();
const html = await fetchText(filing.url, { accept: 'text/html', 'user-agent': LANE_USER_AGENT }, ['text/html']);
- paragraphs = extractParagraphs(html);
+ paragraphs = extractParagraphs(html).filter((p) => !FILING_FURNITURE.test(p) && p.length >= 80);
if (filingCache.size >= FILING_CACHE_MAX) filingCache.delete(filingCache.keys().next().value);
filingCache.set(filing.accession + filing.url, paragraphs);
}
diff --git a/api/_lib/ssrf-guard.js b/api/_lib/ssrf-guard.js
index 247a24d49..b88ab6d52 100644
--- a/api/_lib/ssrf-guard.js
+++ b/api/_lib/ssrf-guard.js
@@ -17,6 +17,7 @@ import { promises as dns } from 'node:dns';
import net from 'node:net';
import https from 'node:https';
import http from 'node:http';
+import zlib from 'node:zlib';
import { isPrivateIPv6 as isPrivateIPv6Shared } from './ip-classify.js';
// Cloud metadata services we never want a server-side fetch to reach.
@@ -176,6 +177,27 @@ export function collectCappedBody(nodeRes, req, maxBytes) {
});
}
+// Decode a response body per its content-encoding. A host can send a
+// compressed body without being asked (the Wayback Machine replays an archived
+// page's original gzip), and `new Response(buffer, { headers })` does not decode
+// it, so callers would read compressed bytes as text. The inflated size is held
+// to the same ceiling as the wire size, so a decompression bomb aborts instead
+// of exhausting memory. Unknown encodings pass through untouched.
+const DEFAULT_DECODED_CEILING = 50 * 1024 * 1024;
+
+export function decodeContentEncoding(body, encoding, maxBytes) {
+ const limit = Number.isFinite(maxBytes) ? maxBytes : DEFAULT_DECODED_CEILING;
+ const enc = String(encoding || '').trim().toLowerCase();
+ const inflate = { gzip: zlib.gunzipSync, 'x-gzip': zlib.gunzipSync, deflate: zlib.inflateSync, br: zlib.brotliDecompressSync }[enc];
+ if (!inflate) return body;
+ try {
+ return inflate(body, { maxOutputLength: limit });
+ } catch (err) {
+ if (err?.code === 'ERR_BUFFER_TOO_LARGE') throw new MaxBytesExceededError(limit + 1, limit);
+ throw err;
+ }
+}
+
const MAX_REDIRECTS = 5;
// Convenience: assert + fetch. Uses the global fetch. Same options as fetch,
@@ -249,13 +271,17 @@ export async function fetchSafePublicUrlPinned(input, init = {}, opts = {}) {
};
const req = mod.request(reqOpts, (nodeRes) => {
collectCappedBody(nodeRes, req, maxBytes)
- .then((body) =>
+ .then((raw) => {
+ const encoded = nodeRes.headers['content-encoding'];
+ const body = decodeContentEncoding(raw, encoded, maxBytes);
+ const headers = { ...nodeRes.headers };
+ if (body !== raw) {
+ delete headers['content-encoding'];
+ delete headers['content-length'];
+ }
// Wrap in a fetch-compatible Response so callers use the same API.
- resolve(new Response(body, {
- status: nodeRes.statusCode,
- headers: nodeRes.headers,
- })),
- )
+ resolve(new Response(body, { status: nodeRes.statusCode, headers }));
+ })
.catch(reject);
});
req.on('error', reject);
diff --git a/mcp-server/src/tools/crypto-news.js b/mcp-server/src/tools/crypto-news.js
index 2d9f80cec..f2c8f2e1c 100644
--- a/mcp-server/src/tools/crypto-news.js
+++ b/mcp-server/src/tools/crypto-news.js
@@ -88,7 +88,7 @@ const RESOLVE_DESCRIPTION =
'priced in, horizon, and a counter-argument). When the publisher blocks fetches or paywalls the story, three.ws ' +
'recovers it from the Internet Archive, the SEC EDGAR filing of the company named in the headline, or other outlets ' +
'covering the same story, and says which lane it used in `extraction`. The excerpt is a quotation, not the article: ' +
- 'follow `full_text_url` for the rest. Free: no payment or API key required.';
+ 'follow `full_text_url` for the rest. Free: no payment or API key required. Use this when you have a crypto news URL and need its content or a trading read, including for blocked or paywalled pages.';
const resolveZodShape = {
url: z.string().url().max(2000).describe('Absolute http(s) link to the article, e.g. a `link` from crypto_news.'),
diff --git a/tests/article-extract-ladder.test.js b/tests/article-extract-ladder.test.js
new file mode 100644
index 000000000..75ea1279b
--- /dev/null
+++ b/tests/article-extract-ladder.test.js
@@ -0,0 +1,91 @@
+// Ladder order for api/_lib/article-extract.js with the extra lanes wired in:
+// page -> reader -> wayback -> primary -> siblings -> feed -> preview.
+// The lane modules are mocked here so the test pins the DECISIONS extractArticle
+// makes (which rung wins, what each lane receives), not the lanes' own parsing,
+// which tests/news-article-extract.test.js and tests/article-extract-ssrf.test.js cover.
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+const pinned = vi.fn();
+vi.mock('../api/_lib/ssrf-guard.js', () => ({
+ fetchSafePublicUrlPinned: (...a) => pinned(...a),
+ assertSafePublicUrl: async () => new URL('https://publisher.example/story'),
+}));
+
+const wayback = vi.fn();
+const primary = vi.fn();
+const siblings = vi.fn();
+vi.mock('../api/_lib/article-lanes.js', () => ({
+ fetchViaWayback: (...a) => wayback(...a),
+ fetchPrimaryFiling: (...a) => primary(...a),
+ fetchSiblings: (...a) => siblings(...a),
+}));
+
+import { extractArticle } from '../api/_lib/article-extract.js';
+
+const URL_ = 'https://publisher.example/story';
+const body = (n) => Array.from({ length: n }, (_, i) => `Paragraph ${i + 1} of the recovered story, long enough to be real prose.`);
+
+beforeEach(() => {
+ pinned.mockReset().mockResolvedValue(new Response('blocked', { status: 403 }));
+ wayback.mockReset().mockResolvedValue({ paragraphs: [], snapshot: null, html: null });
+ primary.mockReset().mockResolvedValue({ paragraphs: [], filing: null });
+ siblings.mockReset().mockResolvedValue({ paragraphs: [], siblings: [] });
+});
+
+describe('extractArticle lanes', () => {
+ it('uses the Wayback snapshot when the page and reader are blocked', async () => {
+ wayback.mockResolvedValue({ paragraphs: body(5), snapshot: 'https://web.archive.org/web/20240105123000id_/' + URL_, html: '<title>Archived headline</title>' });
+ const out = await extractArticle(URL_, { title: 'Headline' });
+ expect(out.extraction).toBe('wayback');
+ expect(out.paragraphs).toHaveLength(5);
+ expect(out.html).toContain('Archived headline');
+ expect(out.lane_sources).toEqual([{ label: 'Internet Archive snapshot', url: expect.stringContaining('web.archive.org') }]);
+ expect(primary).not.toHaveBeenCalled();
+ expect(siblings).not.toHaveBeenCalled();
+ });
+
+ it('falls to the SEC filing, passing the headline and publish time, when nothing is archived', async () => {
+ primary.mockResolvedValue({
+ paragraphs: body(3),
+ filing: { filer: 'Acme Holdings Corp', form: '8-K', filed: '2026-10-01', url: 'https://www.sec.gov/Archives/edgar/data/1/2/ex.htm' },
+ });
+ const out = await extractArticle(URL_, { title: 'Acme Holdings Corp reports', publishedAt: '2026-10-01T00:00:00Z' });
+ expect(out.extraction).toBe('primary');
+ expect(primary.mock.calls[0].slice(0, 2)).toEqual(['Acme Holdings Corp reports', '2026-10-01T00:00:00Z']);
+ expect(out.lane_sources[0].label).toBe('Acme Holdings Corp 8-K filed 2026-10-01');
+ expect(siblings).not.toHaveBeenCalled();
+ });
+
+ it('falls to sibling coverage only when it names at least two outlets', async () => {
+ siblings.mockResolvedValue({ paragraphs: ['A reports: x', 'B reports: y'], siblings: [{ source: 'A', link: 'https://a.example/1' }, { source: 'B', link: 'https://b.example/1' }] });
+ const out = await extractArticle(URL_, { title: 'Headline' });
+ expect(out.extraction).toBe('siblings');
+ expect(out.lane_sources.map((s) => s.label)).toEqual(['A', 'B']);
+
+ siblings.mockResolvedValue({ paragraphs: ['A reports: x'], siblings: [{ source: 'A', link: 'https://a.example/1' }] });
+ expect((await extractArticle(URL_, { title: 'Headline' })).extraction).toBe('preview');
+ });
+
+ it('skips the title-driven lanes when no headline is known', async () => {
+ await extractArticle(URL_);
+ expect(primary).not.toHaveBeenCalled();
+ expect(siblings).not.toHaveBeenCalled();
+ });
+
+ it('lets a real feed body beat an empty preview after every lane misses', async () => {
+ const feed = 'The publisher feed carries the whole story in its content body. '.repeat(12);
+ const out = await extractArticle(URL_, { title: 'Headline', feedContentText: feed });
+ expect(out.extraction).toBe('feed');
+ });
+
+ it('survives every lane throwing and reports an honest preview', async () => {
+ wayback.mockRejectedValue(new Error('archive down'));
+ primary.mockRejectedValue(new Error('edgar down'));
+ siblings.mockRejectedValue(new Error('feeds down'));
+ const out = await extractArticle(URL_, { title: 'Headline' });
+ expect(out.extraction).toBe('preview');
+ expect(out.paragraphs).toEqual([]);
+ expect(out.blocked_reason).toMatch(/403/);
+ });
+});
diff --git a/tests/article-extract-ssrf.test.js b/tests/article-extract-ssrf.test.js
index edb9d6193..b62afa13e 100644
--- a/tests/article-extract-ssrf.test.js
+++ b/tests/article-extract-ssrf.test.js
@@ -131,6 +131,7 @@ describe('wayback lane', () => {
expect(pinned.mock.calls[1][2]).toEqual({ maxBytes: MAX_BYTES });
expect(rawFetch).not.toHaveBeenCalled();
expect(out.paragraphs.length).toBe(1);
+ expect(out.html).toContain('archived story body');
});
it('does not touch the archive for a target the guard rejects', async () => {
@@ -142,7 +143,7 @@ describe('wayback lane', () => {
it('returns no paragraphs and no snapshot fetch when nothing is archived', async () => {
pinned.mockResolvedValueOnce(json({ archived_snapshots: {} }));
const out = await fetchViaWayback('https://publisher.example/story', extractParagraphs);
- expect(out).toEqual({ paragraphs: [], snapshot: null });
+ expect(out).toEqual({ paragraphs: [], snapshot: null, html: null });
expect(pinned).toHaveBeenCalledTimes(1);
});
});
diff --git a/tests/ssrf-pinned-lookup.test.js b/tests/ssrf-pinned-lookup.test.js
index a2845d901..ef54ae1c9 100644
--- a/tests/ssrf-pinned-lookup.test.js
+++ b/tests/ssrf-pinned-lookup.test.js
@@ -130,3 +130,27 @@ describe('collectCappedBody streaming byte ceiling', () => {
expect(err.code).toBe('max_bytes_exceeded');
});
});
+
+// A host may answer with a compressed body without being asked (the Wayback
+// Machine replays an archived page's original gzip). The pinned fetch wraps raw
+// bytes in a Response, which does not decode content-encoding, so callers used
+// to read compressed garbage as text.
+describe('decodeContentEncoding', () => {
+ it('inflates gzip, deflate and brotli bodies and passes identity through', async () => {
+ const zlib = await import('node:zlib');
+ const { decodeContentEncoding } = await import('../api/_lib/ssrf-guard.js');
+ const plain = Buffer.from('<html><body><p>archived story body</p></body></html>');
+ expect(decodeContentEncoding(zlib.gzipSync(plain), 'gzip', 1_000_000).equals(plain)).toBe(true);
+ expect(decodeContentEncoding(zlib.deflateSync(plain), 'deflate', 1_000_000).equals(plain)).toBe(true);
+ expect(decodeContentEncoding(zlib.brotliCompressSync(plain), 'br', 1_000_000).equals(plain)).toBe(true);
+ expect(decodeContentEncoding(plain, undefined, 1_000_000).equals(plain)).toBe(true);
+ expect(decodeContentEncoding(plain, 'identity', 1_000_000).equals(plain)).toBe(true);
+ });
+
+ it('aborts a decompression bomb at the byte ceiling', async () => {
+ const zlib = await import('node:zlib');
+ const { decodeContentEncoding, MaxBytesExceededError } = await import('../api/_lib/ssrf-guard.js');
+ const bomb = zlib.gzipSync(Buffer.alloc(5_000_000));
+ expect(() => decodeContentEncoding(bomb, 'gzip', 100_000)).toThrow(MaxBytesExceededError);
+ });
+});