Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(news): decode gzip at the SSRF guard, drop rung-comment dashes, and cover the lane ladder order so archive and EDGAR lanes read real bodies
nirholase1de07deae6e
diff --git a/api/_lib/article-extract.js b/api/_lib/article-extract.jsindex c1c59a256..729cb4c50 100644--- a/api/_lib/article-extract.js+++ b/api/_lib/article-extract.js@@ -239,11 +239,65 @@ export async function extractArticle(url, { feedContentText = null, title = nullreaderError = err;}- // A partial direct page (2 paragraphs) still beats the feed teaser.+ // Rung 3: Wayback. The same article as it was archived: strictly better than+ // a partial page, so it runs for any page thinner than a full read.+ let laneError = null;+ try {+ const { paragraphs: archived, snapshot, html: archivedHtml } = await fetchViaWayback(url, extractParagraphs);+ if (archived.length >= 3 && archived.length > paragraphs.length) {+ return {+ paragraphs: archived,+ extraction: 'wayback',+ blocked_reason: null,+ // The snapshot's own metadata (headline, image, byline) stands in+ // when the live page was blocked.+ html: html || archivedHtml,+ lane_sources: [{ label: 'Internet Archive snapshot', url: snapshot }],+ };+ }+ } catch (err) {+ laneError = err;+ }++ // A partial direct page (2 paragraphs) still beats the substitutes below.if (paragraphs.length >= 2) {return { paragraphs, extraction: 'page', blocked_reason: null, html, lane_sources: [] };}+ // Rung 4: primary document: the SEC filing the headline's company made.+ if (title) {+ try {+ const { paragraphs: filed, filing } = await fetchPrimaryFiling(title, publishedAt, extractParagraphs);+ if (filed.length >= 2) {+ return {+ paragraphs: filed,+ extraction: 'primary',+ blocked_reason: null,+ html,+ lane_sources: [{ label: `${filing.filer} ${filing.form} filed ${filing.filed}`, url: filing.url }],+ };+ }+ } catch (err) {+ laneError = laneError || err;+ }++ // Rung 5: sibling coverage from other outlets.+ try {+ const { paragraphs: sibling, siblings } = await fetchSiblings(title, url);+ if (siblings.length >= 2) {+ return {+ paragraphs: sibling,+ extraction: 'siblings',+ blocked_reason: null,+ html,+ lane_sources: siblings.map((s) => ({ label: s.source, url: s.link })),+ };+ }+ } catch (err) {+ laneError = laneError || err;+ }+ }+// Rung 6, publisher feed body, when it is more than a teaser.if (feedContentText) {const feedParas = paragraphsFromFeed(feedContentText);diff --git a/api/_lib/article-lanes.js b/api/_lib/article-lanes.jsindex d93ebe46a..3e6b9f375 100644--- a/api/_lib/article-lanes.js+++ b/api/_lib/article-lanes.js@@ -90,13 +90,13 @@ export async function fetchViaWayback(url, extractParagraphs) {['json'],);const raw = waybackRawUrl(JSON.parse(lookup)?.archived_snapshots?.closest, url);- if (!raw) return { paragraphs: [], snapshot: null };+ if (!raw) return { paragraphs: [], snapshot: null, html: null };const html = await fetchText(raw,{ accept: 'text/html,application/xhtml+xml', 'user-agent': LANE_USER_AGENT },['text/html', 'application/xhtml+xml'],);- return { paragraphs: extractParagraphs(html), snapshot: raw };+ return { paragraphs: extractParagraphs(html), snapshot: raw, html };}// ── Primary documents: SEC EDGAR ───────────────────────────────────────────@@ -115,6 +115,9 @@ function secSlot() {return turn;}+// SEC cover-page and signature furniture that leads every filing.+const FILING_FURNITURE = /securities and exchange commission|^form (8-k|10-k|10-q|s-1)\b|check the appropriate box|commission file number|state or other jurisdiction|\(address of principal|emerging growth company|indicate by check mark|pursuant to section 13 or 15\(d\)|date of report|written communications|soliciting material|pre-commencement|rule 1[34][ade]-|pursuant to the requirements of the securities exchange act/i;+const filingCache = new Map(); // accession -> paragraphsconst FILING_CACHE_MAX = 128;@@ -192,7 +195,7 @@ export async function fetchPrimaryFiling(title, publishedAt, extractParagraphs,if (!paragraphs) {await secSlot();const html = await fetchText(filing.url, { accept: 'text/html', 'user-agent': LANE_USER_AGENT }, ['text/html']);- paragraphs = extractParagraphs(html);+ paragraphs = extractParagraphs(html).filter((p) => !FILING_FURNITURE.test(p) && p.length >= 80);if (filingCache.size >= FILING_CACHE_MAX) filingCache.delete(filingCache.keys().next().value);filingCache.set(filing.accession + filing.url, paragraphs);}diff --git a/api/_lib/ssrf-guard.js b/api/_lib/ssrf-guard.jsindex 247a24d49..b88ab6d52 100644--- a/api/_lib/ssrf-guard.js+++ b/api/_lib/ssrf-guard.js@@ -17,6 +17,7 @@ import { promises as dns } from 'node:dns';import net from 'node:net';import https from 'node:https';import http from 'node:http';+import zlib from 'node:zlib';import { isPrivateIPv6 as isPrivateIPv6Shared } from './ip-classify.js';// Cloud metadata services we never want a server-side fetch to reach.@@ -176,6 +177,27 @@ export function collectCappedBody(nodeRes, req, maxBytes) {});}+// Decode a response body per its content-encoding. A host can send a+// compressed body without being asked (the Wayback Machine replays an archived+// page's original gzip), and `new Response(buffer, { headers })` does not decode+// it, so callers would read compressed bytes as text. The inflated size is held+// to the same ceiling as the wire size, so a decompression bomb aborts instead+// of exhausting memory. Unknown encodings pass through untouched.+const DEFAULT_DECODED_CEILING = 50 * 1024 * 1024;++export function decodeContentEncoding(body, encoding, maxBytes) {+ const limit = Number.isFinite(maxBytes) ? maxBytes : DEFAULT_DECODED_CEILING;+ const enc = String(encoding || '').trim().toLowerCase();+ const inflate = { gzip: zlib.gunzipSync, 'x-gzip': zlib.gunzipSync, deflate: zlib.inflateSync, br: zlib.brotliDecompressSync }[enc];+ if (!inflate) return body;+ try {+ return inflate(body, { maxOutputLength: limit });+ } catch (err) {+ if (err?.code === 'ERR_BUFFER_TOO_LARGE') throw new MaxBytesExceededError(limit + 1, limit);+ throw err;+ }+}+const MAX_REDIRECTS = 5;// Convenience: assert + fetch. Uses the global fetch. Same options as fetch,@@ -249,13 +271,17 @@ export async function fetchSafePublicUrlPinned(input, init = {}, opts = {}) {};const req = mod.request(reqOpts, (nodeRes) => {collectCappedBody(nodeRes, req, maxBytes)- .then((body) =>+ .then((raw) => {+ const encoded = nodeRes.headers['content-encoding'];+ const body = decodeContentEncoding(raw, encoded, maxBytes);+ const headers = { ...nodeRes.headers };+ if (body !== raw) {+ delete headers['content-encoding'];+ delete headers['content-length'];+ }// Wrap in a fetch-compatible Response so callers use the same API.- resolve(new Response(body, {- status: nodeRes.statusCode,- headers: nodeRes.headers,- })),- )+ resolve(new Response(body, { status: nodeRes.statusCode, headers }));+ }).catch(reject);});req.on('error', reject);diff --git a/mcp-server/src/tools/crypto-news.js b/mcp-server/src/tools/crypto-news.jsindex 2d9f80cec..f2c8f2e1c 100644--- a/mcp-server/src/tools/crypto-news.js+++ b/mcp-server/src/tools/crypto-news.js@@ -88,7 +88,7 @@ const RESOLVE_DESCRIPTION ='priced in, horizon, and a counter-argument). When the publisher blocks fetches or paywalls the story, three.ws ' +'recovers it from the Internet Archive, the SEC EDGAR filing of the company named in the headline, or other outlets ' +'covering the same story, and says which lane it used in `extraction`. The excerpt is a quotation, not the article: ' +- 'follow `full_text_url` for the rest. Free: no payment or API key required.';+ 'follow `full_text_url` for the rest. Free: no payment or API key required. Use this when you have a crypto news URL and need its content or a trading read, including for blocked or paywalled pages.';const resolveZodShape = {url: z.string().url().max(2000).describe('Absolute http(s) link to the article, e.g. a `link` from crypto_news.'),diff --git a/tests/article-extract-ladder.test.js b/tests/article-extract-ladder.test.jsnew file mode 100644index 000000000..75ea1279b--- /dev/null+++ b/tests/article-extract-ladder.test.js@@ -0,0 +1,91 @@+// Ladder order for api/_lib/article-extract.js with the extra lanes wired in:+// page -> reader -> wayback -> primary -> siblings -> feed -> preview.+// The lane modules are mocked here so the test pins the DECISIONS extractArticle+// makes (which rung wins, what each lane receives), not the lanes' own parsing,+// which tests/news-article-extract.test.js and tests/article-extract-ssrf.test.js cover.++import { describe, it, expect, vi, beforeEach } from 'vitest';++const pinned = vi.fn();+vi.mock('../api/_lib/ssrf-guard.js', () => ({+ fetchSafePublicUrlPinned: (...a) => pinned(...a),+ assertSafePublicUrl: async () => new URL('https://publisher.example/story'),+}));++const wayback = vi.fn();+const primary = vi.fn();+const siblings = vi.fn();+vi.mock('../api/_lib/article-lanes.js', () => ({+ fetchViaWayback: (...a) => wayback(...a),+ fetchPrimaryFiling: (...a) => primary(...a),+ fetchSiblings: (...a) => siblings(...a),+}));++import { extractArticle } from '../api/_lib/article-extract.js';++const URL_ = 'https://publisher.example/story';+const body = (n) => Array.from({ length: n }, (_, i) => `Paragraph ${i + 1} of the recovered story, long enough to be real prose.`);++beforeEach(() => {+ pinned.mockReset().mockResolvedValue(new Response('blocked', { status: 403 }));+ wayback.mockReset().mockResolvedValue({ paragraphs: [], snapshot: null, html: null });+ primary.mockReset().mockResolvedValue({ paragraphs: [], filing: null });+ siblings.mockReset().mockResolvedValue({ paragraphs: [], siblings: [] });+});++describe('extractArticle lanes', () => {+ it('uses the Wayback snapshot when the page and reader are blocked', async () => {+ wayback.mockResolvedValue({ paragraphs: body(5), snapshot: 'https://web.archive.org/web/20240105123000id_/' + URL_, html: '<title>Archived headline</title>' });+ const out = await extractArticle(URL_, { title: 'Headline' });+ expect(out.extraction).toBe('wayback');+ expect(out.paragraphs).toHaveLength(5);+ expect(out.html).toContain('Archived headline');+ expect(out.lane_sources).toEqual([{ label: 'Internet Archive snapshot', url: expect.stringContaining('web.archive.org') }]);+ expect(primary).not.toHaveBeenCalled();+ expect(siblings).not.toHaveBeenCalled();+ });++ it('falls to the SEC filing, passing the headline and publish time, when nothing is archived', async () => {+ primary.mockResolvedValue({+ paragraphs: body(3),+ filing: { filer: 'Acme Holdings Corp', form: '8-K', filed: '2026-10-01', url: 'https://www.sec.gov/Archives/edgar/data/1/2/ex.htm' },+ });+ const out = await extractArticle(URL_, { title: 'Acme Holdings Corp reports', publishedAt: '2026-10-01T00:00:00Z' });+ expect(out.extraction).toBe('primary');+ expect(primary.mock.calls[0].slice(0, 2)).toEqual(['Acme Holdings Corp reports', '2026-10-01T00:00:00Z']);+ expect(out.lane_sources[0].label).toBe('Acme Holdings Corp 8-K filed 2026-10-01');+ expect(siblings).not.toHaveBeenCalled();+ });++ it('falls to sibling coverage only when it names at least two outlets', async () => {+ siblings.mockResolvedValue({ paragraphs: ['A reports: x', 'B reports: y'], siblings: [{ source: 'A', link: 'https://a.example/1' }, { source: 'B', link: 'https://b.example/1' }] });+ const out = await extractArticle(URL_, { title: 'Headline' });+ expect(out.extraction).toBe('siblings');+ expect(out.lane_sources.map((s) => s.label)).toEqual(['A', 'B']);++ siblings.mockResolvedValue({ paragraphs: ['A reports: x'], siblings: [{ source: 'A', link: 'https://a.example/1' }] });+ expect((await extractArticle(URL_, { title: 'Headline' })).extraction).toBe('preview');+ });++ it('skips the title-driven lanes when no headline is known', async () => {+ await extractArticle(URL_);+ expect(primary).not.toHaveBeenCalled();+ expect(siblings).not.toHaveBeenCalled();+ });++ it('lets a real feed body beat an empty preview after every lane misses', async () => {+ const feed = 'The publisher feed carries the whole story in its content body. '.repeat(12);+ const out = await extractArticle(URL_, { title: 'Headline', feedContentText: feed });+ expect(out.extraction).toBe('feed');+ });++ it('survives every lane throwing and reports an honest preview', async () => {+ wayback.mockRejectedValue(new Error('archive down'));+ primary.mockRejectedValue(new Error('edgar down'));+ siblings.mockRejectedValue(new Error('feeds down'));+ const out = await extractArticle(URL_, { title: 'Headline' });+ expect(out.extraction).toBe('preview');+ expect(out.paragraphs).toEqual([]);+ expect(out.blocked_reason).toMatch(/403/);+ });+});diff --git a/tests/article-extract-ssrf.test.js b/tests/article-extract-ssrf.test.jsindex edb9d6193..b62afa13e 100644--- a/tests/article-extract-ssrf.test.js+++ b/tests/article-extract-ssrf.test.js@@ -131,6 +131,7 @@ describe('wayback lane', () => {expect(pinned.mock.calls[1][2]).toEqual({ maxBytes: MAX_BYTES });expect(rawFetch).not.toHaveBeenCalled();expect(out.paragraphs.length).toBe(1);+ expect(out.html).toContain('archived story body');});it('does not touch the archive for a target the guard rejects', async () => {@@ -142,7 +143,7 @@ describe('wayback lane', () => {it('returns no paragraphs and no snapshot fetch when nothing is archived', async () => {pinned.mockResolvedValueOnce(json({ archived_snapshots: {} }));const out = await fetchViaWayback('https://publisher.example/story', extractParagraphs);- expect(out).toEqual({ paragraphs: [], snapshot: null });+ expect(out).toEqual({ paragraphs: [], snapshot: null, html: null });expect(pinned).toHaveBeenCalledTimes(1);});});diff --git a/tests/ssrf-pinned-lookup.test.js b/tests/ssrf-pinned-lookup.test.jsindex a2845d901..ef54ae1c9 100644--- a/tests/ssrf-pinned-lookup.test.js+++ b/tests/ssrf-pinned-lookup.test.js@@ -130,3 +130,27 @@ describe('collectCappedBody streaming byte ceiling', () => {expect(err.code).toBe('max_bytes_exceeded');});});++// A host may answer with a compressed body without being asked (the Wayback+// Machine replays an archived page's original gzip). The pinned fetch wraps raw+// bytes in a Response, which does not decode content-encoding, so callers used+// to read compressed garbage as text.+describe('decodeContentEncoding', () => {+ it('inflates gzip, deflate and brotli bodies and passes identity through', async () => {+ const zlib = await import('node:zlib');+ const { decodeContentEncoding } = await import('../api/_lib/ssrf-guard.js');+ const plain = Buffer.from('<html><body><p>archived story body</p></body></html>');+ expect(decodeContentEncoding(zlib.gzipSync(plain), 'gzip', 1_000_000).equals(plain)).toBe(true);+ expect(decodeContentEncoding(zlib.deflateSync(plain), 'deflate', 1_000_000).equals(plain)).toBe(true);+ expect(decodeContentEncoding(zlib.brotliCompressSync(plain), 'br', 1_000_000).equals(plain)).toBe(true);+ expect(decodeContentEncoding(plain, undefined, 1_000_000).equals(plain)).toBe(true);+ expect(decodeContentEncoding(plain, 'identity', 1_000_000).equals(plain)).toBe(true);+ });++ it('aborts a decompression bomb at the byte ceiling', async () => {+ const zlib = await import('node:zlib');+ const { decodeContentEncoding, MaxBytesExceededError } = await import('../api/_lib/ssrf-guard.js');+ const bomb = zlib.gzipSync(Buffer.alloc(5_000_000));+ expect(() => decodeContentEncoding(bomb, 'gzip', 100_000)).toThrow(MaxBytesExceededError);+ });+});