Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

11.0 KB
Raw Blame
// Extra lanes for the article extraction ladder (api/_lib/article-extract.js).
// ---------------------------------------------------------------------------
// When the publisher page and the reader service both come back empty, these
// lanes recover the story from places we may lawfully read:
//
//   wayback  , the Internet Archive's closest snapshot of the same URL. It
//               carries the full body of a story that has since gone behind a
//               paywall or been taken down.
//   primary  , the SEC EDGAR filing (8-K press-release exhibit, 10-K, 10-Q,
//               S-1) of the company the headline names. Public-domain primary
//               source, fetched with the contact User-Agent SEC requires and
//               throttled below its 10 requests/second ceiling.
//   siblings , other outlets covering the same story, from the native feed
//               aggregator. Each contributes only its own bounded feed
//               description, attributed by name.
//
// Every network fetch goes through the shared SSRF guard (fetchSafePublicUrlPinned
// validates all DNS answers, pins the socket, re-validates every redirect hop).
// Siblings reads the in-process feed cache and fetches nothing. Whatever a lane
// returns still passes through the standing excerpt limit in news-rights.js at
// the HTTP boundary, same as every other rung.

import { assertSafePublicUrl, fetchSafePublicUrlPinned } from './ssrf-guard.js';
import { getNews } from './news.js';
import { excerptText, isSuppressed } from './news-rights.js';

const LANE_TIMEOUT_MS = 10_000;
const MAX_LANE_BYTES = 5 * 1024 * 1024;

// SEC fair-access policy: a descriptive User-Agent with a contact address, and
// no more than 10 requests per second.
const SEC_CONTACT = (process.env.SEC_CONTACT_EMAIL || '[email protected]').trim();
export const LANE_USER_AGENT = `three.ws news reader ${SEC_CONTACT}`;
const SEC_MIN_INTERVAL_MS = 125; // 8 requests/second, under the 10/s ceiling

const STOPWORDS = new Set(
	('a an the and or of to in on at for with by from as is are was were be been it its this that these those ' +
		'after before over under into about how why what who when will would could should says said say new ' +
		'more most than not no up down out off has have had can may').split(/\s+/),
);

// ── Shared helpers ─────────────────────────────────────────────────────────

function words(text) {
	return String(text || '')
		.toLowerCase()
		.replace(/[^\p{L}\p{N}\s]/gu, ' ')
		.split(/\s+/)
		.filter((w) => w.length > 2 && !STOPWORDS.has(w));
}

async function fetchText(url, headers, accept) {
	const resp = await fetchSafePublicUrlPinned(
		url,
		{ headers, signal: AbortSignal.timeout(LANE_TIMEOUT_MS) },
		{ maxBytes: MAX_LANE_BYTES },
	);
	if (!resp.ok) throw Object.assign(new Error(`${new URL(url).hostname} responded ${resp.status}`), { status: resp.status });
	const type = resp.headers.get('content-type') || '';
	if (!accept.some((a) => type.includes(a))) {
		throw new Error(`unsupported content type: ${type.split(';')[0] || 'unknown'}`);
	}
	return resp.text();
}

// ── Wayback ────────────────────────────────────────────────────────────────

/**
 * Turn the availability API's `closest` snapshot into the raw-capture URL
 * (`id_` serves the archived bytes without the Wayback toolbar rewrite).
 * Returns null unless the snapshot is genuinely a web.archive.org capture.
 */
export function waybackRawUrl(closest, originalUrl) {
	if (!closest?.available || !/^\d{14}$/.test(String(closest.timestamp))) return null;
	let snap;
	try {
		snap = new URL(String(closest.url).replace(/^http:/, 'https:'));
	} catch {
		return null;
	}
	if (snap.hostname !== 'web.archive.org' || !snap.pathname.startsWith('/web/')) return null;
	return `https://web.archive.org/web/${closest.timestamp}id_/${originalUrl}`;
}

export async function fetchViaWayback(url, extractParagraphs) {
	await assertSafePublicUrl(url, { allowHttp: true });
	const lookup = await fetchText(
		`https://archive.org/wayback/available?url=${encodeURIComponent(url)}`,
		{ accept: 'application/json', 'user-agent': LANE_USER_AGENT },
		['json'],
	);
	const raw = waybackRawUrl(JSON.parse(lookup)?.archived_snapshots?.closest, url);
	if (!raw) return { paragraphs: [], snapshot: null, html: null };
	const html = await fetchText(
		raw,
		{ accept: 'text/html,application/xhtml+xml', 'user-agent': LANE_USER_AGENT },
		['text/html', 'application/xhtml+xml'],
	);
	return { paragraphs: extractParagraphs(html), snapshot: raw, html };
}

// ── Primary documents: SEC EDGAR ───────────────────────────────────────────

let secLastAt = 0;
let secQueue = Promise.resolve();

/** Serialise EDGAR requests at least SEC_MIN_INTERVAL_MS apart. */
function secSlot() {
	const turn = secQueue.then(async () => {
		const wait = secLastAt + SEC_MIN_INTERVAL_MS - Date.now();
		if (wait > 0) await new Promise((r) => setTimeout(r, wait));
		secLastAt = Date.now();
	});
	secQueue = turn.catch(() => {});
	return turn;
}

// SEC cover-page and signature furniture that leads every filing.
const FILING_FURNITURE = /securities and exchange commission|^form (8-k|10-k|10-q|s-1)\b|check the appropriate box|commission file number|state or other jurisdiction|\(address of principal|emerging growth company|indicate by check mark|pursuant to section 13 or 15\(d\)|date of report|written communications|soliciting material|pre-commencement|rule 1[34][ade]-|pursuant to the requirements of the securities exchange act/i;

const filingCache = new Map(); // accession -> paragraphs
const FILING_CACHE_MAX = 128;

/**
 * The company or organisation a headline names: the longest run of capitalised
 * words that is not the sentence-initial word alone. Null when the headline
 * names nobody a filing could be matched to.
 */
export function filerPhrase(title) {
	const runs = [...String(title || '').matchAll(/\b[A-Z][\p{L}\p{N}&.'-]+(?:\s+(?:of\s+)?[A-Z][\p{L}\p{N}&.'-]+)*/gu)]
		.map((m) => m[0].trim())
		.filter((r) => r.length >= 4 && !STOPWORDS.has(r.toLowerCase()));
	if (!runs.length) return null;
	runs.sort((a, b) => b.length - a.length);
	const best = runs[0];
	// A lone sentence-opening word ("Bitcoin") is a topic, not a filer.
	const isFirstWordOnly = !/\s/.test(best) && String(title).trimStart().startsWith(best);
	return isFirstWordOnly ? null : best;
}

/**
 * Pick the best document hit from an EDGAR full-text search response: the
 * filer's display name must contain the headline phrase (so a loose full-text
 * match on a common word never yields an unrelated company's filing), and a
 * press-release exhibit beats the bare form.
 *
 * @returns {{ url: string, accession: string, form: string, filer: string, filed: string } | null}
 */
export function pickFiling(response, phrase) {
	const need = String(phrase || '').toLowerCase();
	let best = null;
	for (const hit of response?.hits?.hits || []) {
		const s = hit?._source || {};
		const [accession, file] = String(hit?._id || '').split(':');
		const cik = String(s.ciks?.[0] || '').replace(/^0+/, '');
		const filer = String(s.display_names?.[0] || '');
		if (!accession || !file || !cik || !need || !filer.toLowerCase().includes(need)) continue;
		if (!/\.html?$/i.test(file)) continue;
		const rank = s.file_type === 'EX-99.1' ? 2 : s.file_type === s.form ? 1 : 0;
		if (!rank) continue;
		if (!best || rank > best.rank) {
			best = {
				rank,
				url: `https://www.sec.gov/Archives/edgar/data/${cik}/${accession.replace(/-/g, '')}/${file}`,
				accession,
				form: String(s.form || ''),
				filer: filer.replace(/\s*\(CIK[^)]*\)\s*$/, '').trim(),
				filed: String(s.file_date || ''),
			};
		}
	}
	if (!best) return null;
	const { rank, ...filing } = best;
	return filing;
}

export async function fetchPrimaryFiling(title, publishedAt, extractParagraphs, now = Date.now()) {
	const phrase = filerPhrase(title);
	if (!phrase) return { paragraphs: [], filing: null };

	const end = new Date(now);
	const published = Date.parse(publishedAt || '');
	const start = new Date((Number.isFinite(published) ? published : now) - 14 * 86_400_000);
	const day = (d) => d.toISOString().slice(0, 10);
	const search =
		`https://efts.sec.gov/LATEST/search-index?q=${encodeURIComponent(`"${phrase}"`)}` +
		`&forms=8-K,10-K,10-Q,S-1&dateRange=custom&startdt=${day(start)}&enddt=${day(end)}`;

	await secSlot();
	const found = JSON.parse(await fetchText(search, { accept: 'application/json', 'user-agent': LANE_USER_AGENT }, ['json']));
	const filing = pickFiling(found, phrase);
	if (!filing) return { paragraphs: [], filing: null };

	let paragraphs = filingCache.get(filing.accession + filing.url);
	if (!paragraphs) {
		await secSlot();
		const html = await fetchText(filing.url, { accept: 'text/html', 'user-agent': LANE_USER_AGENT }, ['text/html']);
		paragraphs = extractParagraphs(html).filter((p) => !FILING_FURNITURE.test(p) && p.length >= 80);
		if (filingCache.size >= FILING_CACHE_MAX) filingCache.delete(filingCache.keys().next().value);
		filingCache.set(filing.accession + filing.url, paragraphs);
	}
	return { paragraphs, filing };
}

// ── Sibling coverage ───────────────────────────────────────────────────────

const MIN_SHARED_WORDS = 3;
const MIN_OVERLAP = 0.4;

/**
 * Other outlets' coverage of the same story: feed items whose headline shares
 * enough distinctive words with ours, from a different publisher, newest
 * first. Pure so it can be tested without the aggregator.
 */
export function pickSiblings(candidates, { title, url, max = 4 }) {
	const mine = new Set(words(title));
	if (mine.size < MIN_SHARED_WORDS) return [];
	let myHost = '';
	try {
		myHost = new URL(url).hostname.replace(/^www\./, '');
	} catch {
		/* url validated upstream */
	}
	const seenSources = new Set();
	const out = [];
	for (const c of candidates || []) {
		if (!c?.link || c.link === url || isSuppressed(c)) continue;
		let host = '';
		try {
			host = new URL(c.link).hostname.replace(/^www\./, '');
		} catch {
			continue;
		}
		if (host === myHost || seenSources.has(host)) continue;
		const theirs = new Set(words(c.title));
		const shared = [...theirs].filter((w) => mine.has(w)).length;
		if (shared < MIN_SHARED_WORDS || shared / Math.min(mine.size, theirs.size) < MIN_OVERLAP) continue;
		const body = excerptText(c.description);
		if (body.length < 60) continue;
		seenSources.add(host);
		out.push({ source: c.source || host, link: c.link, text: body });
		if (out.length >= max) break;
	}
	return out;
}

export async function fetchSiblings(title, url) {
	const pool = await getNews({ limit: 300, lang: 'all' });
	const siblings = pickSiblings(pool.articles, { title, url });
	return {
		paragraphs: siblings.map((s) => `${s.source} reports: ${s.text}`),
		siblings: siblings.map(({ source, link }) => ({ source, link })),
	};
}