Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

18.4 KB
Raw Blame
// GET /api/news/archive
// ---------------------------------------------------------------------------
// The three.ws historical crypto-news archive: 660k+ enriched articles from
// September 2017 onward (CryptoPanic english corpus + Odaily chinese corpus +
// the cryptocurrency.cv live archiver), recovered from the cryptocurrency.cv
// archive and hosted on the platform's own GCS bucket
// (gs://three-ws-news-archive). Largest open crypto-news dataset anywhere;
// every record carries tickers, tags, sentiment, language, and (where
// captured) market context at publication time.
//
// The corpus grows hourly (api/cron/news-archive-append.js appends the live
// feed), so never hardcode its size or end date here: meta/stats.json
// (total_articles, last_article_date) is the live count, and ?stats=true
// serves it.
//
// Data layout and month-file access live in api/_lib/news-archive-store.js
// (shared with the story pages and the news sitemap):
//   articles/YYYY-MM.jsonl   one enriched article per line
//   meta/stats.json          corpus-wide statistics
//   indexes/by-{date,source,ticker}.json
//
// Modes:
//   ?stats=true              corpus statistics + available month range (free)
//   ?months=true             list of queryable months (free)
//   ?trending=true           top tickers over the newest archived weeks (free)
//   default                  query mode — free daily quota, then x402
//
// Access model (freemium, same shape as api/v1/ai/tts.js):
//   • stats/months/trending are cached + tiny → always free.
//   • Each SEARCH fans out to GCS month files (2–11 MB each, up to 12 per
//     request), so query mode carries a free per-IP daily quota
//     (limits.newsArchiveFreeIp — the funnel and the anti-scrape bound) and
//     falls through to an x402 402 challenge ($0.001 USDC per search,
//     env-overridable via X402_PRICE_NEWS_ARCHIVE) once exhausted. A request
//     arriving with an X-PAYMENT header goes straight to the paid rail.
//
// Query params (query mode):
//   q           full-text (title + description)
//   ticker      e.g. BTC — matches the enriched tickers field
//   source      source_key or source name substring
//   category    enriched category
//   sentiment   positive | negative | neutral
//   lang        en | zh
//   start_date / end_date    YYYY-MM-DD, bounds the month window
//   limit       1–100 (default 50), offset for pagination
//
// Serving model: newest-first month scan with early stop once the requested
// page is filled, capped at MAX_MONTHS_PER_SCAN months per request. Parsed
// months are LRU-cached in compact form so repeat queries are warm. The
// response always reports exactly which months were scanned so the UI can be
// honest about coverage ("narrow the date range to search older years").

import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { priceFor } from '../_lib/x402-prices.js';
import { paidEndpoint } from '../_lib/x402-paid-endpoint.js';
import { declareHttpDiscovery, withService } from '../_lib/x402/bazaar-helpers.js';
import { installAccessControl } from '../_lib/x402/access-control.js';
import { getStats, getMonths, loadMonth } from '../_lib/news-archive-store.js';
import { isSuppressed, excerptText } from '../_lib/news-rights.js';

const MAX_MONTHS_PER_SCAN = 12;
const MONTH_FETCH_CONCURRENCY = 3;

const ROUTE = '/api/news/archive';
const PRICE_ATOMICS = priceFor('news-archive', '1000'); // $0.001/search (env: X402_PRICE_NEWS_ARCHIVE)
const FREE_SEARCHES_PER_DAY = 60; // keep in sync with limits.newsArchiveFreeIp

function httpErr(status, code, message) {
	const e = new Error(message);
	e.status = status;
	e.code = code;
	// Marks a status this endpoint chose deliberately. The shared upstream fetch
	// synthesizes its own status on a transport failure (503 for an unreachable
	// host), and without this flag that transport code would surface at the
	// boundary in place of the 502 archive_unavailable contract this endpoint
	// documents and its callers match on.
	e.deliberate = true;
	return e;
}

function buildPredicate({ q, ticker, source, category, sentiment, lang, startDate, endDate }) {
	const needle = q ? q.toLowerCase() : null;
	const tickerUpper = ticker ? ticker.toUpperCase() : null;
	const sourceLower = source ? source.toLowerCase() : null;
	const startIso = startDate ? `${startDate}T00:00:00.000Z` : null;
	const endIso = endDate ? `${endDate}T23:59:59.999Z` : null;
	return (a) => {
		if (tickerUpper && !a.tickers.includes(tickerUpper)) return false;
		if (sourceLower && !a.source_key.toLowerCase().includes(sourceLower) && !a.source.toLowerCase().includes(sourceLower))
			return false;
		if (category && a.category !== category) return false;
		if (lang && a.lang !== lang) return false;
		if (sentiment) {
			const label = a.sentiment?.label || 'neutral';
			if (sentiment === 'positive' && !label.includes('positive')) return false;
			if (sentiment === 'negative' && !label.includes('negative')) return false;
			if (sentiment === 'neutral' && label !== 'neutral') return false;
		}
		if ((startIso || endIso) && !a.pub_date) return false;
		if (startIso && a.pub_date < startIso) return false;
		if (endIso && a.pub_date > endIso) return false;
		if (needle && !`${a.title} ${a.description || ''}`.toLowerCase().includes(needle)) return false;
		return true;
	};
}

// Parse + validate query-mode params. Throws httpErr(400, …) on bad input so
// both lanes reject it cleanly — genuinely bad input must be a 400, never a
// payment prompt.
function parseSearchParams(params) {
	const q = (params.get('q') || '').trim().slice(0, 120) || null;
	const ticker = (params.get('ticker') || '').trim().slice(0, 12) || null;
	const source = (params.get('source') || '').trim().slice(0, 40) || null;
	const category = (params.get('category') || '').trim().toLowerCase() || null;
	const sentiment = (params.get('sentiment') || '').trim().toLowerCase() || null;
	const lang = (params.get('lang') || '').trim().toLowerCase() || null;
	const startDate = (params.get('start_date') || '').trim() || null;
	const endDate = (params.get('end_date') || '').trim() || null;
	const limit = Math.min(Math.max(1, parseInt(params.get('limit') || '50', 10) || 50), 100);
	const offset = Math.max(0, parseInt(params.get('offset') || '0', 10) || 0);

	const dateRe = /^\d{4}-\d{2}-\d{2}$/;
	if (startDate && !dateRe.test(startDate)) throw httpErr(400, 'bad_date', 'start_date must be YYYY-MM-DD');
	if (endDate && !dateRe.test(endDate)) throw httpErr(400, 'bad_date', 'end_date must be YYYY-MM-DD');
	if (startDate && endDate && startDate > endDate)
		throw httpErr(400, 'bad_range', 'start_date is after end_date');
	if (sentiment && !['positive', 'negative', 'neutral'].includes(sentiment))
		throw httpErr(400, 'bad_sentiment', 'sentiment must be positive, negative, or neutral');
	if (lang && !['en', 'zh'].includes(lang)) throw httpErr(400, 'bad_lang', 'lang must be en or zh');

	return { q, ticker, source, category, sentiment, lang, startDate, endDate, limit, offset };
}

// Run one archive search (the expensive month scan) and return the response
// body. Shared verbatim by the free lane and the paid rail so both tiers serve
// the SAME engine — the only difference is who paid for the scan.
async function archiveSearch(parsed) {
	const { q, ticker, source, category, sentiment, lang, startDate, endDate, limit, offset } = parsed;
	try {
		const allMonths = await getMonths();
		let window = allMonths;
		if (startDate) window = window.filter((m) => m >= startDate.slice(0, 7));
		if (endDate) window = window.filter((m) => m <= endDate.slice(0, 7));
		if (!window.length) {
			return {
				articles: [], total_scanned_matches: 0, limit, offset,
				has_more: false,
				scanned: { months: [], complete: true },
				hint: 'no archive months inside that date range',
			};
		}
		// newest → oldest
		window = [...window].reverse();

		const matches = buildPredicate({ q, ticker, source, category, sentiment, lang, startDate, endDate });
		const need = offset + limit + 1; // +1 lets hasMore be exact within the scan
		const scannedMonths = [];
		let found = [];
		for (let i = 0; i < window.length && scannedMonths.length < MAX_MONTHS_PER_SCAN; i += MONTH_FETCH_CONCURRENCY) {
			const batch = window.slice(i, Math.min(i + MONTH_FETCH_CONCURRENCY, window.length))
				.slice(0, MAX_MONTHS_PER_SCAN - scannedMonths.length);
			const loaded = await Promise.all(batch.map((m) => loadMonth(m).catch(() => null)));
			for (let b = 0; b < batch.length; b++) {
				if (!loaded[b]) continue; // month failed to load; reported via scanned list
				scannedMonths.push(batch[b]);
				found = found.concat(loaded[b].filter((a) => matches(a) && !isSuppressed(a)));
			}
			if (found.length >= need) break;
		}

		// Zero months loaded with months available = the archive store is down,
		// not an empty result — surface that instead of a misleading "no matches".
		if (!scannedMonths.length && window.length) {
			throw new Error('no archive months could be loaded');
		}

		found.sort((a, b) => new Date(b.pub_date || 0) - new Date(a.pub_date || 0));
		const monthsRemaining = window.length - scannedMonths.length;
		const body = {
			// Bound the quoted text on the way out. The corpus holds 660k records
			// captured before the rights layer existed, many with a full-body
			// `description` from a content:encoded feed; this caps them at read
			// time so the API cannot serve a publisher's article regardless of
			// what any individual stored row contains.
			articles: found.slice(offset, offset + limit).map((a) => ({ ...a, description: excerptText(a.description) })),
			total_scanned_matches: found.length,
			limit,
			offset,
			has_more: offset + limit < found.length || monthsRemaining > 0,
			scanned: {
				months: scannedMonths,
				from: scannedMonths[scannedMonths.length - 1] || null,
				to: scannedMonths[0] || null,
				complete: monthsRemaining === 0,
				months_remaining: monthsRemaining,
			},
		};
		if (monthsRemaining > 0) {
			body.hint = `scanned the newest ${scannedMonths.length} months of this range; add start_date/end_date to reach older articles`;
		}
		return body;
	} catch (err) {
		if (err.status) throw err;
		throw httpErr(502, 'archive_unavailable', `news archive is unreachable right now: ${err.message}`);
	}
}

// ── x402 paid rail ───────────────────────────────────────────────────────────

// Uniqueness-first: the first sentence answers "what can I only get here".
const DESCRIPTION =
	'Search the largest open crypto-news archive over x402: 660,000+ enriched articles back to ' +
	'September 2017 (english + chinese corpora, refreshed hourly), queryable by keyword, ticker, ' +
	'source, date range, sentiment, and language. Every record carries tickers, tags, sentiment, ' +
	'and (where captured) BTC/ETH price + Fear & Greed at publication. $0.001 USDC per search; a ' +
	`free daily quota (${FREE_SEARCHES_PER_DAY} searches/day per IP) lets you try it before paying. ` +
	'Corpus stats and trending modes are always free.';

const INPUT_EXAMPLE = {
	q: 'bitcoin etf',
	ticker: 'BTC',
	start_date: '2024-01-01',
	end_date: '2024-03-31',
	limit: 50,
};

const INPUT_SCHEMA = {
	$schema: 'https://json-schema.org/draft/2020-12/schema',
	type: 'object',
	properties: {
		q: { type: 'string', maxLength: 120, description: 'Full-text query over title + description.' },
		ticker: { type: 'string', maxLength: 12, description: 'Filter by enriched ticker, e.g. BTC.' },
		source: { type: 'string', maxLength: 40, description: 'Source key or name substring, e.g. coindesk, odaily.' },
		category: { type: 'string', description: 'Enriched category.' },
		sentiment: { type: 'string', enum: ['positive', 'negative', 'neutral'] },
		lang: { type: 'string', enum: ['en', 'zh'] },
		start_date: { type: 'string', pattern: '^\\d{4}-\\d{2}-\\d{2}$', description: 'Earliest publish date (archive starts 2017-09-23).' },
		end_date: { type: 'string', pattern: '^\\d{4}-\\d{2}-\\d{2}$', description: 'Latest publish date.' },
		limit: { type: 'integer', minimum: 1, maximum: 100, default: 50 },
		offset: { type: 'integer', minimum: 0, default: 0 },
	},
};

const OUTPUT_EXAMPLE = {
	articles: [
		{
			id: 'a1b2c3d4e5f60718',
			title: 'Bitcoin ETF sees record inflows',
			link: 'https://example.com/article',
			source: 'CoinDesk',
			category: 'general',
			pub_date: '2024-01-11T14:00:00.000Z',
			tickers: ['BTC'],
			sentiment: { label: 'positive', score: 0.4 },
			lang: 'en',
		},
	],
	total_scanned_matches: 128,
	limit: 50,
	offset: 0,
	has_more: true,
	scanned: { months: ['2024-03', '2024-02', '2024-01'], from: '2024-01', to: '2024-03', complete: true, months_remaining: 0 },
	tier: 'paid',
};

const OUTPUT_SCHEMA = {
	$schema: 'https://json-schema.org/draft/2020-12/schema',
	type: 'object',
	required: ['articles', 'total_scanned_matches', 'scanned'],
	properties: {
		articles: { type: 'array', items: { type: 'object' } },
		total_scanned_matches: { type: 'integer' },
		limit: { type: 'integer' },
		offset: { type: 'integer' },
		has_more: { type: 'boolean' },
		scanned: {
			type: 'object',
			description: 'Exactly which YYYY-MM months this search covered (newest-first scan).',
		},
		hint: { type: 'string' },
		tier: { type: 'string', enum: ['free', 'paid'] },
	},
};

const BAZAAR = declareHttpDiscovery({
	method: 'GET',
	input: INPUT_EXAMPLE,
	inputSchema: INPUT_SCHEMA,
	output: { example: OUTPUT_EXAMPLE, schema: OUTPUT_SCHEMA },
});

// The x402 paid twin — built once, lazily (constructing it touches env-derived
// pay-to config). Reused for every over-quota / paying request.
let _paid = null;
function paidRail() {
	if (_paid) return _paid;
	_paid = paidEndpoint({
		route: ROUTE,
		method: 'GET',
		priceAtomics: PRICE_ATOMICS,
		networks: ['solana', 'base'],
		description: DESCRIPTION,
		bazaar: BAZAAR,
		service: withService({
			serviceName: 'three.ws Crypto News Archive',
			tags: ['news', 'crypto', 'archive', 'search', 'data'],
		}),
		requiredScope: 'x402:bypass',
		// Bypass lanes, in order: internal key → x402_live_ API key (premium
		// passes mint one — see api/_lib/premium.js) → OAuth bearer.
		accessControl: installAccessControl({ requiredScope: 'x402:bypass' }),
		// Browser lane: a wallet holding a SIWX grant re-enters by signature.
		// A premium-pass purchase records a grant until the pass expires; a
		// per-call $0.001 payment earns a short search session so one payment
		// isn't one query.
		siwx: {
			statement: 'Access the three.ws crypto-news archive with your premium pass or recent payment.',
			ttlSeconds: 900,
		},
		async handler({ req }) {
			const params = new URL(req.url, 'http://x').searchParams;
			const body = await archiveSearch(parseSearchParams(params));
			return { ...body, tier: 'paid' };
		},
	});
	return _paid;
}

export default wrap(async (req, res) => {
	if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
	if (!method(req, res, ['GET'])) return;

	const rl = await limits.marketFeedIp(clientIp(req));
	if (!rl.success) return rateLimited(res, rl);

	const params = new URL(req.url, 'http://x').searchParams;

	// ── Free modes: cached, tiny, never metered ───────────────────────────────
	try {
		if (params.get('stats') === 'true') {
			const [stats, months] = await Promise.all([getStats(), getMonths()]);
			return json(
				res, 200,
				{
					stats: {
						total_articles: stats.total_articles,
						total_with_date: stats.total_with_date,
						total_with_url: stats.total_with_url,
						undated_articles: stats.total_articles - stats.total_with_date,
						first_article_date: stats.first_article_date,
						last_article_date: stats.last_article_date,
						languages: ['en', 'zh'],
						top_sources: Object.entries(stats.sources || {})
							.sort((a, b) => b[1] - a[1])
							.slice(0, 30)
							.map(([key, count]) => ({ key, count })),
					},
					months: { first: months[0], last: months[months.length - 1], count: months.length },
					search_access: {
						free_per_day: FREE_SEARCHES_PER_DAY,
						price_usdc: Number(PRICE_ATOMICS) / 1e6,
						protocol: 'x402',
					},
				},
				{ 'cache-control': 'public, max-age=600, s-maxage=3600, stale-while-revalidate=86400' },
			);
		}

		if (params.get('months') === 'true') {
			const months = await getMonths();
			return json(res, 200, { months }, {
				'cache-control': 'public, max-age=600, s-maxage=3600, stale-while-revalidate=86400',
			});
		}

		if (params.get('trending') === 'true') {
			const months = await getMonths();
			const recent = months.slice(-2);
			const counts = new Map();
			for (const month of recent) {
				for (const a of await loadMonth(month)) {
					for (const t of a.tickers) {
						// the corpus enrichment has occasional junk tickers ("A", "4") —
						// keep them queryable but out of the trending strip
						if (t.length < 2 || /^\d+$/.test(t)) continue;
						counts.set(t, (counts.get(t) || 0) + 1);
					}
				}
			}
			const trending = [...counts.entries()]
				.sort((a, b) => b[1] - a[1])
				.slice(0, 20)
				.map(([ticker, count]) => ({ ticker, count }));
			return json(res, 200, { trending, window: recent }, {
				'cache-control': 'public, max-age=600, s-maxage=3600, stale-while-revalidate=86400',
			});
		}
	} catch (err) {
		return error(res, 502, 'archive_unavailable', `news archive is unreachable right now: ${err.message}`);
	}

	// ── Query mode: free daily quota → x402 metered overage ──────────────────
	// A payment header means the caller is on the paid rail already.
	if (req.headers['x-payment'] || req.headers['payment-signature']) {
		return paidRail()(req, res);
	}

	// Validate before the quota check so genuinely bad input is a clean 400
	// rather than a payment prompt (and never burns a free search).
	let parsed;
	try {
		parsed = parseSearchParams(params);
	} catch (e) {
		return error(res, e.status || 400, e.code || 'bad_request', e.message);
	}

	// Free daily quota (per IP). Exhausted → the 402 challenge.
	const quota = await limits.newsArchiveFreeIp(clientIp(req));
	if (!quota.success) return paidRail()(req, res);

	try {
		const body = await archiveSearch(parsed);
		return json(
			res, 200,
			{ ...body, tier: 'free', free_remaining_today: Math.max(0, quota.remaining ?? 0) },
			{ 'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=3600' },
		);
	} catch (e) {
		const status = e.deliberate && e.status ? e.status : 502;
		return error(res, status, e.code || 'archive_unavailable', e.message);
	}
});