Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

11.9 KB
Raw Blame
// GET /api/cron/news-archive-append - hourly continuous archiver for the
// crypto-news corpus on gs://three-ws-news-archive.
//
// The recovered historical archive ends at 2025-12; this cron keeps it
// current by appending the live aggregator's articles (api/_lib/news.js,
// the full publisher registry) to the current month's JSONL, in the exact enriched
// schema the corpus already uses. Records are content-addressed (16-hex id
// of the link), so appends are idempotent - an article seen by ten cron
// runs lands exactly once.
//
// Writes go through the GCS JSON API with the platform's own credentials
// (api/_lib/gcp-auth.js - GCP_SERVICE_ACCOUNT_JSON or the Cloud Run runtime
// SA, which holds roles/storage.objectAdmin on the bucket) and are guarded
// with x-goog-if-generation-match so two overlapping runs can't clobber
// each other; the loser retries next hour.
//
// ?dry_run=1 computes and reports without writing.

import { json, method, wrapCron } from '../_lib/http.js';
import { getGcpAccessToken } from '../_lib/gcp-auth.js';
import { getNews } from '../_lib/news.js';
import { geckoFetch } from '../_lib/coingecko.js';
import { excerptText } from '../_lib/news-rights.js';
import { requireCron } from '../_lib/cron-auth.js';

const BUCKET = 'three-ws-news-archive';
const API = `https://storage.googleapis.com/storage/v1/b/${BUCKET}/o`;
const UPLOAD = `https://storage.googleapis.com/upload/storage/v1/b/${BUCKET}/o`;

// ── GCS helpers (JSON API, no client-library dependency) ────────────────────

async function gcsGetObject(token, name) {
	// metadata (for generation) + content in two small calls
	const metaResp = await fetch(`${API}/${encodeURIComponent(name)}`, {
		headers: { authorization: `Bearer ${token}` },
		signal: AbortSignal.timeout(15_000),
	});
	if (metaResp.status === 404) return { exists: false, generation: '0', text: '' };
	if (!metaResp.ok) throw new Error(`GCS meta ${name} → ${metaResp.status}`);
	const meta = await metaResp.json();
	const dataResp = await fetch(`${API}/${encodeURIComponent(name)}?alt=media`, {
		headers: { authorization: `Bearer ${token}` },
		signal: AbortSignal.timeout(60_000),
	});
	if (!dataResp.ok) throw new Error(`GCS read ${name} → ${dataResp.status}`);
	return { exists: true, generation: meta.generation, text: await dataResp.text() };
}

async function gcsPutObject(token, name, body, generation, contentType) {
	const resp = await fetch(
		`${UPLOAD}?uploadType=media&name=${encodeURIComponent(name)}&ifGenerationMatch=${generation}`,
		{
			method: 'POST',
			headers: {
				authorization: `Bearer ${token}`,
				'content-type': contentType,
				'x-goog-if-generation-match': generation,
			},
			body,
			signal: AbortSignal.timeout(60_000),
		},
	);
	if (resp.status === 412) throw Object.assign(new Error(`GCS precondition failed on ${name}`), { code: 'race' });
	if (!resp.ok) throw new Error(`GCS write ${name} → ${resp.status} ${(await resp.text()).slice(0, 120)}`);
	return resp.json();
}

// ── Record conversion - live article → enriched archive schema ──────────────

function detectLanguage(title) {
	if (/[぀-ヿ]/.test(title)) return 'ja'; // hiragana/katakana
	if (/[一-鿿]/.test(title)) return 'zh';
	return 'en';
}

function toArchiveRecord(a, marketContext, nowIso) {
	return {
		id: a.id,
		schema_version: '2.0.0',
		title: a.title,
		link: a.link,
		canonical_link: a.link,
		// Bounded at capture: feeds shipping content:encoded put the entire
		// article in `description`, and the archive is a durable, publicly
		// searchable store. We keep a quotable lead and the link, never the body.
		description: excerptText(a.description),
		source: a.source,
		source_key: a.source_key,
		category: a.category,
		pub_date: a.pub_date,
		first_seen: nowIso,
		last_seen: nowIso,
		fetch_count: 1,
		tickers: a.tickers,
		entities: { people: [], companies: [], protocols: [] },
		tags: [],
		sentiment: a.sentiment,
		market_context: marketContext,
		content_hash: a.id,
		meta: {
			word_count: (a.description || a.title).split(/\s+/).length,
			has_numbers: /\d/.test(`${a.title} ${a.description || ''}`),
			is_breaking: false,
			is_opinion: false,
			has_url: true,
			import_source: 'three.ws-live-archiver',
			language: detectLanguage(a.title),
		},
	};
}

// Market context at capture time - real values or null, never fabricated.
async function currentMarketContext() {
	try {
		const [global, fng] = await Promise.all([
			geckoFetch('/global', { ttlMs: 120_000 }),
			fetch('https://api.alternative.me/fng/?limit=1', { signal: AbortSignal.timeout(6000) })
				.then((r) => (r.ok ? r.json() : null))
				.catch(() => null),
		]);
		const prices = await geckoFetch('/simple/price?ids=bitcoin,ethereum,solana&vs_currencies=usd', {
			ttlMs: 120_000,
		});
		return {
			btc_price: prices?.bitcoin?.usd ?? null,
			eth_price: prices?.ethereum?.usd ?? null,
			sol_price: prices?.solana?.usd ?? null,
			total_market_cap: global?.data?.total_market_cap?.usd ?? null,
			btc_dominance: global?.data?.market_cap_percentage?.btc ?? null,
			fear_greed_index: fng?.data?.[0]?.value != null ? Number(fng.data[0].value) : null,
		};
	} catch {
		return null;
	}
}

// ─────────────────────────────────────────────────────────────────────────────

function previousMonth(month) {
	const d = new Date(`${month}-01T00:00:00Z`);
	d.setUTCMonth(d.getUTCMonth() - 1);
	return d.toISOString().slice(0, 7);
}

export default wrapCron(async (req, res) => {
	if (!method(req, res, ['GET'])) return;
	if (!requireCron(req, res)) return;

	const params = new URL(req.url, 'http://x').searchParams;
	const dryRun = params.get('dry_run') === '1';
	const nowIso = new Date().toISOString();
	const month = nowIso.slice(0, 7); // YYYY-MM

	let token = null;
	if (!dryRun) {
		// getGcpAccessToken() throws `unconfigured` both when no credentials are
		// set AND when the Cloud Run metadata server is momentarily unreachable.
		// Unguarded, that turned a transient credential blip into an opaque 500
		// ("internal error, quote ref …") and an ops page, for the same reason the
		// generation-guard race below is caught rather than thrown: the archiver
		// is content-addressed and hourly, so a skipped run costs nothing but a
		// delay. Degrade the way wrapCron degrades an unavailable DB - a 200 the
		// scheduler will not retry-storm, carrying ok:false so monitoring can see
		// the archiver is not writing.
		try {
			token = await getGcpAccessToken();
		} catch (err) {
			if (err?.code !== 'unconfigured') throw err;
			console.error('news-archive-append: no GCP credentials, skipping this run:', err.message);
			return json(res, 200, { ok: false, month, appended: 0, reason: 'gcp_unconfigured' });
		}
	}

	// Live sweep across every source - the aggregator serves from its
	// per-source cache, so this is one bounded fan-out at most. The limit must
	// exceed the whole live corpus (~3k across 159 sources): anything the slice
	// drops here is silently never archived, and its permalink dies when the
	// story rotates out of the feed.
	const live = await getNews({ limit: 10_000 });
	if (!live.articles.length) {
		return json(res, 200, { ok: true, month, appended: 0, note: 'aggregator returned no articles' });
	}

	// Bucket each article into ITS publication month - the same month its
	// permalink (/markets/news/<YYYY-MM>/<id>) points at. Archiving a late-June
	// story into the July file would strand its canonical URL forever. One
	// month of grace covers boundary stragglers; anything older is feed
	// backlog that stays out so it can't pollute the historical corpus.
	const grace = previousMonth(month);
	const buckets = new Map(); // YYYY-MM → articles
	let skippedBacklog = 0;
	for (const a of live.articles) {
		const m = a.pub_date ? a.pub_date.slice(0, 7) : month;
		if (m !== month && m !== grace) {
			skippedBacklog++;
			continue;
		}
		if (!buckets.has(m)) buckets.set(m, []);
		buckets.get(m).push(a);
	}

	// Per-month dedupe against the existing archive object.
	const months = {}; // YYYY-MM → { existing, fresh }
	let alreadyArchived = 0;
	for (const [m, articles] of buckets) {
		const existing = dryRun
			? { exists: false, generation: '0', text: '' }
			: await gcsGetObject(token, `articles/${m}.jsonl`);
		const seen = new Set();
		for (const line of existing.text.split('\n')) {
			if (!line.trim()) continue;
			try {
				seen.add(JSON.parse(line).id);
			} catch {
				// malformed line - leave it in place, it just can't dedupe
			}
		}
		alreadyArchived += seen.size;
		months[m] = { existing, fresh: articles.filter((a) => !seen.has(a.id)) };
	}

	const allFresh = Object.values(months).flatMap((x) => x.fresh);
	if (!allFresh.length) {
		return json(res, 200, {
			ok: true, month, appended: 0, live_considered: live.articles.length,
			already_archived: alreadyArchived, skipped_backlog: skippedBacklog, dry_run: dryRun,
		});
	}

	const marketContext = await currentMarketContext();
	const appendedByMonth = {};
	const racedMonths = [];
	// Only the articles this run actually landed. The stats rollup below must
	// not count a month whose write lost the generation guard.
	const written = [];

	if (!dryRun) {
		for (const [m, { existing, fresh }] of Object.entries(months)) {
			if (!fresh.length) continue;
			const lines = fresh.map((a) => JSON.stringify(toArchiveRecord(a, marketContext, nowIso)));
			const body = existing.text
				? `${existing.text.replace(/\n$/, '')}\n${lines.join('\n')}\n`
				: `${lines.join('\n')}\n`;
			try {
				await gcsPutObject(token, `articles/${m}.jsonl`, body, existing.generation, 'application/x-ndjson');
			} catch (err) {
				// The header promises the loser of an overlapping run just retries
				// next hour, but an uncaught 412 threw out of the handler instead:
				// wrap() turns that into a 5xx and pages ops for a race the design
				// already tolerates. Records are content-addressed, so the winner
				// wrote the same ids and nothing is lost.
				if (err?.code !== 'race') throw err;
				racedMonths.push(m);
				console.warn(`news-archive-append: ${m} lost the generation guard to an overlapping run; retrying next hour`);
				continue;
			}
			appendedByMonth[m] = fresh.length;
			written.push(...fresh);
		}

		// Keep corpus stats truthful for /markets/archive and the API's
		// stats mode. Best-effort: a stats race loses to next hour's run.
		try {
			const stats = written.length ? await gcsGetObject(token, 'meta/stats.json') : { exists: false };
			if (stats.exists) {
				const parsed = JSON.parse(stats.text);
				parsed.total_articles += written.length;
				parsed.total_with_url += written.length;
				parsed.total_with_date += written.filter((a) => a.pub_date).length;
				parsed.total_with_description += written.filter((a) => a.description).length;
				const newest = written.map((a) => a.pub_date).filter(Boolean).sort().pop();
				if (newest && (!parsed.last_article_date || newest > parsed.last_article_date)) {
					parsed.last_article_date = newest;
				}
				for (const a of written) {
					parsed.sources[a.source_key] = (parsed.sources[a.source_key] || 0) + 1;
				}
				await gcsPutObject(token, 'meta/stats.json', JSON.stringify(parsed, null, 2), stats.generation, 'application/json');
			}
		} catch (err) {
			if (err?.code !== 'race') console.error('news-archive-append: stats update skipped:', err.message);
		}
	} else {
		for (const [m, { fresh }] of Object.entries(months)) {
			if (fresh.length) appendedByMonth[m] = fresh.length;
		}
	}

	return json(res, 200, {
		ok: true,
		month,
		appended: dryRun ? allFresh.length : written.length,
		appended_by_month: appendedByMonth,
		...(racedMonths.length ? { raced_months: racedMonths } : {}),
		live_considered: live.articles.length,
		already_archived: alreadyArchived,
		skipped_backlog: skippedBacklog,
		sources_live: `${live.sources_ok}/${live.sources_total}`,
		market_context: marketContext ? 'captured' : 'unavailable',
		dry_run: dryRun,
	});
});