Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

10.8 KB
Raw Blame
// Coverage for api/cron/news-archive-append.js — the hourly continuous
// archiver. Verifies the cron gate, dry-run accounting, id-dedupe against the
// existing month object, month-scoping of stale feed items, the enriched
// record schema, and the generation-guarded GCS writes. All I/O mocked.

import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';

vi.mock('../api/_lib/http.js', () => ({
	wrapCron: (fn) => fn,
	method: () => true,
	json: (res, status, body) => {
		res._json = { status, body };
		return res;
	},
	error: (res, status, code, message) => {
		res._json = { status, body: { error: code, message } };
		return res;
	},
}));
vi.mock('../api/_lib/env.js', () => ({ env: {} }));
vi.mock('../api/_lib/crypto.js', () => ({
	constantTimeEquals: (a, b) => a === b,
}));
vi.mock('../api/_lib/gcp-auth.js', () => ({
	getGcpAccessToken: vi.fn(async () => 'gcs-token'),
}));
vi.mock('../api/_lib/coingecko.js', () => ({
	geckoFetch: vi.fn(async (path) =>
		path.startsWith('/global')
			? { data: { total_market_cap: { usd: 2.2e12 }, market_cap_percentage: { btc: 56.3 } } }
			: { bitcoin: { usd: 64000 }, ethereum: { usd: 1800 }, solana: { usd: 80 } },
	),
}));

const NOW_MONTH = new Date().toISOString().slice(0, 7);
const PREV_MONTH = (() => {
	const d = new Date(`${NOW_MONTH}-01T00:00:00Z`);
	d.setUTCMonth(d.getUTCMonth() - 1);
	return d.toISOString().slice(0, 7);
})();
const LIVE = [
	{
		id: 'aaaaaaaaaaaaaaaa', title: 'BTC rallies', link: 'https://x.com/a', description: 'Bitcoin gains 5%',
		image: null, author: null, source: 'CoinDesk', source_key: 'coindesk', category: 'general',
		pub_date: `${NOW_MONTH}-05T10:00:00.000Z`, tickers: ['BTC'],
		sentiment: { score: 0.4, label: 'positive', confidence: 0.7 },
	},
	{
		id: 'bbbbbbbbbbbbbbbb', title: 'Already archived story', link: 'https://x.com/b', description: null,
		image: null, author: null, source: 'Decrypt', source_key: 'decrypt', category: 'general',
		pub_date: `${NOW_MONTH}-04T10:00:00.000Z`, tickers: [],
		sentiment: { score: 0, label: 'neutral', confidence: 0.5 },
	},
	{
		id: 'cccccccccccccccc', title: 'Stale backlog item', link: 'https://x.com/c', description: null,
		image: null, author: null, source: 'Protos', source_key: 'protos', category: 'journalism',
		pub_date: '2020-01-01T10:00:00.000Z', tickers: [],
		sentiment: { score: 0, label: 'neutral', confidence: 0.5 },
	},
	{
		id: 'dddddddddddddddd', title: 'Late boundary story', link: 'https://x.com/d', description: 'Published just before the month rolled',
		image: null, author: null, source: 'The Block', source_key: 'theblock', category: 'general',
		pub_date: `${PREV_MONTH}-28T23:50:00.000Z`, tickers: [],
		sentiment: { score: 0, label: 'neutral', confidence: 0.5 },
	},
];
vi.mock('../api/_lib/news.js', () => ({
	getNews: vi.fn(async () => ({ articles: LIVE, total: LIVE.length, sources_ok: 37, sources_total: 38 })),
}));

const { default: handler } = await import('../api/cron/news-archive-append.js');

function call(url, auth = 'Bearer s3cret') {
	const res = { setHeader() {}, end() {}, statusCode: 200 };
	return handler({ method: 'GET', url, headers: { authorization: auth } }, res).then(() => res);
}

const originalFetch = global.fetch;
let writes;

beforeEach(() => {
	process.env.CRON_SECRET = 's3cret';
	writes = [];
	global.fetch = vi.fn(async (url, opts = {}) => {
		const u = String(url);
		if (opts.method === 'POST') {
			writes.push({ url: u, body: String(opts.body), headers: opts.headers });
			return { ok: true, status: 200, json: async () => ({ generation: '2' }), text: async () => '' };
		}
		// existing month object holds article b already
		if (u.includes(`articles%2F${NOW_MONTH}.jsonl`) && !u.includes('alt=media')) {
			return { ok: true, status: 200, json: async () => ({ generation: '41' }) };
		}
		if (u.includes(`articles%2F${NOW_MONTH}.jsonl`) && u.includes('alt=media')) {
			return { ok: true, status: 200, text: async () => `${JSON.stringify({ id: 'bbbbbbbbbbbbbbbb' })}\n` };
		}
		if (u.includes('meta%2Fstats.json') && !u.includes('alt=media')) {
			return { ok: true, status: 200, json: async () => ({ generation: '7' }) };
		}
		if (u.includes('meta%2Fstats.json') && u.includes('alt=media')) {
			return {
				ok: true, status: 200,
				text: async () => JSON.stringify({
					total_articles: 662047, total_with_url: 340047, total_with_date: 487690,
					total_with_description: 27791, last_article_date: '2025-12-03T10:28:51.000Z',
					sources: { coindesk: 2108 },
				}),
			};
		}
		if (u.includes('alternative.me')) {
			return { ok: true, status: 200, json: async () => ({ data: [{ value: '23' }] }) };
		}
		return { ok: false, status: 404, json: async () => ({}), text: async () => '' };
	});
});
afterEach(() => {
	delete process.env.CRON_SECRET;
	global.fetch = originalFetch;
	vi.restoreAllMocks();
});

describe('GET /api/cron/news-archive-append', () => {
	it('rejects a bad cron secret', async () => {
		const res = await call('/api/cron/news-archive-append', 'Bearer wrong');
		expect(res._json.status).toBe(401);
	});

	it('dry_run reports counts without any GCS write', async () => {
		const res = await call('/api/cron/news-archive-append?dry_run=1');
		expect(res._json.status).toBe(200);
		expect(res._json.body.dry_run).toBe(true);
		// dry run skips the existing-object read, so all in-window items count
		expect(res._json.body.appended).toBe(3);
		expect(res._json.body.appended_by_month).toEqual({ [NOW_MONTH]: 2, [PREV_MONTH]: 1 });
		expect(res._json.body.skipped_backlog).toBe(1);
		expect(writes).toHaveLength(0);
	});

	it('appends new articles into their own pub_date month, schema-complete, generation-guarded', async () => {
		const res = await call('/api/cron/news-archive-append');
		expect(res._json.status).toBe(200);
		expect(res._json.body.appended).toBe(2); // b deduped, c out-of-window
		expect(res._json.body.appended_by_month).toEqual({ [NOW_MONTH]: 1, [PREV_MONTH]: 1 });
		expect(res._json.body.skipped_backlog).toBe(1);
		const monthWrite = writes.find((w) => w.url.includes(`articles%2F${NOW_MONTH}.jsonl`));
		expect(monthWrite).toBeTruthy();
		expect(monthWrite.url).toContain('ifGenerationMatch=41');
		const lines = monthWrite.body.trim().split('\n');
		expect(lines).toHaveLength(2); // existing line + 1 appended
		const rec = JSON.parse(lines[1]);
		expect(rec).toMatchObject({
			id: 'aaaaaaaaaaaaaaaa',
			schema_version: '2.0.0',
			source_key: 'coindesk',
			tickers: ['BTC'],
			content_hash: 'aaaaaaaaaaaaaaaa',
		});
		expect(rec.market_context.btc_price).toBe(64000);
		expect(rec.market_context.fear_greed_index).toBe(23);
		expect(rec.meta.language).toBe('en');
		expect(rec.meta.import_source).toBe('three.ws-live-archiver');
	});

	it('a boundary straggler lands in the PREVIOUS month file, not the current one', async () => {
		await call('/api/cron/news-archive-append');
		const graceWrite = writes.find((w) => w.url.includes(`articles%2F${PREV_MONTH}.jsonl`));
		expect(graceWrite).toBeTruthy();
		expect(graceWrite.url).toContain('ifGenerationMatch=0'); // fresh object
		const lines = graceWrite.body.trim().split('\n');
		expect(lines).toHaveLength(1);
		expect(JSON.parse(lines[0]).id).toBe('dddddddddddddddd');
		// and it must NOT leak into the current month file
		const monthWrite = writes.find((w) => w.url.includes(`articles%2F${NOW_MONTH}.jsonl`));
		expect(monthWrite.body).not.toContain('dddddddddddddddd');
	});

	it('updates corpus stats after an append', async () => {
		await call('/api/cron/news-archive-append');
		const statsWrite = writes.find((w) => w.url.includes('meta%2Fstats.json'));
		expect(statsWrite).toBeTruthy();
		expect(statsWrite.url).toContain('ifGenerationMatch=7');
		const stats = JSON.parse(statsWrite.body);
		expect(stats.total_articles).toBe(662049);
		expect(stats.sources.coindesk).toBe(2109);
		expect(stats.sources.theblock).toBe(1);
		expect(stats.last_article_date).toBe(`${NOW_MONTH}-05T10:00:00.000Z`);
	});
});

describe('an overlapping run losing the GCS generation guard', () => {
	// The header promises the loser of a race just retries next hour, but the
	// 412 threw straight out of the handler, so wrap() turned a benign,
	// designed-for race into a 5xx that pages ops. Records are content
	// addressed, so the winner wrote the same ids and nothing was lost.
	it('is tolerated per month: the run still 200s and the other month still lands', async () => {
		const base = global.fetch;
		global.fetch = vi.fn(async (url, opts = {}) => {
			if (opts.method === 'POST' && String(url).includes(`articles%2F${NOW_MONTH}.jsonl`)) {
				return { ok: false, status: 412, json: async () => ({}), text: async () => 'precondition failed' };
			}
			return base(url, opts);
		});
		const res = await call('/api/cron/news-archive-append');
		expect(res._json.status).toBe(200);
		expect(res._json.body.raced_months).toEqual([NOW_MONTH]);
		expect(res._json.body.appended_by_month).toEqual({ [PREV_MONTH]: 1 });
		expect(res._json.body.appended).toBe(1);
	});

	it('does not credit the raced month to the corpus stats', async () => {
		const base = global.fetch;
		global.fetch = vi.fn(async (url, opts = {}) => {
			if (opts.method === 'POST' && String(url).includes(`articles%2F${NOW_MONTH}.jsonl`)) {
				return { ok: false, status: 412, json: async () => ({}), text: async () => 'precondition failed' };
			}
			return base(url, opts);
		});
		await call('/api/cron/news-archive-append');
		const statsWrite = writes.find((w) => w.url.includes('meta%2Fstats.json'));
		// 662047 + only the one article that actually landed
		expect(JSON.parse(statsWrite.body).total_articles).toBe(662048);
		expect(JSON.parse(statsWrite.body).sources.coindesk).toBe(2108);
	});

	it('degrades to a skipped run when GCP credentials are unavailable', async () => {
		const { getGcpAccessToken } = await import('../api/_lib/gcp-auth.js');
		getGcpAccessToken.mockRejectedValueOnce(
			Object.assign(new Error('No GCP credentials found.'), { code: 'unconfigured' }),
		);
		const res = await call('/api/cron/news-archive-append');
		expect(res._json.status).toBe(200);
		expect(res._json.body.ok).toBe(false);
		expect(res._json.body.reason).toBe('gcp_unconfigured');
		expect(res._json.body.appended).toBe(0);
		expect(writes).toHaveLength(0);
	});

	it('a credential failure that is not `unconfigured` still fails the run', async () => {
		const { getGcpAccessToken } = await import('../api/_lib/gcp-auth.js');
		getGcpAccessToken.mockRejectedValueOnce(new Error('token exchange returned 500'));
		await expect(call('/api/cron/news-archive-append')).rejects.toThrow(/token exchange/);
	});

	it('a real GCS write failure still fails the run', async () => {
		const base = global.fetch;
		global.fetch = vi.fn(async (url, opts = {}) => {
			if (opts.method === 'POST') {
				return { ok: false, status: 503, json: async () => ({}), text: async () => 'backend error' };
			}
			return base(url, opts);
		});
		await expect(call('/api/cron/news-archive-append')).rejects.toThrow(/GCS write/);
	});
});