Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

17.7 KB
Raw Blame
// @ts-check
// Automatic changelog delivery to the community channels (Telegram + X).
//
// This is the server-side twin of scripts/changelog-telegram.mjs and
// scripts/changelog-x.mjs. Those scripts are manual tools with repo-file
// state; this module is driven by /api/cron/changelog-push on Cloud
// Scheduler, reads the changelog feed BAKED INTO THE RUNNING IMAGE
// (public/changelog.json, regenerated by every deploy build), and keeps its
// posted-state in Postgres (app_settings). Because the feed comes from the
// deployed image, an entry can only ever be announced after the deploy that
// ships it is live; no cron ever announces an undeployed feature.
//
// State keys in app_settings:
//   changelog_push_telegram  { posted: [ "date:title", … ] }
//   changelog_push_x         { posted: […], thread: {anchor,last}|null, recent: [epochMs,…] }
//   changelog_push_lock      { until: epochSeconds }, the cross-instance run lock
//
// First run seeds each lane's posted-set from the repo state files
// (data/changelog-*-state.json, also baked into the image) so the cron
// continues exactly where the manual scripts left off. Entries older than
// CUTOFF_DAYS are never posted regardless of state, so a reset can never
// flood the channels with backlog.
//
// Message formats are kept byte-identical to the manual scripts.

import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { sql } from './db.js';
import { provenanceByEntry } from './ship-provenance.js';

import { fetchUpstream } from './upstream-fetch.js';
const CUTOFF_DAYS = 3;
const TELEGRAM_LIMIT = 8; // per run; Bot API allows ~20 msg/min per chat
const TELEGRAM_PACE_MS = 3500;
const X_LIMIT = 3; // per run
const X_PACE_MS = 10000;
const X_DAILY_CAP = 15; // free tier caps user writes at ~17/24h, stay under
// Wait applied after a 429 whose headers carry no usable reset time.
const X_BACKOFF_FALLBACK_MS = 15 * 60_000;
const LOCK_KEY = 'changelog_push_lock';
const LOCK_TTL_S = 240;
// Bounded, and well under LOCK_TTL_S on purpose: a send that hangs longer than
// the lock it is holding lets the next tick acquire that lock and re-post the
// same entry, which is exactly the double-post the lock exists to prevent.
const TELEGRAM_TIMEOUT_MS = 20_000;

const ANCHOR_TEXT = [
	'everything we ship at three.ws lands in this thread. one reply per release, straight from the changelog.',
	'',
	'full log: https://three.ws/changelog',
].join('\n');

export const entryKey = (e) => `${e.date}:${e.title}`;

function slugify(title, date) {
	const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80);
	return date + '-' + slug;
}

export function loadFeed() {
	return JSON.parse(readFileSync(join(process.cwd(), 'public', 'changelog.json'), 'utf8'));
}

function loadRepoState(file) {
	try {
		return JSON.parse(readFileSync(join(process.cwd(), 'data', file), 'utf8'));
	} catch {
		return null;
	}
}

// --- state -----------------------------------------------------------------

async function ensureTable() {
	await sql`
		CREATE TABLE IF NOT EXISTS app_settings (
			key text PRIMARY KEY,
			value jsonb NOT NULL,
			updated_at timestamptz NOT NULL DEFAULT now()
		)
	`;
}

async function getState(key) {
	const [row] = await sql`SELECT value FROM app_settings WHERE key = ${key}`;
	return row?.value ?? null;
}

async function setState(key, value) {
	await sql`
		INSERT INTO app_settings (key, value) VALUES (${key}, ${JSON.stringify(value)}::jsonb)
		ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = now()
	`;
}

// Cross-instance run lock: claim succeeds only when no unexpired claim exists.
export async function acquireLock() {
	await ensureTable();
	const rows = await sql`
		INSERT INTO app_settings (key, value)
		VALUES (${LOCK_KEY}, jsonb_build_object('until', extract(epoch from now()) + ${LOCK_TTL_S}))
		ON CONFLICT (key) DO UPDATE
			SET value = excluded.value, updated_at = now()
			WHERE (app_settings.value->>'until')::numeric < extract(epoch from now())
		RETURNING key
	`;
	return rows.length > 0;
}

export async function releaseLock() {
	await setState(LOCK_KEY, { until: 0 });
}

// --- shared selection ------------------------------------------------------

function cutoffDate() {
	return new Date(Date.now() - CUTOFF_DAYS * 86400000).toISOString().slice(0, 10);
}

export function pendingEntries(feed, posted, limit, { newestWin = false, accept = () => true } = {}) {
	const cutoff = cutoffDate();
	// The posted-set is keyed by entryKey, so it can only suppress an entry that
	// went out on an EARLIER tick. Two feed entries sharing one key are both
	// unposted when the batch is built, and the send loop posts each of them:
	// the community channel gets the identical announcement twice. That is not
	// hypothetical, `data/changelog.json` has carried six such pairs, one of
	// them added the day this guard was written. Collapse them here, keeping the
	// first, so a duplicated feed entry costs a wasted line in the data file
	// rather than a repeated message to every subscriber.
	const seen = new Set();
	const unposted = feed.entries
		.filter((e) => !posted.has(entryKey(e)) && e.date >= cutoff && e.type !== 'launch' && accept(e))
		.filter((e) => {
			const key = entryKey(e);
			if (seen.has(key)) return false;
			seen.add(key);
			return true;
		})
		.sort((a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0));
	// Both lanes pass newestWin, keeping the NEWEST entries when over the limit;
	// without it the OLDEST win. Either way the batch posts in chronological order.
	const pending = newestWin ? unposted.slice(-limit) : unposted.slice(0, limit);
	return { pending, backlog: unposted.length };
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// --- Telegram lane ----------------------------------------------------------

const escapeHtml = (s) =>
	String(s).replace(/[<>&]/g, (c) => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;' })[c]);

/**
 * @param {object} e changelog entry
 * @param {{count: number, compareUrl: string}} [provenance] the commits behind
 *   this entry, from api/_lib/ship-provenance.js. Optional on purpose: an
 *   announcement must still go out when GitHub cannot be read, so the footer
 *   line simply does not appear rather than the message failing to send.
 */
export function formatTelegramMessage(e, provenance = null) {
	const detailUrl = `https://three.ws/changelog/${slugify(e.title, e.date)}`;
	const label = e.type === 'launch' ? 'New on three.ws' : 'Update';
	const hashtags = (e.type === 'launch' ? ['launch'] : e.tags).map((t) => `#${t}`).join(' ');
	const shipped =
		provenance && provenance.count > 0
			? [
					'',
					`shipped in <a href="${provenance.compareUrl}">${provenance.count} commit${provenance.count === 1 ? '' : 's'}</a>`,
				]
			: [];
	return [
		`<b>${escapeHtml(label)}: ${escapeHtml(e.title)}</b>`,
		'',
		escapeHtml(e.summary),
		...shipped,
		'',
		`<a href="${detailUrl}">${escapeHtml(detailUrl.replace('https://', ''))}</a> · ${escapeHtml(e.date)} · ${escapeHtml(hashtags)}`,
	].join('\n');
}

async function sendTelegram(botToken, chatId, text) {
	const res = await fetchUpstream(`https://api.telegram.org/bot${botToken}/sendMessage`, {
		method: 'POST',
		headers: { 'content-type': 'application/json' },
		body: JSON.stringify({
			chat_id: chatId,
			text,
			parse_mode: 'HTML',
			link_preview_options: { is_disabled: false, prefer_small_media: true },
		}),
		signal: AbortSignal.timeout(TELEGRAM_TIMEOUT_MS),
	}, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
	const body = await res.json().catch(() => ({}));
	if (!res.ok || !body.ok) {
		throw new Error(`Telegram sendMessage failed (${res.status}): ${body.description || 'unknown error'}`);
	}
}

export async function pushTelegramLane(feed) {
	const botToken = process.env.TELEGRAM_BOT_TOKEN;
	const chatId = process.env.TELEGRAM_CHANGELOG_CHAT_ID;
	if (!botToken || !chatId) return { skipped: 'not_configured' };

	let state = await getState('changelog_push_telegram');
	if (!state) {
		// First run: adopt the manual script's committed state so nothing
		// already announced is announced twice.
		state = { posted: loadRepoState('changelog-telegram-state.json')?.posted || [] };
		await setState('changelog_push_telegram', state);
	}
	const posted = new Set(state.posted);
	const { pending, backlog } = pendingEntries(feed, posted, TELEGRAM_LIMIT, { newestWin: true });
	if (pending.length === 0) return { posted: 0, backlog };

	// One GitHub read covers the whole batch, and an unreachable GitHub simply
	// means no footer: the announcement itself never waits on it.
	const provenance = await provenanceByEntry(pending).catch(() => new Map());

	let sent = 0;
	try {
		for (const e of pending) {
			await sendTelegram(botToken, chatId, formatTelegramMessage(e, provenance.get(entryKey(e))));
			posted.add(entryKey(e));
			sent++;
			await setState('changelog_push_telegram', { posted: [...posted] });
			await sleep(TELEGRAM_PACE_MS);
		}
	} catch (err) {
		// State was persisted after every send, so the next tick resumes cleanly.
		return { posted: sent, backlog, error: String(err?.message || err) };
	}
	return { posted: sent, backlog };
}

// --- X lane ------------------------------------------------------------------

// The X lane posts to a public feed with no human in the loop, so it only ever
// sees entries that pass data/changelog-x-filter.json: nothing about wallets,
// funds, payment internals, keys, outages, or other coins, and nothing that is
// only a fix or plumbing. Telegram, a community channel, still gets them all.
export function loadXFilter() {
	const filter = JSON.parse(readFileSync(join(process.cwd(), 'data', 'changelog-x-filter.json'), 'utf8'));
	// Projects the owner put under the coin gate for announcements stay off X too.
	const gated = loadRepoState('announce-gate-terms.json')?.terms || [];
	const escape = (t) => t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
	for (const term of gated) {
		filter.patterns.push({ pattern: `\\b${escape(term)}\\b`, reason: 'A project name the owner gated in data/announce-gate-terms.json.' });
	}
	return filter;
}

/** @returns {string|null} why the entry must not go to X, or null when it may */
export function xFilterReason(e, filter) {
	const tags = e.tags || [];
	const blocked = tags.find((t) => filter.blockedTags?.[t]);
	if (blocked) return `tag ${blocked}: ${filter.blockedTags[blocked]}`;
	const required = filter.requireAnyTag;
	if (required?.tags?.length && !tags.some((t) => required.tags.includes(t))) return required.reason;
	const text = `${e.title}\n${e.summary}`;
	for (const rule of filter.patterns || []) {
		if (new RegExp(rule.pattern, 'i').test(text)) return rule.reason;
	}
	return null;
}

// X counts every URL as 23 chars (t.co wrapping) and each emoji/astral
// codepoint as 2. Compose title + summary + link within the 280 budget,
// trimming the summary on a word boundary when it overflows.
const URL_WEIGHT = 23;
export const weightedLength = (s) => [...s].reduce((n, ch) => n + (ch.codePointAt(0) > 0xffff ? 2 : 1), 0);

export function formatTweet(e) {
	const detailUrl = `https://three.ws/changelog/${slugify(e.title, e.date)}`;
	const suffix = `\n\n${detailUrl}`;
	const suffixWeight = 2 + URL_WEIGHT;
	const budget = 280 - suffixWeight;

	let body = `${e.title}\n\n${e.summary}`;
	if (weightedLength(body) > budget) {
		const titlePart = `${e.title}\n\n`;
		const summaryBudget = budget - weightedLength(titlePart) - 1; // 1 for the ellipsis
		let summary = e.summary;
		while (weightedLength(summary) > summaryBudget && summary.includes(' ')) {
			summary = summary.slice(0, summary.lastIndexOf(' '));
		}
		if (weightedLength(summary) > summaryBudget) summary = summary.slice(0, summaryBudget);
		body = `${titlePart}${summary.replace(/[\s,.;:]+$/, '')}…`;
	}
	return `${body}${suffix}`;
}

// How an X write failure is handled, decided from the error alone so the
// policy is testable without the network. twitter-api-v2 raises an
// ApiResponseError carrying the HTTP status (`code`), the v2 problem body
// (`data`), and the parsed rate-limit headers (`rateLimit`).
//   rate_limited  wait until X's own reset time, then resume (not a failure)
//   duplicate     X already holds this exact text; count it posted and move on,
//                 otherwise the same entry blocks the lane on every tick
//   thread_gone   the reply target was deleted; re-chain and retry
//   fatal         auth, suspension, outage: surface it as a lane error
export function classifyXError(err, now = Date.now()) {
	const detail = [err?.data?.detail, err?.data?.title, err?.data?.errors?.[0]?.message, err?.message]
		.filter(Boolean)
		.join(' ');
	if (err?.code === 429 || err?.rateLimitError) {
		const rl = err?.rateLimit;
		// The 24h user/app write windows reset far later than the 15-minute one;
		// when one of them is the window that ran out, its reset is the real one.
		const window = [rl?.userDay, rl?.day].find((w) => w && w.remaining === 0) || rl;
		const resetMs = Number(window?.reset) * 1000;
		const until = Number.isFinite(resetMs) && resetMs > now ? resetMs : now + X_BACKOFF_FALLBACK_MS;
		return { kind: 'rate_limited', until };
	}
	if (/duplicate content/i.test(detail)) return { kind: 'duplicate' };
	if (/deleted or not visible|in_reply_to_tweet_id/i.test(detail)) return { kind: 'thread_gone' };
	return { kind: 'fatal' };
}

export async function pushXLane(feed) {
	const creds = {
		appKey: process.env.X_API_KEY,
		appSecret: process.env.X_API_SECRET,
		accessToken: process.env.X_ACCESS_TOKEN,
		accessSecret: process.env.X_ACCESS_SECRET,
	};
	if (!(creds.appKey && creds.appSecret && creds.accessToken && creds.accessSecret)) {
		return { skipped: 'not_configured' };
	}

	let state = await getState('changelog_push_x');
	if (!state) {
		const repo = loadRepoState('changelog-x-state.json') || {};
		state = { posted: repo.posted || [], thread: repo.thread || null, recent: [] };
		await setState('changelog_push_x', state);
	}
	const posted = new Set(state.posted);
	let thread = state.thread || null;
	// Free-tier write budget: timestamps of posts in the trailing 24h.
	let recent = (state.recent || []).filter((t) => Date.now() - t < 86400000);
	let backoffUntil = state.backoffUntil || 0;
	let lastPostAt = state.lastPostAt || null;
	let lastError = state.lastError || null;

	const saveState = () =>
		setState('changelog_push_x', { posted: [...posted], thread, recent, backoffUntil, lastPostAt, lastError });

	// X is a rate-limited showcase, not the full log: when there is more to say
	// than the daily budget allows, the newest releases win (as on Telegram),
	// otherwise a busy week keeps the lane posting entries days late.
	const filter = loadXFilter();
	const { pending, backlog } = pendingEntries(feed, posted, X_LIMIT, { newestWin: true, accept: (e) => !xFilterReason(e, filter) });
	if (pending.length === 0) {
		await saveState(); // persist the pruned 24h window
		return { posted: 0, backlog };
	}
	if (backoffUntil > Date.now()) {
		return { posted: 0, backlog, skipped: 'rate_limited', until: new Date(backoffUntil).toISOString() };
	}
	if (recent.length >= X_DAILY_CAP) {
		return { posted: 0, backlog, skipped: 'daily_cap' };
	}

	const { TwitterApi } = await import('twitter-api-v2');
	const client = new TwitterApi(creds);
	const post = async (text, replyToId) => {
		const payload = replyToId ? { reply: { in_reply_to_tweet_id: replyToId } } : undefined;
		const { data } = await client.v2.tweet(text, payload);
		recent.push(Date.now());
		lastPostAt = new Date().toISOString();
		return data.id;
	};

	let sent = 0;
	let anchorPosted = null;
	// The profile keeps a single pinned anchor; every release chains under it
	// as a reply. The owner pins the anchor once, and again after a re-anchor.
	const startThread = async () => {
		let id;
		try {
			id = await post(ANCHOR_TEXT);
		} catch (err) {
			// Never let an anchor failure read as a per-entry duplicate below: that
			// would mark releases posted that never went out.
			if (classifyXError(err).kind !== 'duplicate') throw err;
			throw Object.assign(new Error(`X rejected the thread anchor as duplicate content: ${err.message}`), { code: err.code });
		}
		thread = { anchor: id, last: id };
		anchorPosted = id;
		await saveState();
		await sleep(X_PACE_MS);
	};
	// Reply under the chain. A deleted tweet in the chain would otherwise fail
	// every tick forever: fall back to the anchor, then to a fresh anchor.
	const postReply = async (text) => {
		for (let attempt = 0; ; attempt++) {
			if (!thread) await startThread();
			try {
				return await post(text, thread.last);
			} catch (err) {
				if (attempt >= 2 || classifyXError(err).kind !== 'thread_gone') throw err;
				thread = thread.last !== thread.anchor ? { ...thread, last: thread.anchor } : null;
				await saveState();
			}
		}
	};

	try {
		for (const e of pending) {
			if (recent.length >= X_DAILY_CAP) break;
			let id;
			try {
				id = await postReply(formatTweet(e));
			} catch (err) {
				if (classifyXError(err).kind !== 'duplicate') throw err;
				posted.add(entryKey(e));
				await saveState();
				continue;
			}
			posted.add(entryKey(e));
			thread = { ...thread, last: id };
			sent++;
			lastError = null;
			// Persist after every post so a crash can't double-post or fork the chain.
			await saveState();
			await sleep(X_PACE_MS);
		}
	} catch (err) {
		const verdict = classifyXError(err);
		if (verdict.kind === 'rate_limited') {
			backoffUntil = verdict.until;
			await saveState().catch(() => {});
			return { posted: sent, backlog, anchor: anchorPosted, skipped: 'rate_limited', until: new Date(backoffUntil).toISOString() };
		}
		lastError = { at: new Date().toISOString(), status: err?.code ?? null, message: String(err?.message || err).slice(0, 300) };
		await saveState().catch(() => {});
		return { posted: sent, backlog, anchor: anchorPosted, error: lastError.message };
	}
	return { posted: sent, backlog, anchor: anchorPosted };
}