Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(shipfeed): join every release note to the commits that produced it
nirholasd8240b5e5d59
diff --git a/api/ship/feed.js b/api/ship/feed.jsnew file mode 100644index 000000000..9bf95fa29--- /dev/null+++ b/api/ship/feed.js@@ -0,0 +1,229 @@+/**+ * Ship log: every release note joined to the commits that produced it.+ *+ * GET /api/ship/feed+ * GET /api/ship/feed?limit=300&audience=holder&format=markdown+ * GET /api/ship/feed?explain=<sha>+ *+ * The changelog says what shipped in a sentence a holder can read. The commit+ * log says what shipped in code. Nothing has ever pointed the two at each+ * other, which is why the same work reaches the Telegram channel twice, in two+ * voices, with no way to tell that it is the same work. This endpoint is that+ * join, computed on demand by @three-ws/shipfeed:+ *+ * releases[] a changelog entry, its commits, and a confidence + reason for+ * every link, so a wrong link is visible instead of implied+ * ships[] bursts of commits no entry ever claimed (the unannounced work)+ * stats audience mix, coverage, velocity, types, authors+ *+ * Sources are both real and both first-party-adjacent: the changelog is the+ * feed baked into this running image (public/changelog.json, regenerated by+ * every deploy), and the commits come from GitHub's REST API through+ * fetchUpstream, so the call is bounded, retried, breaker-guarded, and served+ * from last-good on an upstream blip.+ *+ * Public, no auth, cached for 5 minutes.+ *+ * Params+ * limit commits to read, 1..500 (default 200)+ * audience holder | developer | internal (default internal, i.e. everything)+ * format json | markdown | rss (default json)+ * explain a commit sha; returns that one commit's scoring and link reasons+ */++import { readFileSync } from 'node:fs';+import { join } from 'node:path';++import { cors, json, method, wrap, rateLimited, error } from '../_lib/http.js';+import { limits, clientIp } from '../_lib/rate-limit.js';+import { cacheGet, cacheSet } from '../_lib/cache.js';+import { fetchUpstream } from '../_lib/upstream-fetch.js';+import {+ buildShipFeed,+ classify,+ fetchGitHubCommits,+ linkCommits,+ normalizeChangelog,+ parseCommit,+ renderMarkdown,+ renderRss,+} from '../../packages/shipfeed/src/index.js';++const REPO = 'nirholas/three.ws';+const SITE = 'https://three.ws';+const CACHE_TTL_S = 300;+const DEFAULT_LIMIT = 200;+const MAX_LIMIT = 500;+const AUDIENCES = new Set(['holder', 'developer', 'internal']);+const FORMATS = new Set(['json', 'markdown', 'rss']);++// Scopes this repo uses for parts of the product a reader has actually seen.+// Naming them is what lifts "fix(forge): ..." above "fix(build): ..." in the+// signal score; everything unnamed still scores on its own merits.+const PRODUCT_SCOPES = [+ 'agent',+ 'agents',+ 'avatar',+ 'avatars',+ 'chat',+ 'companion',+ 'crews',+ 'discover',+ 'embed',+ 'embeds',+ 'forge',+ 'launch',+ 'marketplace',+ 'oracle',+ 'payments',+ 'pump',+ 'seeker',+ 'studio',+ 'viewer',+ 'wallet',+ 'x402',+];++/** The changelog baked into this image by the deploy that shipped it. */+function loadChangelog() {+ return normalizeChangelog(+ JSON.parse(readFileSync(join(process.cwd(), 'public', 'changelog.json'), 'utf8')),+ );+}++/**+ * GitHub through the shared upstream wrapper: one deadline, one retry, one+ * breaker. `okWhen` is deliberately permissive so the package sees the real+ * status and can name a rate-limit reset instead of a bare 503.+ */+const boundedFetch = (url, init) =>+ fetchUpstream(url, init, {+ name: 'github-commits',+ timeoutMs: 12_000,+ attempts: 2,+ okWhen: () => true,+ });++async function buildFeed({ limit, audience }) {+ const key = `ship:feed:${limit}:${audience}`;+ const cached = await cacheGet(key);+ if (cached) return { ...cached, cached: true };++ const entries = loadChangelog();+ const commits = await fetchGitHubCommits({+ repo: REPO,+ branch: 'main',+ limit,+ token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '',+ fetchImpl: boundedFetch,+ userAgent: 'three.ws-shipfeed',+ });++ const feed = buildShipFeed({+ commits,+ entries,+ repo: REPO,+ siteUrl: SITE,+ minAudience: audience,+ classify: { productScopes: PRODUCT_SCOPES },+ });+ await cacheSet(key, feed, CACHE_TTL_S);+ return { ...feed, cached: false };+}++/**+ * One commit's full reasoning: why it scored what it scored, and why it is or+ * is not attached to a release note. This is the accountability half of the+ * feature; a heuristic nobody can interrogate is a heuristic nobody should+ * trust.+ */+async function explainCommit(sha, { limit }) {+ const entries = loadChangelog();+ const raw = await fetchGitHubCommits({+ repo: REPO,+ branch: 'main',+ limit,+ token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '',+ fetchImpl: boundedFetch,+ userAgent: 'three.ws-shipfeed',+ });+ const commits = raw.map((c) => parseCommit(c));+ const commit = commits.find((c) => c.sha.startsWith(sha));+ if (!commit) return null;++ const classification = classify(commit, { productScopes: PRODUCT_SCOPES });+ const { byCommit } = linkCommits(entries, commits);+ const link = byCommit.get(commit.sha) || null;+ const entry = link ? entries.find((e) => `${e.date}:${e.title}` === link.entryKey) : null;++ return {+ commit: {+ sha: commit.sha,+ shortSha: commit.shortSha,+ url: commit.url,+ subject: commit.subject,+ type: commit.type,+ scope: commit.scope,+ breaking: commit.breaking,+ date: commit.committedAt,+ author: commit.author.login || commit.author.name,+ },+ classification,+ link: link+ ? { confidence: link.score, reasons: link.reasons, entry, key: link.entryKey }+ : null,+ };+}++export default wrap(async (req, res) => {+ if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;+ if (!method(req, res, ['GET'])) return;++ const rl = await limits.publicIp(clientIp(req));+ if (!rl.success) return rateLimited(res, rl);++ const params = new URL(req.url, `http://${req.headers.host || 'x'}`).searchParams;+ const limit = Math.min(MAX_LIMIT, Math.max(1, Number(params.get('limit')) || DEFAULT_LIMIT));+ const audience = AUDIENCES.has(params.get('audience')) ? params.get('audience') : 'internal';+ const format = FORMATS.has(params.get('format')) ? params.get('format') : 'json';+ const explain = String(params.get('explain') || '').toLowerCase();++ if (explain) {+ if (!/^[0-9a-f]{7,40}$/.test(explain)) {+ return error(res, 400, 'bad_sha', 'explain must be a 7 to 40 character commit sha');+ }+ const result = await explainCommit(explain, { limit });+ if (!result) {+ return error(+ res,+ 404,+ 'commit_not_found',+ `${explain} is not in the last ${limit} commits on main`,+ );+ }+ return json(res, 200, result, { 'cache-control': `public, max-age=${CACHE_TTL_S}` });+ }++ const feed = await buildFeed({ limit, audience });++ if (format === 'markdown') {+ res.statusCode = 200;+ res.setHeader('content-type', 'text/markdown; charset=utf-8');+ res.setHeader('cache-control', `public, max-age=${CACHE_TTL_S}`);+ return res.end(renderMarkdown(feed));+ }+ if (format === 'rss') {+ res.statusCode = 200;+ res.setHeader('content-type', 'application/rss+xml; charset=utf-8');+ res.setHeader('cache-control', `public, max-age=${CACHE_TTL_S}`);+ return res.end(+ renderRss(feed, {+ title: 'three.ws ship log',+ siteUrl: `${SITE}/ship`,+ description: 'Every three.ws release note, with the commits that produced it.',+ }),+ );+ }++ return json(res, 200, feed, { 'cache-control': `public, max-age=${CACHE_TTL_S}` });+});diff --git a/packages/shipfeed/LICENSE b/packages/shipfeed/LICENSEnew file mode 100644index 000000000..dff21b7f4--- /dev/null+++ b/packages/shipfeed/LICENSE@@ -0,0 +1,20 @@+Copyright 2026 nirholas++All rights reserved.++This software is proprietary and may not be used, copied, modified, distributed,+translated, or made available to any third party without the express written+permission of the copyright owner.++No rights are granted by implication, estoppel, or otherwise. Use of this+software is subject to the terms of a separate license agreement with the+copyright owner.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT.++IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY CLAIM, DAMAGES, OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE,+ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.diff --git a/packages/shipfeed/bin/shipfeed.mjs b/packages/shipfeed/bin/shipfeed.mjsnew file mode 100755index 000000000..0b4f15ac1--- /dev/null+++ b/packages/shipfeed/bin/shipfeed.mjs@@ -0,0 +1,306 @@+#!/usr/bin/env node+// shipfeed: read a repository's releases and the commits behind them.+//+// npx @three-ws/shipfeed the current repo, from GitHub+// npx @three-ws/shipfeed --local the current repo, from git log+// npx @three-ws/shipfeed --repo vercel/next.js --limit 300+// npx @three-ws/shipfeed --changelog https://three.ws/changelog.json+// npx @three-ws/shipfeed explain <sha>+// npx @three-ws/shipfeed stats+// npx @three-ws/shipfeed post --telegram --chat <id>+//+// No configuration file, no state, no network beyond GitHub (and Telegram when+// you explicitly ask it to post).++import { execFileSync } from 'node:child_process';+import { readFileSync } from 'node:fs';+import { createRequire } from 'node:module';+import process from 'node:process';++import {+ buildShipFeed,+ classify,+ commitsFromGitLog,+ fetchChangelog,+ fetchGitHubCommits,+ GIT_LOG_FORMAT,+ linkCommits,+ normalizeChangelog,+ parseCommit,+ renderCommitTelegram,+ renderMarkdown,+ renderReleaseTelegram,+ renderRss,+ renderTerminal,+} from '../src/index.js';++const require = createRequire(import.meta.url);+const { version: VERSION } = require('../package.json');++const HELP = `shipfeed ${VERSION}++ shipfeed [feed] build the unified release feed (default)+ shipfeed explain <sha> why one commit was classified and linked as it was+ shipfeed stats velocity, audience mix, and changelog coverage+ shipfeed post render what would go to a channel (add --send to deliver)++Options+ --repo <owner/name> default: parsed from the git "origin" remote+ --branch <name> default: main+ --limit <n> commits to read (default 200)+ --local read commits from the local clone with git log+ --changelog <url|path> changelog JSON to link commits against+ --site <url> site base URL used for permalinks+ --scopes <a,b,c> scopes that mean "product" when scoring signal+ --min-audience <a> holder | developer | internal (default internal)+ --json | --markdown | --rss | --no-color+ --telegram --chat <id> --token <t> --send post mode; dry run without --send+ --help | --version+`;++function parseArgs(argv) {+ const flags = {};+ const positional = [];+ for (let i = 0; i < argv.length; i++) {+ const arg = argv[i];+ if (!arg.startsWith('--')) {+ positional.push(arg);+ continue;+ }+ const [name, inline] = arg.slice(2).split('=');+ const next = argv[i + 1];+ const wantsValue = [+ 'repo',+ 'branch',+ 'limit',+ 'changelog',+ 'site',+ 'scopes',+ 'min-audience',+ 'chat',+ 'token',+ ].includes(name);+ if (inline !== undefined) flags[name] = inline;+ else if (wantsValue && next && !next.startsWith('--')) flags[name] = argv[++i];+ else flags[name] = true;+ }+ return { flags, positional };+}++function detectRepo() {+ try {+ const url = execFileSync('git', ['remote', 'get-url', 'origin'], {+ encoding: 'utf8',+ stdio: ['ignore', 'pipe', 'ignore'],+ }).trim();+ const m = /github\.com[:/]([\w.-]+\/[\w.-]+?)(?:\.git)?$/.exec(url);+ return m ? m[1] : '';+ } catch {+ return '';+ }+}++function readLocalCommits(limit, repo) {+ const out = execFileSync('git', ['log', `-n${limit}`, `--format=${GIT_LOG_FORMAT}`], {+ encoding: 'utf8',+ maxBuffer: 1 << 28,+ });+ return commitsFromGitLog(out, { repo });+}++async function loadChangelog(source) {+ if (!source || source === true) return [];+ if (/^https?:\/\//.test(source)) return fetchChangelog(source);+ return normalizeChangelog(JSON.parse(readFileSync(source, 'utf8')));+}++const SPARK = '▁▂▃▄▅▆▇█';+function sparkline(values) {+ if (values.length === 0) return '';+ const max = Math.max(...values, 1);+ return values.map((v) => SPARK[Math.min(SPARK.length - 1, Math.round((v / max) * 7))]).join('');+}++function fail(message) {+ process.stderr.write(`shipfeed: ${message}\n`);+ process.exitCode = 1;+}++async function collect(flags) {+ const repo = flags.repo || detectRepo();+ if (!repo && !flags.local) {+ throw new Error('no repository. Pass --repo owner/name, or run inside a GitHub clone.');+ }+ const limit = Math.max(1, Number(flags.limit) || 200);+ const commits = flags.local+ ? readLocalCommits(limit, repo)+ : await fetchGitHubCommits({+ repo,+ branch: flags.branch || 'main',+ limit,+ token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '',+ });+ const entries = await loadChangelog(flags.changelog);+ const feed = buildShipFeed({+ commits,+ entries,+ repo,+ siteUrl: flags.site || '',+ minAudience: flags['min-audience'] || 'internal',+ classify: {+ productScopes: String(flags.scopes || '')+ .split(',')+ .map((s) => s.trim())+ .filter(Boolean),+ },+ });+ return { repo, commits, entries, feed };+}++async function cmdFeed(flags) {+ const { feed } = await collect(flags);+ if (flags.json) return process.stdout.write(`${JSON.stringify(feed, null, 2)}\n`);+ if (flags.markdown) return process.stdout.write(renderMarkdown(feed));+ if (flags.rss) {+ return process.stdout.write(+ `${renderRss(feed, { title: `${feed.repo} ship log`, siteUrl: feed.siteUrl || '' })}\n`,+ );+ }+ return process.stdout.write(+ `${renderTerminal(feed, { color: !flags['no-color'] && process.stdout.isTTY })}\n`,+ );+}++async function cmdExplain(flags, sha) {+ if (!sha) throw new Error('explain needs a commit sha');+ const { commits, entries, feed } = await collect(flags);+ const parsed = commits.map((c) => parseCommit(c));+ const commit = parsed.find((c) => c.sha.startsWith(sha));+ if (!commit) throw new Error(`commit ${sha} is not in the last ${parsed.length} commits`);++ const cls = classify(commit, {+ productScopes: String(flags.scopes || '')+ .split(',')+ .map((s) => s.trim())+ .filter(Boolean),+ });+ const { byCommit } = linkCommits(entries, parsed);+ const link = byCommit.get(commit.sha);++ const lines = [+ `${commit.shortSha} ${commit.subject}`,+ '',+ ` type ${commit.type || '(none)'}${commit.scope ? ` (${commit.scope})` : ''}`,+ ` breaking ${commit.breaking}`,+ ` audience ${cls.audience}`,+ ` signal ${cls.signal}`,+ '',+ ' scoring',+ ...cls.reasons.map(+ (r) => ` ${r.delta >= 0 ? '+' : ''}${r.delta.toFixed(2)} ${r.rule.padEnd(22)} ${r.note}`,+ ),+ '',+ ];+ if (link) {+ const release = feed.releases.find((r) => r.key === link.entryKey);+ lines.push(+ ` linked to ${release ? release.title : link.entryKey}`,+ ` confidence ${link.score}`,+ ' because',+ ...link.reasons.map((r) => ` ${r}`),+ );+ } else {+ lines.push(' linked to nothing: no changelog entry claims this commit');+ }+ process.stdout.write(`${lines.join('\n')}\n`);+}++async function cmdStats(flags) {+ const { feed } = await collect(flags);+ const velocity = feed.stats.velocity;+ const lines = [+ `${feed.repo}`,+ '',+ ` commits ${feed.stats.commits}`,+ ` releases ${feed.stats.releases}`,+ ` linked ${feed.stats.linked} (${Math.round(feed.stats.coverage * 100)}% of visible commits)`,+ ` unannounced ${feed.stats.orphans}`,+ '',+ ` holder ${feed.stats.byAudience.holder}`,+ ` developer ${feed.stats.byAudience.developer}`,+ ` internal ${feed.stats.byAudience.internal}`,+ '',+ ` velocity ${sparkline(velocity.map((v) => v.count))}`,+ ` ${velocity.length ? `${velocity[0].date} to ${velocity[velocity.length - 1].date}` : 'no dated commits'}`,+ '',+ ' types ' ++ Object.entries(feed.stats.byType)+ .sort((a, b) => b[1] - a[1])+ .slice(0, 8)+ .map(([t, n]) => `${t}:${n}`)+ .join(' '),+ ' authors ' + feed.stats.topAuthors.map((a) => `${a.name}:${a.count}`).join(' '),+ ];+ process.stdout.write(`${lines.join('\n')}\n`);+}++async function postTelegram(chatId, token, text) {+ const res = await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {+ method: 'POST',+ headers: { 'content-type': 'application/json' },+ body: JSON.stringify({ chat_id: chatId, text, parse_mode: 'HTML' }),+ signal: AbortSignal.timeout(20_000),+ });+ const body = await res.json().catch(() => ({}));+ if (!res.ok || !body.ok) {+ throw new Error(`Telegram sendMessage failed (${res.status}): ${body.description || 'unknown'}`);+ }+}++async function cmdPost(flags) {+ const { feed, repo } = await collect(flags);+ const messages = [];+ for (const release of feed.releases.slice(0, 3)) {+ messages.push(renderReleaseTelegram(release, { repo, siteUrl: feed.siteUrl || '' }));+ }+ for (const ship of feed.ships.slice(0, 3)) {+ for (const commit of ship.commits.slice(0, 3)) {+ messages.push(renderCommitTelegram(commit, { repo }));+ }+ }++ if (!flags.send) {+ process.stdout.write(+ `${messages.join('\n\n---\n\n')}\n\n(dry run: ${messages.length} messages. Add --send to deliver.)\n`,+ );+ return;+ }++ const token = flags.token || process.env.TELEGRAM_BOT_TOKEN || '';+ const chat = flags.chat || process.env.TELEGRAM_CHAT_ID || '';+ if (!token || !chat) throw new Error('post --send needs --token and --chat (or the env vars)');+ for (const [i, message] of messages.entries()) {+ await postTelegram(chat, token, message);+ process.stdout.write(`sent ${i + 1}/${messages.length}\n`);+ if (i < messages.length - 1) await new Promise((r) => setTimeout(r, 3500));+ }+}++const { flags, positional } = parseArgs(process.argv.slice(2));++if (flags.help || positional[0] === 'help') {+ process.stdout.write(HELP);+} else if (flags.version) {+ process.stdout.write(`${VERSION}\n`);+} else {+ const command = positional[0] && !positional[0].startsWith('-') ? positional[0] : 'feed';+ const run =+ command === 'explain'+ ? () => cmdExplain(flags, positional[1])+ : command === 'stats'+ ? () => cmdStats(flags)+ : command === 'post'+ ? () => cmdPost(flags)+ : () => cmdFeed(flags);+ run().catch((err) => fail(err?.message || String(err)));+}diff --git a/packages/shipfeed/package.json b/packages/shipfeed/package.jsonnew file mode 100644index 000000000..0967053c2--- /dev/null+++ b/packages/shipfeed/package.json@@ -0,0 +1,59 @@+{+ "name": "@three-ws/shipfeed",+ "version": "0.1.0",+ "description": "Turn a repo's raw commit history into a release feed humans actually read: conventional-commit parsing, audience classification, and changelog-to-commit provenance with no extra metadata.",+ "type": "module",+ "main": "./src/index.js",+ "module": "./src/index.js",+ "types": "./src/index.d.ts",+ "bin": {+ "shipfeed": "./bin/shipfeed.mjs"+ },+ "exports": {+ ".": {+ "types": "./src/index.d.ts",+ "import": "./src/index.js",+ "default": "./src/index.js"+ },+ "./package.json": "./package.json"+ },+ "files": [+ "src",+ "bin",+ "README.md",+ "LICENSE"+ ],+ "scripts": {+ "test": "vitest run --root ../.. packages/shipfeed"+ },+ "keywords": [+ "changelog",+ "conventional-commits",+ "release-notes",+ "commit-parser",+ "provenance",+ "git",+ "github",+ "telegram",+ "rss",+ "three.ws"+ ],+ "author": "nirholas <[email protected]>",+ "license": "Apache-2.0",+ "homepage": "https://three.ws/ship",+ "repository": {+ "type": "git",+ "url": "git+https://github.com/nirholas/three.ws.git",+ "directory": "packages/shipfeed"+ },+ "bugs": {+ "url": "https://github.com/nirholas/three.ws/issues"+ },+ "engines": {+ "node": ">=18"+ },+ "dependencies": {},+ "publishConfig": {+ "access": "public"+ }+}diff --git a/packages/shipfeed/src/classify.js b/packages/shipfeed/src/classify.jsnew file mode 100644index 000000000..e50c25a5a--- /dev/null+++ b/packages/shipfeed/src/classify.js@@ -0,0 +1,182 @@+// Audience classification: which commits are worth a stranger's attention.+//+// A raw commit feed is unreadable not because it is long but because it is+// undifferentiated: a lockfile bump and a new payment rail arrive with the same+// weight. This module scores each commit from 0 to 1 and puts it in one of+// three audiences, and it always says WHY, so a feed that hides something can+// be argued with instead of trusted blindly.+//+// holder someone who uses or owns a piece of the product+// developer someone who builds against it+// internal someone who works on it+//+// Pure and synchronous. Optional `files` sharpen the result but are never+// required, so classification costs no extra API calls.++import { parseCommit } from './parse.js';++/** Base signal per conventional type. Unknown types land mid-scale. */+const TYPE_SIGNAL = {+ feat: 0.72,+ security: 0.72,+ fix: 0.6,+ perf: 0.58,+ revert: 0.5,+ release: 0.55,+ refactor: 0.3,+ docs: 0.34,+ build: 0.2,+ test: 0.16,+ tests: 0.16,+ ci: 0.12,+ chore: 0.14,+ style: 0.1,+ deps: 0.12,+};++const TYPE_AUDIENCE = {+ feat: 'holder',+ fix: 'holder',+ perf: 'holder',+ security: 'holder',+ release: 'holder',+ revert: 'developer',+ docs: 'developer',+ refactor: 'internal',+ build: 'internal',+ test: 'internal',+ tests: 'internal',+ ci: 'internal',+ chore: 'internal',+ style: 'internal',+ deps: 'internal',+};++/** Paths whose presence alone never justifies a post. */+const NOISE_FILE_RE =+ /(^|\/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb|\.gitignore|\.editorconfig)$/;++/** Scopes that mean "the machinery", not "the product". */+const NOISE_SCOPES = new Set(['deps', 'dependencies', 'lockfile', 'ci', 'infra-ci', 'typo']);++const clamp01 = (n) => Math.max(0, Math.min(1, n));++/**+ * @typedef {Object} Classification+ * @property {'holder'|'developer'|'internal'} audience+ * @property {number} signal 0..1, rounded to 2 decimals+ * @property {boolean} noise true when the commit carries no reader-facing meaning+ * @property {{rule: string, delta: number, note: string}[]} reasons+ */++/**+ * Classify one commit.+ *+ * @param {object} commit raw GitHub commit or a parseCommit() record+ * @param {{productScopes?: string[], noiseScopes?: string[]}} [options]+ * productScopes are the parts of the repo a reader cares about by name;+ * naming yours is the single highest-leverage tuning knob here.+ * @returns {Classification}+ */+export function classify(commit, options = {}) {+ const c = commit && commit.subject !== undefined ? commit : parseCommit(commit);+ const productScopes = new Set((options.productScopes || []).map((s) => s.toLowerCase()));+ const noiseScopes = new Set([+ ...NOISE_SCOPES,+ ...(options.noiseScopes || []).map((s) => s.toLowerCase()),+ ]);++ const reasons = [];+ const add = (rule, delta, note) => {+ reasons.push({ rule, delta: Number(delta.toFixed(2)), note });+ };++ let signal = c.type && TYPE_SIGNAL[c.type] !== undefined ? TYPE_SIGNAL[c.type] : 0.4;+ add(+ c.type ? `type:${c.type}` : 'type:none',+ signal,+ c.type ? `conventional type "${c.type}"` : 'no conventional type; scored mid-scale',+ );++ let audience = (c.type && TYPE_AUDIENCE[c.type]) || 'developer';+ let noise = false;++ if (c.merge) {+ signal = 0;+ audience = 'internal';+ noise = true;+ add('merge', -1, 'merge commit: the merged commits carry the content');+ }++ const scope = (c.scope || '').toLowerCase();+ if (scope && noiseScopes.has(scope)) {+ signal -= 0.25;+ audience = 'internal';+ noise = true;+ add(`scope:${scope}`, -0.25, 'scope is machinery, not product');+ } else if (scope && productScopes.has(scope)) {+ signal += 0.12;+ add(`scope:${scope}`, 0.12, 'named product scope');+ }++ if (c.breaking) {+ signal += 0.25;+ if (audience === 'internal') audience = 'developer';+ add('breaking', 0.25, 'breaking change: every consumer needs to know');+ }++ if (c.revert) {+ add('revert', 0, 'revert: kept visible, a rollback is news');+ if (audience === 'internal') audience = 'developer';+ }++ if (Array.isArray(c.files) && c.files.length > 0) {+ const meaningful = c.files.filter((f) => !NOISE_FILE_RE.test(f));+ if (meaningful.length === 0) {+ signal = 0;+ audience = 'internal';+ noise = true;+ add('files:lockfile-only', -1, 'touches only lockfiles and dotfiles');+ } else {+ const publicSurface = meaningful.some((f) => /^(api|src|pages|public|packages)\//.test(f));+ if (publicSurface) {+ signal += 0.08;+ add('files:public-surface', 0.08, 'touches a user-reachable surface');+ }+ if (meaningful.every((f) => /^(docs|README|\.github)/.test(f))) {+ audience = audience === 'holder' ? 'developer' : audience;+ add('files:docs-only', 0, 'documentation only');+ }+ }+ }++ const desc = c.description || c.subject || '';+ if (desc.length < 12) {+ signal -= 0.15;+ add('description:terse', -0.15, 'description too short to tell a reader anything');+ } else if (desc.length > 40) {+ signal += 0.06;+ add('description:explanatory', 0.06, 'description explains the change');+ }++ signal = clamp01(signal);+ if (signal >= 0.55 && audience === 'developer' && c.type === 'feat') audience = 'holder';+ if (signal <= 0.15 && !noise) {+ audience = 'internal';+ add('signal:floor', 0, 'scored below the reader-interest floor');+ }++ return { audience, signal: Number(signal.toFixed(2)), noise, reasons };+}++/** Rank order for the three audiences, widest first. */+export const AUDIENCE_RANK = { holder: 3, developer: 2, internal: 1 };++/**+ * Filter a list of commits down to one audience and above.+ * `minAudience: 'developer'` keeps holder + developer, drops internal.+ */+export function filterByAudience(commits, minAudience = 'internal', options = {}) {+ const floor = AUDIENCE_RANK[minAudience] || 1;+ return commits.filter((c) => AUDIENCE_RANK[classify(c, options).audience] >= floor);+}diff --git a/packages/shipfeed/src/feed.js b/packages/shipfeed/src/feed.jsnew file mode 100644index 000000000..31421ed08--- /dev/null+++ b/packages/shipfeed/src/feed.js@@ -0,0 +1,161 @@+// The unified stream: curated release notes with the code that produced them+// underneath, and everything the notes never mentioned grouped beside them.+//+// This is the whole point of the package. A changelog alone hides the work; a+// commit log alone buries it. One structure holds both, and every commit in it+// knows whether it is spoken for.++import { parseCommit, headline, summaryLine } from './parse.js';+import { classify, AUDIENCE_RANK } from './classify.js';+import { linkCommits, entryKey, entrySlug } from './link.js';+import { groupIntoShips } from './group.js';++export const FEED_VERSION = 1;++function serializeCommit(commit, cls, provenance) {+ return {+ sha: commit.sha,+ shortSha: commit.shortSha,+ url: commit.url,+ date: commit.committedAt,+ author: commit.author.login || commit.author.name,+ type: commit.type,+ scope: commit.scope,+ breaking: commit.breaking,+ headline: headline(commit),+ summary: summaryLine(commit),+ audience: cls.audience,+ signal: cls.signal,+ issues: commit.issues,+ ...(provenance ? { confidence: provenance.score, why: provenance.reasons } : {}),+ };+}++/**+ * Build the unified ship feed.+ *+ * @param {{+ * commits: object[],+ * entries?: object[],+ * repo?: string,+ * siteUrl?: string,+ * now?: number,+ * minAudience?: 'holder'|'developer'|'internal',+ * link?: object,+ * classify?: object,+ * velocityDays?: number+ * }} input+ */+export function buildShipFeed(input) {+ const {+ commits: rawCommits = [],+ entries = [],+ repo = '',+ siteUrl = '',+ now = Date.now(),+ minAudience = 'internal',+ velocityDays = 30,+ } = input;++ const commits = rawCommits.map((c) => (c.subject !== undefined ? c : parseCommit(c)));+ const classifications = new Map(commits.map((c) => [c.sha, classify(c, input.classify)]));+ const floor = AUDIENCE_RANK[minAudience] || 1;+ const visible = commits.filter(+ (c) => AUDIENCE_RANK[classifications.get(c.sha).audience] >= floor,+ );++ const { byEntry, byCommit, orphans } = linkCommits(entries, visible, input.link);++ // A changelog can hold years of entries while a feed window holds days. Keep+ // only the entries the commits could possibly speak to, plus one window of+ // slack on each side, so an entry that legitimately found no code still+ // shows up (which is itself worth seeing) and 2,000 historical ones do not.+ const windowDays = input.link?.windowDays ?? 4;+ const times = commits+ .map((c) => Date.parse(c.committedAt || c.authoredAt || ''))+ .filter((t) => Number.isFinite(t));+ const oldest = times.length ? Math.min(...times) - windowDays * 86_400_000 : -Infinity;+ const newest = times.length ? Math.max(...times) + windowDays * 86_400_000 : Infinity;+ const inWindow = entries.filter((entry) => {+ const t = Date.parse(`${entry.date}T12:00:00Z`);+ return !Number.isFinite(t) || (t >= oldest && t <= newest);+ });++ const releases = inWindow+ .map((entry) => {+ const bucket = byEntry.get(entryKey(entry));+ const linked = bucket ? bucket.commits : [];+ const authors = [...new Set(linked.map((c) => c.author.login || c.author.name))];+ return {+ key: entryKey(entry),+ slug: entrySlug(entry),+ date: entry.date,+ title: entry.title,+ summary: entry.summary || '',+ tags: Array.isArray(entry.tags) ? entry.tags : [],+ link: entry.link || null,+ url: siteUrl ? `${siteUrl}/changelog/${entrySlug(entry)}` : null,+ commits: linked.map((c) =>+ serializeCommit(c, classifications.get(c.sha), byCommit.get(c.sha)),+ ),+ stats: {+ commits: linked.length,+ authors: authors.length,+ range: linked.length+ ? `${linked[0].shortSha}..${linked[linked.length - 1].shortSha}`+ : null,+ },+ };+ })+ .sort((a, b) => String(b.date).localeCompare(String(a.date)));++ const ships = groupIntoShips(orphans, { options: input.classify }).map((ship) => ({+ ...ship,+ commits: ship.commits.map((c) => serializeCommit(c, classifications.get(c.sha))),+ }));++ const byType = {};+ const byAuthor = {};+ const byAudience = { holder: 0, developer: 0, internal: 0 };+ const velocity = new Map();+ const since = now - velocityDays * 86_400_000;+ for (const c of commits) {+ const cls = classifications.get(c.sha);+ byType[c.type || 'other'] = (byType[c.type || 'other'] || 0) + 1;+ const who = c.author.login || c.author.name;+ byAuthor[who] = (byAuthor[who] || 0) + 1;+ byAudience[cls.audience]++;+ const t = Date.parse(c.committedAt || '');+ if (Number.isFinite(t) && t >= since) {+ const day = new Date(t).toISOString().slice(0, 10);+ velocity.set(day, (velocity.get(day) || 0) + 1);+ }+ }++ return {+ version: FEED_VERSION,+ generatedAt: new Date(now).toISOString(),+ repo,+ siteUrl: siteUrl || null,+ releases,+ ships,+ stats: {+ commits: commits.length,+ hidden: commits.length - visible.length,+ releases: releases.length,+ entriesConsidered: entries.length,+ linked: byCommit.size,+ orphans: orphans.length,+ coverage: visible.length ? Number((byCommit.size / visible.length).toFixed(3)) : 0,+ byType,+ byAudience,+ topAuthors: Object.entries(byAuthor)+ .sort((a, b) => b[1] - a[1])+ .slice(0, 10)+ .map(([name, count]) => ({ name, count })),+ velocity: [...velocity.entries()]+ .sort((a, b) => a[0].localeCompare(b[0]))+ .map(([date, count]) => ({ date, count })),+ },+ };+}diff --git a/packages/shipfeed/src/group.js b/packages/shipfeed/src/group.jsnew file mode 100644index 000000000..599e763b4--- /dev/null+++ b/packages/shipfeed/src/group.js@@ -0,0 +1,59 @@+// Burst grouping: commits land in clusters, and a cluster is what a human+// would call "a ship". Anything that is not claimed by a changelog entry still+// deserves to be readable, so it gets grouped by time gap and headlined by its+// own strongest commit rather than dumped as a flat list.++import { parseCommit, headline, summaryLine } from './parse.js';+import { classify } from './classify.js';++const MINUTE_MS = 60_000;++/**+ * Cluster commits into ships. A new ship starts whenever more than `gapMinutes`+ * of quiet separates two consecutive commits.+ *+ * @param {object[]} commits raw or parsed commits, any order+ * @param {{gapMinutes?: number, options?: object}} [opts]+ * @returns {{id: string, start: string, end: string, title: string, summary: string,+ * authors: string[], commits: object[], signal: number}[]} newest ship first+ */+export function groupIntoShips(commits, opts = {}) {+ const gap = (opts.gapMinutes ?? 90) * MINUTE_MS;+ const parsed = commits+ .map((c) => (c.subject !== undefined ? c : parseCommit(c)))+ .filter((c) => c.sha)+ .sort((a, b) => String(a.committedAt).localeCompare(String(b.committedAt)));++ const ships = [];+ let current = null;+ for (const commit of parsed) {+ const t = Date.parse(commit.committedAt || commit.authoredAt || '');+ if (!current || !Number.isFinite(t) || t - current.lastAt > gap) {+ current = { commits: [commit], lastAt: Number.isFinite(t) ? t : 0 };+ ships.push(current);+ } else {+ current.commits.push(commit);+ current.lastAt = t;+ }+ }++ return ships+ .map((ship) => {+ const scored = ship.commits.map((c) => ({ commit: c, cls: classify(c, opts.options) }));+ const lead = scored.reduce((best, x) => (x.cls.signal > best.cls.signal ? x : best), scored[0]);+ const authors = [...new Set(ship.commits.map((c) => c.author.login || c.author.name))];+ const start = ship.commits[0].committedAt;+ const end = ship.commits[ship.commits.length - 1].committedAt;+ return {+ id: `${ship.commits[0].shortSha}..${ship.commits[ship.commits.length - 1].shortSha}`,+ start,+ end,+ title: headline(lead.commit),+ summary: summaryLine(lead.commit),+ authors,+ signal: lead.cls.signal,+ commits: ship.commits,+ };+ })+ .reverse();+}diff --git a/packages/shipfeed/src/index.d.ts b/packages/shipfeed/src/index.d.tsnew file mode 100644index 000000000..d95ebe816--- /dev/null+++ b/packages/shipfeed/src/index.d.ts@@ -0,0 +1,228 @@+// Type definitions for @three-ws/shipfeed.++export type Audience = 'holder' | 'developer' | 'internal';++export interface ParsedMessage {+ subject: string;+ type: string | null;+ scope: string | null;+ breaking: boolean;+ description: string;+ body: string;+ trailers: Record<string, string>;+ coAuthors: string[];+ issues: string[];+ merge: boolean;+ revert: boolean;+ revertedSubject: string | null;+ conventional: boolean;+}++export interface ParsedCommit extends ParsedMessage {+ sha: string;+ shortSha: string;+ url: string;+ author: { login: string | null; name: string };+ authoredAt: string;+ committedAt: string;+ parents: number;+ files: string[] | null;+}++export interface ClassifyOptions {+ productScopes?: string[];+ noiseScopes?: string[];+}++export interface Classification {+ audience: Audience;+ signal: number;+ noise: boolean;+ reasons: { rule: string; delta: number; note: string }[];+}++export interface ChangelogEntry {+ date: string;+ title: string;+ summary?: string;+ tags?: string[];+ link?: string | null;+}++export interface LinkOptions {+ windowDays?: number;+ leadDays?: number;+ threshold?: number;+ maxPerEntry?: number;+}++export interface LinkResult {+ byEntry: Map<string, { entry: ChangelogEntry; commits: ParsedCommit[] }>;+ byCommit: Map<string, { entryKey: string; score: number; reasons: string[] }>;+ orphans: ParsedCommit[];+}++export interface FeedCommit {+ sha: string;+ shortSha: string;+ url: string;+ date: string;+ author: string;+ type: string | null;+ scope: string | null;+ breaking: boolean;+ headline: string;+ summary: string;+ audience: Audience;+ signal: number;+ issues: string[];+ confidence?: number;+ why?: string[];+}++export interface FeedRelease {+ key: string;+ slug: string;+ date: string;+ title: string;+ summary: string;+ tags: string[];+ link: string | null;+ url: string | null;+ commits: FeedCommit[];+ stats: { commits: number; authors: number; range: string | null };+}++export interface FeedShip {+ id: string;+ start: string;+ end: string;+ title: string;+ summary: string;+ authors: string[];+ signal: number;+ commits: FeedCommit[];+}++export interface ShipFeed {+ version: number;+ generatedAt: string;+ repo: string;+ siteUrl: string | null;+ releases: FeedRelease[];+ ships: FeedShip[];+ stats: {+ commits: number;+ hidden: number;+ releases: number;+ linked: number;+ orphans: number;+ coverage: number;+ byType: Record<string, number>;+ byAudience: Record<Audience, number>;+ topAuthors: { name: string; count: number }[];+ velocity: { date: string; count: number }[];+ };+}++export declare const KNOWN_TYPES: Set<string>;+export declare const TYPE_LABELS: Record<string, string>;+export declare const AUDIENCE_RANK: Record<Audience, number>;+export declare const FEED_VERSION: number;+export declare const GIT_LOG_FORMAT: string;++export declare function parseCommitMessage(message: string): ParsedMessage;+export declare function parseCommit(input: unknown): ParsedCommit;+export declare function parseTrailers(body: string): {+ trailers: Record<string, string>;+ coAuthors: string[];+};+export declare function splitMessage(message: string): { subject: string; body: string };+export declare function headline(+ commit: ParsedCommit | ParsedMessage | string,+ options?: { separator?: string },+): string;+export declare function summaryLine(commit: ParsedCommit | ParsedMessage | string): string;++export declare function classify(commit: unknown, options?: ClassifyOptions): Classification;+export declare function filterByAudience<T>(+ commits: T[],+ minAudience?: Audience,+ options?: ClassifyOptions,+): T[];++export declare function linkCommits(+ entries: ChangelogEntry[],+ commits: unknown[],+ options?: LinkOptions,+): LinkResult;+export declare function tokenize(text: string): string[];+export declare function buildIdf(documents: string[][]): Map<string, number>;+export declare function entryKey(entry: ChangelogEntry): string;+export declare function entrySlug(entry: ChangelogEntry): string;++export declare function groupIntoShips(+ commits: unknown[],+ options?: { gapMinutes?: number; options?: ClassifyOptions },+): Omit<FeedShip, 'commits'> & { commits: ParsedCommit[] }[];++export declare function buildShipFeed(input: {+ commits: unknown[];+ entries?: ChangelogEntry[];+ repo?: string;+ siteUrl?: string;+ now?: number;+ minAudience?: Audience;+ link?: LinkOptions;+ classify?: ClassifyOptions;+ velocityDays?: number;+}): ShipFeed;++export declare function renderCommitTelegram(+ commit: FeedCommit,+ options?: { repo?: string; linkText?: string },+): string;+export declare function renderReleaseTelegram(+ release: FeedRelease,+ options?: { repo?: string; siteUrl?: string },+): string;+export declare function renderMarkdown(feed: ShipFeed, options?: { includeShips?: boolean }): string;+export declare function renderRss(+ feed: ShipFeed,+ options?: { title?: string; siteUrl?: string; description?: string },+): string;+export declare function renderTerminal(+ feed: ShipFeed,+ options?: { color?: boolean; maxCommits?: number },+): string;++export declare function fetchGitHubCommits(options: {+ repo: string;+ branch?: string;+ limit?: number;+ token?: string;+ fetchImpl?: typeof fetch;+ signal?: AbortSignal;+ userAgent?: string;+}): Promise<unknown[]>;+export declare function fetchChangelog(+ url: string,+ options?: { fetchImpl?: typeof fetch; signal?: AbortSignal },+): Promise<ChangelogEntry[]>;+export declare function commitsFromGitLog(text: string, options?: { repo?: string }): unknown[];+export declare function normalizeChangelog(input: unknown): ChangelogEntry[];++export declare function shipfeed(options: {+ repo: string;+ branch?: string;+ limit?: number;+ token?: string;+ changelogUrl?: string;+ changelog?: unknown;+ siteUrl?: string;+ minAudience?: Audience;+ productScopes?: string[];+ fetchImpl?: typeof fetch;+ signal?: AbortSignal;+ now?: number;+}): Promise<ShipFeed>;diff --git a/packages/shipfeed/src/index.js b/packages/shipfeed/src/index.jsnew file mode 100644index 000000000..7f7e54e76--- /dev/null+++ b/packages/shipfeed/src/index.js@@ -0,0 +1,84 @@+// @three-ws/shipfeed+//+// A repository's commit log and its changelog describe the same work twice and+// never point at each other. This package joins them: it parses commits, scores+// who each one is for, and works out which commits produced which release note,+// so one feed can carry both the human sentence and the code behind it.+//+// Zero dependencies, pure functions, and one network module you can replace.+// See README.md for the full reference.++export {+ parseCommit,+ parseCommitMessage,+ parseTrailers,+ splitMessage,+ headline,+ summaryLine,+ KNOWN_TYPES,+ TYPE_LABELS,+} from './parse.js';++export { classify, filterByAudience, AUDIENCE_RANK } from './classify.js';++export { linkCommits, tokenize, buildIdf, entryKey, entrySlug } from './link.js';++export { groupIntoShips } from './group.js';++export { buildShipFeed, FEED_VERSION } from './feed.js';++export {+ renderCommitTelegram,+ renderReleaseTelegram,+ renderMarkdown,+ renderRss,+ renderTerminal,+} from './render.js';++export {+ fetchGitHubCommits,+ fetchChangelog,+ commitsFromGitLog,+ normalizeChangelog,+ GIT_LOG_FORMAT,+} from './sources.js';++import { fetchGitHubCommits, fetchChangelog, normalizeChangelog } from './sources.js';+import { buildShipFeed } from './feed.js';++/**+ * The one-call path: read a repo (and optionally its changelog), return the+ * unified feed.+ *+ * ```js+ * const feed = await shipfeed({ repo: 'nirholas/three.ws', changelogUrl: 'https://three.ws/changelog.json' });+ * console.log(feed.releases[0].commits.map((c) => c.shortSha));+ * ```+ *+ * @param {{+ * repo: string, branch?: string, limit?: number, token?: string,+ * changelogUrl?: string, changelog?: object, siteUrl?: string,+ * minAudience?: 'holder'|'developer'|'internal',+ * productScopes?: string[], fetchImpl?: typeof fetch, signal?: AbortSignal, now?: number+ * }} options+ */+export async function shipfeed(options) {+ const { repo, branch, limit, token, changelogUrl, changelog, fetchImpl, signal } = options;+ const [commits, entries] = await Promise.all([+ fetchGitHubCommits({ repo, branch, limit, token, fetchImpl, signal }),+ changelog+ ? Promise.resolve(normalizeChangelog(changelog))+ : changelogUrl+ ? fetchChangelog(changelogUrl, { fetchImpl, signal })+ : Promise.resolve([]),+ ]);+ return buildShipFeed({+ commits,+ entries,+ repo,+ siteUrl: options.siteUrl,+ now: options.now,+ minAudience: options.minAudience,+ classify: { productScopes: options.productScopes },+ });+}diff --git a/packages/shipfeed/src/link.js b/packages/shipfeed/src/link.jsnew file mode 100644index 000000000..9d7742547--- /dev/null+++ b/packages/shipfeed/src/link.js@@ -0,0 +1,228 @@+// Provenance without metadata: work out which commits produced which changelog+// entry, on a repo where nobody remembered to write that down.+//+// The honest way to link a release note to its code is a trailer in the commit+// (`Changelog: <slug>`), and this module always prefers one when it is there.+// Almost no repository has them, so the fallback has to earn its answer:+//+// 1. a time window, because an entry is written the day its work lands;+// 2. IDF-weighted term overlap, so shared rare words ("meshopt", "settle",+// "fingerspelling") count and shared common ones ("agent", "the", "fix")+// do not;+// 3. agreement between the entry's tags and the commit's conventional type;+// 4. the entry's `link` path, which usually names the surface that changed.+//+// Every link carries its score and the reasons behind it, so a wrong link is+// visible and arguable rather than mysterious. Pure and synchronous.++import { parseCommit } from './parse.js';++const STOPWORDS = new Set(+ ('a an and are as at be been but by for from has have how in into is it its of on or that the ' ++ 'their then there these they this to was were what when which who will with you your now not ' ++ 'can could should would we our us also more most other some such only own same than too very ' ++ 'just add adds added update updates updated make makes made use uses used new')+ .split(' ')+ .filter(Boolean),+);++const DAY_MS = 86_400_000;++/** Lowercase word tokens, minus stopwords and single characters. */+export function tokenize(text) {+ return String(text || '')+ .toLowerCase()+ .split(/[^a-z0-9]+/)+ .filter((t) => t.length > 2 && !STOPWORDS.has(t) && !/^\d+$/.test(t));+}++/** Slug used by three.ws changelog permalinks, and by `Changelog:` trailers. */+export function entrySlug(entry) {+ const slug = String(entry?.title || '')+ .toLowerCase()+ .replace(/[^a-z0-9]+/g, '-')+ .replace(/^-+|-+$/g, '')+ .slice(0, 80);+ return `${entry?.date || ''}-${slug}`;+}++export const entryKey = (entry) => `${entry?.date || ''}:${entry?.title || ''}`;++/**+ * Inverse document frequency over a token corpus. Rare terms carry the signal;+ * without this step every commit "matches" every entry through words like+ * "agent" that appear in half the repository.+ */+export function buildIdf(documents) {+ const df = new Map();+ for (const tokens of documents) {+ for (const t of new Set(tokens)) df.set(t, (df.get(t) || 0) + 1);+ }+ const n = Math.max(1, documents.length);+ const idf = new Map();+ for (const [term, count] of df) idf.set(term, Math.log((n + 1) / (count + 0.5)));+ return idf;+}++// Similarity between an entry and a commit, in IDF-weighted term mass.+//+// Two normalizations, because either one alone is wrong. Against the entry's+// mass, a commit that explains only part of a long release note scores low,+// which is right for ranking but punishes the normal case: a release note is a+// paragraph and a commit is a sentence. Against the smaller of the two (the+// overlap coefficient), a short commit whose few meaningful words all appear in+// the entry scores high, which is right for containment but too eager on its+// own. The blend keeps both signals and neither dominates.+function weightedOverlap(entryTokens, commitTokens, idf) {+ const weight = (term) => idf.get(term) ?? Math.log(2);+ const commitSet = new Set(commitTokens);+ const entrySet = new Set(entryTokens);+ let shared = 0;+ let entryMass = 0;+ let commitMass = 0;+ const matched = [];+ for (const term of entrySet) {+ entryMass += weight(term);+ if (commitSet.has(term)) {+ shared += weight(term);+ matched.push(term);+ }+ }+ for (const term of commitSet) commitMass += weight(term);+ if (entryMass === 0 || commitMass === 0) return { score: 0, matched };++ const coverage = shared / entryMass;+ const containment = shared / Math.min(entryMass, commitMass);+ matched.sort((a, b) => weight(b) - weight(a));+ return { score: 0.6 * coverage + 0.4 * containment, matched: matched.slice(0, 6) };+}++const TAG_TYPES = {+ feature: ['feat'],+ improvement: ['perf', 'refactor', 'feat'],+ fix: ['fix', 'revert'],+ security: ['security', 'fix'],+ docs: ['docs'],+ sdk: ['feat', 'build'],+ infra: ['build', 'ci', 'chore', 'perf'],+};++function commitText(commit) {+ const firstParagraph = String(commit.body || '').split('\n\n')[0] || '';+ return `${commit.subject} ${commit.scope || ''} ${firstParagraph}`;+}++function entryText(entry) {+ const linkWords = String(entry.link || '').replace(/[/-]/g, ' ');+ return `${entry.title} ${entry.summary || ''} ${linkWords}`;+}++/**+ * Link changelog entries to the commits that produced them.+ *+ * @param {{date: string, title: string, summary?: string, tags?: string[], link?: string}[]} entries+ * @param {object[]} commits raw GitHub commits or parseCommit() records+ * @param {{windowDays?: number, threshold?: number, maxPerEntry?: number, leadDays?: number}} [options]+ * @returns {{+ * byEntry: Map<string, {entry: object, commits: object[]}>,+ * byCommit: Map<string, {entryKey: string, score: number, reasons: string[]}>,+ * orphans: object[]+ * }}+ */+export function linkCommits(entries, commits, options = {}) {+ const windowDays = options.windowDays ?? 4;+ const leadDays = options.leadDays ?? 1;+ const threshold = options.threshold ?? 0.26;+ const maxPerEntry = options.maxPerEntry ?? 40;++ const parsed = commits.map((c) => (c.subject !== undefined ? c : parseCommit(c)));+ const commitTokens = new Map(parsed.map((c) => [c.sha, tokenize(commitText(c))]));+ const entryTokens = new Map(entries.map((e) => [entryKey(e), tokenize(entryText(e))]));+ const idf = buildIdf([...commitTokens.values(), ...entryTokens.values()]);++ const slugIndex = new Map();+ for (const e of entries) {+ slugIndex.set(entrySlug(e), e);+ slugIndex.set(String(e.title || '').toLowerCase(), e);+ }++ const byEntry = new Map(entries.map((e) => [entryKey(e), { entry: e, commits: [] }]));+ const byCommit = new Map();++ for (const commit of parsed) {+ const when = Date.parse(commit.committedAt || commit.authoredAt || '');+ let best = null;++ const declared = commit.trailers?.changelog;+ if (declared) {+ const target =+ slugIndex.get(String(declared).toLowerCase()) ||+ slugIndex.get(+ String(declared)+ .toLowerCase()+ .replace(/^\/?changelog\//, ''),+ );+ if (target) {+ best = {+ entryKey: entryKey(target),+ score: 1,+ reasons: [`declared by trailer "Changelog: ${declared}"`],+ };+ }+ }++ if (!best && Number.isFinite(when)) {+ for (const entry of entries) {+ const entryDay = Date.parse(`${entry.date}T23:59:59Z`);+ if (!Number.isFinite(entryDay)) continue;+ const ageDays = (entryDay - when) / DAY_MS;+ if (ageDays < -leadDays || ageDays > windowDays) continue;++ const key = entryKey(entry);+ const { score: overlap, matched } = weightedOverlap(+ entryTokens.get(key) || [],+ commitTokens.get(commit.sha) || [],+ idf,+ );+ const reasons = [];+ let score = overlap;+ if (overlap > 0) reasons.push(`shared terms: ${matched.join(', ')}`);++ const tags = Array.isArray(entry.tags) ? entry.tags : [];+ if (commit.type && tags.some((t) => (TAG_TYPES[t] || []).includes(commit.type))) {+ score += 0.08;+ reasons.push(`type "${commit.type}" matches tag`);+ }++ // Same-day work is far likelier to be the work being described.+ const proximity = Math.max(0, 1 - Math.abs(ageDays) / (windowDays + leadDays));+ score += proximity * 0.1;+ reasons.push(`${ageDays < 0 ? 'after' : 'before'} the entry by ${Math.abs(ageDays).toFixed(1)}d`);++ if (!best || score > best.score) best = { entryKey: key, score, reasons };+ }+ }++ if (best && best.score >= threshold) {+ best.score = Number(best.score.toFixed(3));+ byCommit.set(commit.sha, best);+ byEntry.get(best.entryKey)?.commits.push(commit);+ }+ }++ for (const bucket of byEntry.values()) {+ bucket.commits.sort(+ (a, b) => (byCommit.get(b.sha)?.score || 0) - (byCommit.get(a.sha)?.score || 0),+ );+ if (bucket.commits.length > maxPerEntry) {+ for (const dropped of bucket.commits.slice(maxPerEntry)) byCommit.delete(dropped.sha);+ bucket.commits = bucket.commits.slice(0, maxPerEntry);+ }+ bucket.commits.sort((a, b) =>+ String(a.committedAt).localeCompare(String(b.committedAt)),+ );+ }++ const orphans = parsed.filter((c) => !byCommit.has(c.sha));+ return { byEntry, byCommit, orphans };+}diff --git a/packages/shipfeed/src/parse.js b/packages/shipfeed/src/parse.jsnew file mode 100644index 000000000..80b497675--- /dev/null+++ b/packages/shipfeed/src/parse.js@@ -0,0 +1,228 @@+// Conventional Commits 1.0.0 parsing, plus the three things the spec leaves+// out and every real repository has anyway: revert commits, GitHub's merge+// subjects, and trailers that carry meaning (Co-authored-by, Changelog:).+//+// Everything here is pure and synchronous. A commit message in, a structured+// record out. No network, no git, no filesystem.++/** Conventional-commit header: type(scope)!: description */+const HEADER_RE = /^([a-zA-Z][a-zA-Z0-9_-]*)(?:\(([^)]*)\))?(!?):\s+(.+)$/;+/** GitHub's generated merge subject, and the git default. */+const MERGE_RE = /^Merge (pull request #\d+|branch |remote-tracking branch |commit )/i;+/** git revert's default subject. */+const REVERT_RE = /^Revert\s+"(.*)"\s*$/;+/** A trailer line: `Key: value`, per git-interpret-trailers. */+const TRAILER_RE = /^([A-Za-z][A-Za-z0-9-]*)\s*:\s*(.+)$/;+/** Issue references anywhere in the message. */+const ISSUE_RE = /(?:^|[^\w/])#(\d{1,7})\b/g;++/** Types the spec blesses plus the ones the wider ecosystem settled on. */+export const KNOWN_TYPES = new Set([+ 'feat',+ 'fix',+ 'perf',+ 'refactor',+ 'docs',+ 'test',+ 'tests',+ 'build',+ 'ci',+ 'chore',+ 'style',+ 'revert',+ 'security',+ 'deps',+ 'release',+]);++/** Friendly labels for the conventional types. Anything else passes through. */+export const TYPE_LABELS = {+ feat: 'Feature',+ fix: 'Fix',+ perf: 'Performance',+ refactor: 'Refactor',+ docs: 'Docs',+ test: 'Tests',+ tests: 'Tests',+ build: 'Build',+ ci: 'CI',+ chore: 'Chore',+ style: 'Style',+ revert: 'Revert',+ security: 'Security',+ deps: 'Dependencies',+ release: 'Release',+};++const trim = (s) => String(s == null ? '' : s).trim();++/**+ * Split a commit message into its subject and body, tolerating CRLF and the+ * missing blank line that hand-written messages often have.+ */+export function splitMessage(message) {+ const text = String(message == null ? '' : message).replace(/\r\n?/g, '\n');+ const nl = text.indexOf('\n');+ if (nl === -1) return { subject: trim(text), body: '' };+ return { subject: trim(text.slice(0, nl)), body: text.slice(nl + 1).replace(/^\n+/, '').trimEnd() };+}++/**+ * Read git trailers from the end of a body. Only the final contiguous block of+ * `Key: value` lines counts, which is what git itself does, so a colon inside+ * prose never becomes a trailer.+ */+export function parseTrailers(body) {+ const lines = String(body || '').split('\n');+ const trailers = {};+ const coAuthors = [];+ let i = lines.length - 1;+ while (i >= 0 && trim(lines[i]) === '') i--;+ const block = [];+ for (; i >= 0; i--) {+ const line = lines[i];+ if (trim(line) === '') break;+ const m = TRAILER_RE.exec(trim(line));+ if (!m) return finishTrailers(block, trailers, coAuthors);+ block.unshift(m);+ }+ return finishTrailers(block, trailers, coAuthors);+}++function finishTrailers(block, trailers, coAuthors) {+ for (const [, rawKey, rawValue] of block) {+ const key = rawKey.toLowerCase();+ const value = trim(rawValue);+ if (key === 'co-authored-by') coAuthors.push(value);+ else trailers[key] = value;+ }+ return { trailers, coAuthors };+}++/**+ * Parse one commit message.+ *+ * @param {string} message full commit message (subject + body)+ * @returns {{+ * subject: string, type: string|null, scope: string|null, breaking: boolean,+ * description: string, body: string, trailers: Record<string,string>,+ * coAuthors: string[], issues: string[], merge: boolean, revert: boolean,+ * revertedSubject: string|null, conventional: boolean+ * }}+ */+export function parseCommitMessage(message) {+ const { subject, body } = splitMessage(message);+ const { trailers, coAuthors } = parseTrailers(body);++ const merge = MERGE_RE.test(subject);+ const revertMatch = REVERT_RE.exec(subject);+ const header = HEADER_RE.exec(subject);++ const issues = [];+ const haystack = `${subject}\n${body}`;+ ISSUE_RE.lastIndex = 0;+ let issueMatch = ISSUE_RE.exec(haystack);+ while (issueMatch) {+ if (!issues.includes(issueMatch[1])) issues.push(issueMatch[1]);+ issueMatch = ISSUE_RE.exec(haystack);+ }++ // `BREAKING CHANGE:` in the body is the spec's second way to flag a break,+ // and `BREAKING-CHANGE:` is its blessed trailer alias.+ const bodyBreaking = /^BREAKING[ -]CHANGE:/m.test(body);++ if (!header) {+ return {+ subject,+ type: revertMatch ? 'revert' : null,+ scope: null,+ breaking: bodyBreaking,+ description: subject,+ body,+ trailers,+ coAuthors,+ issues,+ merge,+ revert: Boolean(revertMatch),+ revertedSubject: revertMatch ? revertMatch[1] : null,+ conventional: false,+ };+ }++ const [, type, scope, bang, description] = header;+ return {+ subject,+ type: type.toLowerCase(),+ scope: scope ? trim(scope) : null,+ breaking: bang === '!' || bodyBreaking,+ description: trim(description),+ body,+ trailers,+ coAuthors,+ issues,+ merge,+ revert: type.toLowerCase() === 'revert' || Boolean(revertMatch),+ revertedSubject: revertMatch ? revertMatch[1] : null,+ conventional: true,+ };+}++/**+ * Normalize a GitHub REST commit object (or anything shaped like one) into the+ * record the rest of this package works with. Accepts the shapes returned by+ * `/repos/{o}/{r}/commits` and by `git log --format=...` adapters.+ */+export function parseCommit(input) {+ const sha = String(input?.sha || input?.hash || '');+ const message = input?.commit?.message ?? input?.message ?? '';+ const parsed = parseCommitMessage(message);+ const committedAt =+ input?.commit?.committer?.date || input?.commit?.author?.date || input?.date || '';+ const authoredAt = input?.commit?.author?.date || input?.date || committedAt;+ const login = input?.author?.login || null;+ const name = input?.commit?.author?.name || input?.authorName || login || 'unknown';+ const parents = Array.isArray(input?.parents) ? input.parents.length : Number(input?.parents) || 0;++ return {+ ...parsed,+ sha,+ shortSha: sha.slice(0, 7),+ url: input?.html_url || input?.url || '',+ author: { login, name },+ authoredAt,+ committedAt,+ parents,+ // A second parent is the only unambiguous merge signal; the subject+ // pattern is a fallback for logs that do not carry parents.+ merge: parsed.merge || parents > 1,+ files: Array.isArray(input?.files)+ ? input.files.map((f) => String(f?.filename || f)).filter(Boolean)+ : null,+ };+}++/**+ * Human label for a commit's type + scope, e.g. `feat(resilience)` reads as+ * "Feature · resilience". Non-conventional subjects keep whatever leading+ * "Scope: " convention they used, so nothing is invented.+ */+export function headline(commit, { separator = ' · ' } = {}) {+ const c = commit && commit.type !== undefined ? commit : parseCommitMessage(String(commit || ''));+ if (c.type) {+ const label = TYPE_LABELS[c.type] || c.type;+ const bang = c.breaking ? ' (breaking)' : '';+ return c.scope ? `${label}${separator}${c.scope}${bang}` : `${label}${bang}`;+ }+ const idx = c.subject.indexOf(': ');+ if (idx > 0 && idx < 60) return c.subject.slice(0, idx);+ return 'New commit';+}++/** The one-line description a reader sees under the headline. */+export function summaryLine(commit) {+ const c = commit && commit.type !== undefined ? commit : parseCommitMessage(String(commit || ''));+ if (c.type) return c.description;+ const idx = c.subject.indexOf(': ');+ if (idx > 0 && idx < 60) return c.subject.slice(idx + 2);+ return c.subject;+}diff --git a/packages/shipfeed/src/render.js b/packages/shipfeed/src/render.jsnew file mode 100644index 000000000..23f94c835--- /dev/null+++ b/packages/shipfeed/src/render.js@@ -0,0 +1,183 @@+// Renderers. One feed, four audiences: a Telegram channel, a README, a reader,+// and a terminal. Each renderer is pure; nothing here talks to a network.++const escapeHtml = (s) =>+ String(s == null ? '' : s).replace(+ /[<>&]/g,+ (c) => ({ '<': '<', '>': '>', '&': '&' })[c],+ );++const escapeXml = (s) =>+ String(s == null ? '' : s).replace(+ /[<>&"']/g,+ (c) => ({ '<': '<', '>': '>', '&': '&', '"': '"', "'": ''' })[c],+ );++/**+ * One commit as a Telegram HTML message: bold headline, plain description, and+ * a provenance line. Matches the format three.ws posts to its holder channel.+ */+export function renderCommitTelegram(commit, { repo = '', linkText = '' } = {}) {+ const url = commit.url || (repo ? `https://github.com/${repo}/commit/${commit.sha}` : '');+ const shown =+ linkText || (repo ? `github.com/${repo}/commit/${commit.shortSha}` : commit.shortSha);+ const date = String(commit.date || '').slice(0, 10);+ return [+ `<b>${escapeHtml(commit.headline)}</b>`,+ '',+ escapeHtml(commit.summary),+ '',+ `${url ? `<a href="${url}">${escapeHtml(shown)}</a> · ` : ''}${escapeHtml(date)} · ${escapeHtml(commit.author)}`,+ ].join('\n');+}++/**+ * One release as a Telegram HTML message, with the provenance footer that the+ * raw commit feed can never carry: how much code this note actually covers.+ */+export function renderReleaseTelegram(release, { repo = '', siteUrl = '' } = {}) {+ const lines = [`<b>${escapeHtml(release.title)}</b>`, '', escapeHtml(release.summary)];+ if (release.commits.length) {+ const range = release.stats.range;+ const url =+ repo && range ? `https://github.com/${repo}/compare/${range.replace('..', '...')}` : '';+ const label = `${release.commits.length} commit${release.commits.length === 1 ? '' : 's'}`;+ lines.push('', url ? `shipped in <a href="${url}">${label}</a>` : `shipped in ${label}`);+ }+ const tags = release.tags.map((t) => `#${t}`).join(' ');+ const link = release.url || (siteUrl ? `${siteUrl}/changelog` : '');+ lines.push(+ '',+ `${link ? `${escapeHtml(link)} · ` : ''}${escapeHtml(release.date)}${tags ? ` · ${escapeHtml(tags)}` : ''}`,+ );+ return lines.join('\n');+}++/** The whole feed as Markdown, suitable for a RELEASES.md or a PR body. */+export function renderMarkdown(feed, { includeShips = true } = {}) {+ const out = [+ '# Ship log',+ '',+ `_${feed.stats.commits} commits · ${feed.stats.releases} releases · ${Math.round(feed.stats.coverage * 100)}% of visible commits accounted for_`,+ '',+ ];+ for (const r of feed.releases) {+ out.push(`## ${r.date} · ${r.title}`, '');+ if (r.summary) out.push(r.summary, '');+ if (r.tags.length) out.push(r.tags.map((t) => `\`${t}\``).join(' '), '');+ for (const c of r.commits) {+ out.push(+ `- \`${c.shortSha}\` **${c.headline}** ${c.summary}${c.url ? ` ([diff](${c.url}))` : ''}`,+ );+ }+ if (r.commits.length) out.push('');+ }+ if (includeShips && feed.ships.length) {+ out.push('## Unannounced work', '');+ for (const ship of feed.ships) {+ out.push(`### ${String(ship.start).slice(0, 10)} · ${ship.title}`, '');+ for (const c of ship.commits) {+ out.push(+ `- \`${c.shortSha}\` **${c.headline}** ${c.summary}${c.url ? ` ([diff](${c.url}))` : ''}`,+ );+ }+ out.push('');+ }+ }+ return out.join('\n').trimEnd() + '\n';+}++/** RSS 2.0 over the releases, with the commit list inline in each item. */+export function renderRss(feed, { title = 'Ship log', siteUrl = '', description = '' } = {}) {+ const items = feed.releases.slice(0, 50).map((r) => {+ const body = [+ `<p>${escapeXml(r.summary)}</p>`,+ r.commits.length+ ? `<ul>${r.commits+ .map(+ (c) =>+ `<li><code>${escapeXml(c.shortSha)}</code> ${escapeXml(c.headline)}: ${escapeXml(c.summary)}</li>`,+ )+ .join('')}</ul>`+ : '',+ ].join('');+ const link = r.url || siteUrl;+ return [+ ' <item>',+ ` <title>${escapeXml(r.title)}</title>`,+ link ? ` <link>${escapeXml(link)}</link>` : '',+ ` <guid isPermaLink="false">${escapeXml(r.key)}</guid>`,+ ` <pubDate>${new Date(`${r.date}T12:00:00Z`).toUTCString()}</pubDate>`,+ ` <description>${escapeXml(body)}</description>`,+ ' </item>',+ ]+ .filter(Boolean)+ .join('\n');+ });+ return [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<rss version="2.0">',+ ' <channel>',+ ` <title>${escapeXml(title)}</title>`,+ siteUrl ? ` <link>${escapeXml(siteUrl)}</link>` : '',+ ` <description>${escapeXml(description || 'Releases and the commits behind them.')}</description>`,+ ` <lastBuildDate>${new Date(feed.generatedAt).toUTCString()}</lastBuildDate>`,+ ...items,+ ' </channel>',+ '</rss>',+ ]+ .filter(Boolean)+ .join('\n');+}++const ESC = String.fromCharCode(27);+const ANSI = {+ reset: `${ESC}[0m`,+ dim: `${ESC}[2m`,+ bold: `${ESC}[1m`,+ green: `${ESC}[32m`,+ cyan: `${ESC}[36m`,+ yellow: `${ESC}[33m`,+ grey: `${ESC}[90m`,+};++const AUDIENCE_COLOR = { holder: ANSI.green, developer: ANSI.cyan, internal: ANSI.grey };++/** The feed as a terminal report. Set `color: false` for pipes and CI logs. */+export function renderTerminal(feed, { color = true, maxCommits = 6, maxReleases = 20 } = {}) {+ const c = (code, s) => (color ? `${code}${s}${ANSI.reset}` : String(s));+ const out = [];+ out.push(+ c(ANSI.bold, `ship log · ${feed.repo || 'repository'}`),+ c(+ ANSI.dim,+ `${feed.stats.commits} commits · ${feed.stats.releases} releases · ${feed.stats.linked} linked · ${Math.round(feed.stats.coverage * 100)}% coverage`,+ ),+ '',+ );+ for (const r of feed.releases.slice(0, maxReleases)) {+ out.push(`${c(ANSI.yellow, r.date)} ${c(ANSI.bold, r.title)}`);+ for (const commit of r.commits.slice(0, maxCommits)) {+ out.push(+ ` ${c(ANSI.grey, commit.shortSha)} ${c(AUDIENCE_COLOR[commit.audience] || '', commit.headline)} ${c(ANSI.dim, commit.summary)}`,+ );+ }+ if (r.commits.length > maxCommits) {+ out.push(c(ANSI.dim, ` +${r.commits.length - maxCommits} more`));+ }+ out.push('');+ }+ if (feed.ships.length) {+ out.push(c(ANSI.bold, 'unannounced'), '');+ for (const ship of feed.ships.slice(0, 10)) {+ out.push(+ `${c(ANSI.yellow, String(ship.start).slice(0, 10))} ${c(ANSI.bold, ship.title)} ${c(ANSI.dim, ship.summary)}`,+ );+ for (const commit of ship.commits.slice(0, maxCommits)) {+ out.push(` ${c(ANSI.grey, commit.shortSha)} ${c(ANSI.dim, commit.summary)}`);+ }+ out.push('');+ }+ }+ return out.join('\n');+}diff --git a/packages/shipfeed/src/sources.js b/packages/shipfeed/src/sources.jsnew file mode 100644index 000000000..4243936e1--- /dev/null+++ b/packages/shipfeed/src/sources.js@@ -0,0 +1,116 @@+// Where commits and changelog entries come from. Two real sources, no adapters+// in between: GitHub's REST API, and `git log` output from a local clone.+//+// Everything here uses the global fetch, so the module runs unchanged in Node,+// in a Cloud Run handler, and in a browser.++/** The exact `git log --format` string `commitsFromGitLog` expects. */+export const GIT_LOG_FORMAT = '%H%x00%cI%x00%aI%x00%an%x00%aN%x00%P%x00%B%x1e';++// The two separators the format string above emits. NUL cannot appear inside a+// commit message and RS (0x1e) is what git itself uses for record separation,+// so neither can be produced by commit text and confuse the split.+const FIELD_SEP = String.fromCharCode(0);+const RECORD_SEP = String.fromCharCode(30);++const GITHUB_API = 'https://api.github.com';++/**+ * Read commits from GitHub, newest first, following pages until `limit`.+ *+ * @param {{repo: string, branch?: string, limit?: number, token?: string,+ * fetchImpl?: typeof fetch, signal?: AbortSignal, userAgent?: string}} options+ * @returns {Promise<object[]>} raw GitHub commit objects+ */+export async function fetchGitHubCommits(options) {+ const {+ repo,+ branch = 'main',+ limit = 200,+ token = '',+ fetchImpl = fetch,+ signal,+ userAgent = 'shipfeed',+ } = options;+ if (!/^[\w.-]+\/[\w.-]+$/.test(String(repo || ''))) {+ throw new Error(`shipfeed: repo must look like "owner/name", got ${JSON.stringify(repo)}`);+ }++ const headers = { accept: 'application/vnd.github+json', 'user-agent': userAgent };+ if (token) headers.authorization = `Bearer ${token}`;++ const perPage = Math.min(100, Math.max(1, limit));+ const out = [];+ for (let page = 1; out.length < limit; page++) {+ const url = `${GITHUB_API}/repos/${repo}/commits?sha=${encodeURIComponent(branch)}&per_page=${perPage}&page=${page}`;+ const res = await fetchImpl(url, { headers, signal });+ if (!res.ok) {+ if (res.headers?.get?.('x-ratelimit-remaining') === '0') {+ const reset = Number(res.headers.get('x-ratelimit-reset')) * 1000;+ const until = Number.isFinite(reset) ? new Date(reset).toISOString() : 'unknown';+ throw new Error(+ `shipfeed: GitHub rate limit exhausted (resets ${until}); pass a token to raise it above 60/hour`,+ );+ }+ throw new Error(`shipfeed: GitHub commits fetch failed (${res.status}) for ${repo}`);+ }+ const batch = await res.json();+ if (!Array.isArray(batch) || batch.length === 0) break;+ out.push(...batch);+ if (batch.length < perPage) break;+ }+ return out.slice(0, limit);+}++/**+ * Parse `git log --format=<GIT_LOG_FORMAT>` output into GitHub-shaped commits,+ * so a local clone and the API produce identical feeds.+ */+export function commitsFromGitLog(text, { repo = '' } = {}) {+ return String(text || '')+ .split(RECORD_SEP)+ .map((record) => record.replace(/^\n+/, ''))+ .filter((record) => record.trim())+ .map((record) => {+ const [sha, committedAt, authoredAt, committerName, authorName, parents, message] =+ record.split(FIELD_SEP);+ return {+ sha,+ html_url: repo ? `https://github.com/${repo}/commit/${sha}` : '',+ parents: String(parents || '')+ .split(' ')+ .filter(Boolean)+ .map((p) => ({ sha: p })),+ commit: {+ message: message || '',+ author: { name: authorName || committerName || 'unknown', date: authoredAt },+ committer: { date: committedAt },+ },+ author: null,+ };+ });+}++/**+ * Load a changelog feed. Accepts three.ws's `public/changelog.json` shape+ * (`{entries: [...]}`), a bare array, or anything already in that form.+ */+export function normalizeChangelog(input) {+ const entries = Array.isArray(input) ? input : Array.isArray(input?.entries) ? input.entries : [];+ return entries+ .filter((e) => e && e.date && e.title)+ .map((e) => ({+ date: String(e.date).slice(0, 10),+ title: String(e.title),+ summary: String(e.summary || ''),+ tags: Array.isArray(e.tags) ? e.tags.map(String) : [],+ link: e.link ? String(e.link) : null,+ }));+}++/** Fetch and normalize a changelog JSON document over HTTP. */+export async function fetchChangelog(url, { fetchImpl = fetch, signal } = {}) {+ const res = await fetchImpl(url, { headers: { accept: 'application/json' }, signal });+ if (!res.ok) throw new Error(`shipfeed: changelog fetch failed (${res.status}) for ${url}`);+ return normalizeChangelog(await res.json());+}diff --git a/packages/shipfeed/tests/classify.test.js b/packages/shipfeed/tests/classify.test.jsnew file mode 100644index 000000000..95d2163d6--- /dev/null+++ b/packages/shipfeed/tests/classify.test.js@@ -0,0 +1,100 @@+import { describe, it, expect } from 'vitest';+import { classify, filterByAudience } from '../src/classify.js';++const commit = (message, over = {}) => ({+ sha: 'a'.repeat(40),+ parents: [{ sha: 'p' }],+ commit: { message, author: { name: 'nirholas', date: '2026-08-27T10:00:00Z' } },+ ...over,+});++describe('classify', () => {+ it('puts a substantive feature in front of holders', () => {+ const r = classify(commit('feat(wallet): let a user withdraw without a second signature'));+ expect(r.audience).toBe('holder');+ expect(r.signal).toBeGreaterThan(0.6);+ expect(r.noise).toBe(false);+ });++ it('treats dependency chores as internal noise', () => {+ const r = classify(commit('chore(deps): bump ws to 8.21'));+ expect(r.audience).toBe('internal');+ expect(r.noise).toBe(true);+ });++ it('treats merge commits as noise regardless of their subject', () => {+ const r = classify(commit('Merge pull request #4 from a/b'));+ expect(r.noise).toBe(true);+ expect(r.signal).toBe(0);+ });++ it('marks a lockfile-only change as noise when files are supplied', () => {+ const r = classify(+ commit('chore: refresh the lockfile', { files: [{ filename: 'package-lock.json' }] }),+ );+ expect(r.noise).toBe(true);+ expect(r.reasons.some((x) => x.rule === 'files:lockfile-only')).toBe(true);+ });++ it('raises a breaking change out of the internal bucket', () => {+ const r = classify(commit('refactor(sdk)!: rename every export'));+ expect(r.audience).not.toBe('internal');+ expect(r.reasons.some((x) => x.rule === 'breaking')).toBe(true);+ });++ it('keeps reverts visible', () => {+ const r = classify(commit('revert: roll back the prior change'));+ expect(r.audience).not.toBe('internal');+ });++ it('rewards a named product scope', () => {+ const plain = classify(commit('fix(forge): stop a stalled provider holding the request'));+ const named = classify(commit('fix(forge): stop a stalled provider holding the request'), {+ productScopes: ['forge'],+ });+ expect(named.signal).toBeGreaterThan(plain.signal);+ });++ it('explains every point it awarded', () => {+ const r = classify(commit('feat(forge): paint concept images across five providers'));+ expect(r.reasons.length).toBeGreaterThan(1);+ for (const reason of r.reasons) {+ expect(typeof reason.rule).toBe('string');+ expect(typeof reason.note).toBe('string');+ expect(Number.isFinite(reason.delta)).toBe(true);+ }+ });++ it('never returns a signal outside 0..1', () => {+ const messages = [+ 'feat(core)!: a very long and explanatory description of a breaking product change',+ 'style: x',+ 'chore(deps): y',+ ];+ for (const m of messages) {+ const { signal } = classify(commit(m), { productScopes: ['core'] });+ expect(signal).toBeGreaterThanOrEqual(0);+ expect(signal).toBeLessThanOrEqual(1);+ }+ });+});++describe('filterByAudience', () => {+ const commits = [+ commit('feat(wallet): let a user withdraw without a second signature'),+ commit('docs: explain the withdrawal flow for integrators'),+ commit('chore(deps): bump ws'),+ ];++ it('keeps everything at the internal floor', () => {+ expect(filterByAudience(commits, 'internal')).toHaveLength(3);+ });++ it('drops machinery at the developer floor', () => {+ expect(filterByAudience(commits, 'developer')).toHaveLength(2);+ });++ it('keeps only product news at the holder floor', () => {+ expect(filterByAudience(commits, 'holder')).toHaveLength(1);+ });+});diff --git a/packages/shipfeed/tests/link.test.js b/packages/shipfeed/tests/link.test.jsnew file mode 100644index 000000000..4df7557ee--- /dev/null+++ b/packages/shipfeed/tests/link.test.js@@ -0,0 +1,128 @@+import { describe, it, expect } from 'vitest';+import { linkCommits, entryKey, entrySlug, tokenize, buildIdf } from '../src/link.js';++const commit = (sha, message, date) => ({+ sha: sha.padEnd(40, '0'),+ commit: { message, author: { name: 'nirholas', date }, committer: { date } },+ parents: [{ sha: 'p' }],+});++const entries = [+ {+ date: '2026-08-27',+ title: 'Text-to-3D now has five independent image sources behind it',+ summary:+ 'Every text-to-3D generation starts by painting a concept image. The painter now tries five independent providers in order under one shared time budget.',+ tags: ['fix', 'infra'],+ link: '/forge',+ },+ {+ date: '2026-08-27',+ title: 'Agent history on Base and Gnosis is moving again',+ summary: 'The indexer no longer stops on an on-chain string Postgres cannot store.',+ tags: ['fix'],+ },+];++describe('tokenize', () => {+ it('drops stopwords, punctuation, and bare numbers', () => {+ expect(tokenize('The painter tries 5 providers, in order.')).toEqual([+ 'painter',+ 'tries',+ 'providers',+ 'order',+ ]);+ });+});++describe('buildIdf', () => {+ it('weights a rare term above a ubiquitous one', () => {+ const idf = buildIdf([+ ['agent', 'meshopt'],+ ['agent', 'wallet'],+ ['agent', 'forge'],+ ]);+ expect(idf.get('meshopt')).toBeGreaterThan(idf.get('agent'));+ });+});++describe('entrySlug', () => {+ it('matches the three.ws changelog permalink shape', () => {+ expect(entrySlug(entries[0])).toBe(+ '2026-08-27-text-to-3d-now-has-five-independent-image-sources-behind-it',+ );+ });+});++describe('linkCommits', () => {+ it('links a commit to the entry that shares its rare terms', () => {+ const commits = [+ commit('aaa1', 'feat(forge): paint concept images across five image providers', '2026-08-27T09:00:00Z'),+ commit('bbb2', 'fix(agent-index): keep crawling past an on-chain string Postgres cannot store', '2026-08-27T09:30:00Z'),+ ];+ const { byCommit } = linkCommits(entries, commits);+ expect(byCommit.get(commits[0].sha).entryKey).toBe(entryKey(entries[0]));+ expect(byCommit.get(commits[1].sha).entryKey).toBe(entryKey(entries[1]));+ });++ it('always prefers an explicit Changelog trailer', () => {+ const c = commit(+ 'ccc3',+ `chore: unrelated words entirely\n\nChangelog: ${entrySlug(entries[0])}`,+ '2026-08-27T09:00:00Z',+ );+ const { byCommit } = linkCommits(entries, [c]);+ expect(byCommit.get(c.sha)).toMatchObject({ entryKey: entryKey(entries[0]), score: 1 });+ expect(byCommit.get(c.sha).reasons[0]).toContain('trailer');+ });++ it('leaves a commit outside the time window unlinked', () => {+ const c = commit('ddd4', 'feat(forge): paint concept images across five providers', '2026-01-01T09:00:00Z');+ const { byCommit, orphans } = linkCommits(entries, [c]);+ expect(byCommit.size).toBe(0);+ expect(orphans).toHaveLength(1);+ });++ it('leaves an unrelated commit unlinked even inside the window', () => {+ const c = commit('eee5', 'style: reindent the footer', '2026-08-27T09:00:00Z');+ const { byCommit } = linkCommits(entries, [c], { threshold: 0.5 });+ expect(byCommit.size).toBe(0);+ });++ it('records why it linked what it linked', () => {+ const c = commit('fff6', 'fix(forge): give every concept-image provider a shared time budget', '2026-08-27T08:00:00Z');+ const { byCommit } = linkCommits(entries, [c]);+ const link = byCommit.get(c.sha);+ expect(link.reasons.join(' ')).toMatch(/shared terms/);+ expect(link.score).toBeGreaterThan(0);+ expect(link.score).toBeLessThanOrEqual(1);+ });++ it('orders an entry bucket oldest commit first', () => {+ const commits = [+ commit('1111', 'fix(forge): second concept image provider rung', '2026-08-27T11:00:00Z'),+ commit('2222', 'fix(forge): first concept image provider rung', '2026-08-27T08:00:00Z'),+ ];+ const { byEntry } = linkCommits(entries, commits);+ const bucket = byEntry.get(entryKey(entries[0]));+ expect(bucket.commits.map((c) => c.shortSha)).toEqual(['2222000', '1111000']);+ });++ it('caps how many commits one entry may claim', () => {+ const commits = Array.from({ length: 8 }, (_, i) =>+ commit(`c${i}`, `fix(forge): concept image provider rung ${i}`, '2026-08-27T08:00:00Z'),+ );+ const { byEntry, orphans } = linkCommits(entries, commits, { maxPerEntry: 3 });+ expect(byEntry.get(entryKey(entries[0])).commits).toHaveLength(3);+ expect(orphans.length).toBe(5);+ });++ it('never claims a commit for two entries', () => {+ const commits = [+ commit('9999', 'fix(forge): concept image providers and the on-chain string crawl', '2026-08-27T08:00:00Z'),+ ];+ const { byEntry } = linkCommits(entries, commits);+ const claims = [...byEntry.values()].filter((b) => b.commits.length).length;+ expect(claims).toBeLessThanOrEqual(1);+ });+});diff --git a/packages/shipfeed/tests/parse.test.js b/packages/shipfeed/tests/parse.test.jsnew file mode 100644index 000000000..8051114d3--- /dev/null+++ b/packages/shipfeed/tests/parse.test.js@@ -0,0 +1,140 @@+import { describe, it, expect } from 'vitest';+import {+ parseCommitMessage,+ parseCommit,+ parseTrailers,+ splitMessage,+ headline,+ summaryLine,+} from '../src/parse.js';++describe('splitMessage', () => {+ it('splits subject from body and normalizes CRLF', () => {+ expect(splitMessage('subject\r\n\r\nbody line\r\n')).toEqual({+ subject: 'subject',+ body: 'body line',+ });+ });++ it('handles a subject with no body', () => {+ expect(splitMessage('only a subject')).toEqual({ subject: 'only a subject', body: '' });+ });+});++describe('parseCommitMessage', () => {+ it('parses type, scope, and description', () => {+ const p = parseCommitMessage('feat(wallet): let a user withdraw');+ expect(p.type).toBe('feat');+ expect(p.scope).toBe('wallet');+ expect(p.description).toBe('let a user withdraw');+ expect(p.conventional).toBe(true);+ expect(p.breaking).toBe(false);+ });++ it('reads the bang as a breaking change', () => {+ expect(parseCommitMessage('feat(api)!: drop v1').breaking).toBe(true);+ });++ it('reads BREAKING CHANGE in the body as a breaking change', () => {+ const p = parseCommitMessage('fix: tighten a check\n\nBREAKING CHANGE: v1 callers must migrate');+ expect(p.breaking).toBe(true);+ });++ it('keeps a non-conventional subject intact', () => {+ const p = parseCommitMessage('Avatar Studio: 122 sliders');+ expect(p.conventional).toBe(false);+ expect(p.type).toBeNull();+ expect(p.subject).toBe('Avatar Studio: 122 sliders');+ });++ it('detects merges and reverts', () => {+ expect(parseCommitMessage('Merge pull request #12 from a/b').merge).toBe(true);+ const revert = parseCommitMessage('Revert "feat: a thing"');+ expect(revert.revert).toBe(true);+ expect(revert.revertedSubject).toBe('feat: a thing');+ });++ it('collects issue references once each', () => {+ const p = parseCommitMessage('fix: close #12\n\nAlso #12 and #34.');+ expect(p.issues).toEqual(['12', '34']);+ });++ it('lowercases the type but preserves scope casing', () => {+ const p = parseCommitMessage('FEAT(Avatar Studio): ship it');+ expect(p.type).toBe('feat');+ expect(p.scope).toBe('Avatar Studio');+ });+});++describe('parseTrailers', () => {+ it('reads only the final trailer block', () => {+ const { trailers, coAuthors } = parseTrailers(+ 'Prose: this colon is not a trailer\n\nCo-authored-by: A <[email protected]>\nChangelog: my-entry',+ );+ expect(trailers).toEqual({ changelog: 'my-entry' });+ expect(coAuthors).toEqual(['A <[email protected]>']);+ });++ it('returns empty structures for a body with no trailers', () => {+ expect(parseTrailers('just prose')).toEqual({ trailers: {}, coAuthors: [] });+ });+});++describe('parseCommit', () => {+ const github = {+ sha: 'abcdef1234567890abcdef1234567890abcdef12',+ html_url: 'https://github.com/o/r/commit/abcdef1',+ parents: [{ sha: 'p1' }],+ commit: {+ message: 'perf(index): drain the worst backlog first',+ author: { name: 'nirholas', date: '2026-08-27T10:00:00Z' },+ committer: { date: '2026-08-27T10:05:00Z' },+ },+ author: { login: 'nirholas' },+ };++ it('normalizes a GitHub commit object', () => {+ const c = parseCommit(github);+ expect(c.shortSha).toBe('abcdef1');+ expect(c.type).toBe('perf');+ expect(c.scope).toBe('index');+ expect(c.author).toEqual({ login: 'nirholas', name: 'nirholas' });+ expect(c.committedAt).toBe('2026-08-27T10:05:00Z');+ expect(c.merge).toBe(false);+ });++ it('treats a two-parent commit as a merge', () => {+ expect(parseCommit({ ...github, parents: [{ sha: 'a' }, { sha: 'b' }] }).merge).toBe(true);+ });++ it('flattens a files array when the caller supplied one', () => {+ const c = parseCommit({ ...github, files: [{ filename: 'api/x.js' }, 'src/y.js'] });+ expect(c.files).toEqual(['api/x.js', 'src/y.js']);+ });+});++describe('headline and summaryLine', () => {+ it('reads a scoped conventional commit as label plus scope', () => {+ expect(headline(parseCommitMessage('feat(resilience): add a breaker'))).toBe(+ 'Feature · resilience',+ );+ });++ it('marks breaking changes', () => {+ expect(headline(parseCommitMessage('feat(api)!: drop v1'))).toBe('Feature · api (breaking)');+ });++ it('falls back to the legacy "Scope: text" convention', () => {+ expect(headline(parseCommitMessage('Avatar Studio: 122 sliders'))).toBe('Avatar Studio');+ expect(summaryLine(parseCommitMessage('Avatar Studio: 122 sliders'))).toBe('122 sliders');+ });++ it('says "New commit" when a subject carries no convention at all', () => {+ expect(headline(parseCommitMessage('escape a raw NUL byte in the test'))).toBe('New commit');+ });++ it('accepts a raw string as well as a parsed record', () => {+ expect(headline('fix: a thing')).toBe('Fix');+ expect(summaryLine('fix: a thing')).toBe('a thing');+ });+});