7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
refactor(db): replace monolithic migrations.ts with file-per-migration system
Tazrif Raim054d6e51a044
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex f048440c..7b54b86d 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -21,6 +21,9 @@ jobs:- name: Install dependenciesrun: npm install+ - name: Check migration round trip+ run: npm run test:migrations+- name: Run testsrun: npm testdiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex fcdf00c1..99ed2669 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -6,9 +6,10 @@ Contributors are very welcome. This project is a local-first aggregator for free```bashnpm install-npm run dev # server on :3001, dashboard on :5173, both with HMR-npm test # server vitest; also runs client tests if present-npm run build # compile server and dashboard+npm run dev # server on :3001, dashboard on :5173, both with HMR+npm run db:migration:up # apply all the migrations to your local database+npm test # server vitest; also runs client tests if present+npm run build # compile server and dashboard```Every PR should:@@ -18,6 +19,19 @@ Every PR should:- Stay scoped to one change. Smaller PRs get reviewed and merged faster.- Avoid adding paid or card-gated services. This catalog only lists tiers that are genuinely free to start using without a credit card.+## Database migrations++Schema changes must use file-per-migration files under+`server/src/db/migrations/`. Do not edit previously applied migration files.++Control database migrations with ([db/README.md](server/src/db/README.md)):++```bash+npm run db:migration:create --name=add_embedding_index+npm run db:migration:up+npm run db:migration:down+```+## AI and LLM-assisted contributionsLLM-assisted PRs are welcome. A lot of this codebase is itself built that way, so there is no stigma here. The bar is the same as for any other PR: you are responsible for what you submit.@@ -46,4 +60,4 @@ Some useful fixes and experiments live in community forks and branches. If you a- `fix-119-atomic-ratelimits` — atomic SQLite `BEGIN IMMEDIATE` transactions to fix rate-limit race conditions.- `feature-122-auto-routing` — per-request `smart` / `fast` / `cheap` routing strategies.-If you port one of these into a PR, credit the original author in the PR description so they land in the Contributors list.+If you port one of these into a PR, credit the original author in the PR description so they land in the Contributors list.\ No newline at end of filediff --git a/README.md b/README.mdindex 11d7c191..55586205 100644--- a/README.md+++ b/README.md@@ -566,7 +566,17 @@ npm test # server vitest; also runs client tests if the workspace adds tnpm run build # compile server and dashboard```-PRs should include a test, keep the existing test suite green, and match the `.editorconfig` / tsconfig defaults already in the repo. Issues and discussions are open.+PRs should include a test, keep the existing test suite green, and match the `.editorconfig` / tsconfig defaults already in the repo. See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full contributor workflow.++### Database Migrations++In local development, apply pending migrations with:++```bash+NODE_ENV=development npm run db:migration:up+```++See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full migration CLI and workflow.### Contributorsdiff --git a/package.json b/package.jsonindex 7509a06c..3657429e 100644--- a/package.json+++ b/package.json@@ -12,9 +12,15 @@"test": "npm run test -w server && npm run test -w client --if-present","build": "npm run build -w server && npm run build -w client","build:server": "npm run build -w server",+ "test:migrations": "npm run test:migrations -w server","desktop:dev": "npm run build -w client && npm --prefix desktop run dev","desktop:dist": "npm run build -w client && npm --prefix desktop run dist",- "desktop:dist:win": "npm run build -w client && npm --prefix desktop run dist:win"+ "desktop:dist:win": "npm run build -w client && npm --prefix desktop run dist:win",+ "db:migration:up": "npm run db:migration:up -w server",+ "db:migration:down": "npm run db:migration:down -w server",+ "db:migration:fresh": "npm run db:migration:fresh -w server",+ "db:migration:status": "npm run db:migration:status -w server",+ "db:migration:create": "npm run db:migration:create -w server"},"devDependencies": {"concurrently": "^9.1.2"diff --git a/server/package.json b/server/package.jsonindex 2211929f..4cb7a9e1 100644--- a/server/package.json+++ b/server/package.json@@ -9,7 +9,13 @@"start": "node dist/index.js","test": "vitest run --pool=forks --fileParallelism=false","test:watch": "vitest",- "export-catalog": "tsx src/scripts/export-catalog.ts"+ "test:migrations": "vitest run --pool=forks --fileParallelism=false src/__tests__/db/migrate/roundtrip.test.ts",+ "export-catalog": "tsx src/scripts/export-catalog.ts",+ "db:migration:up": "tsx src/db/migrate/cli.ts up",+ "db:migration:down": "tsx src/db/migrate/cli.ts down",+ "db:migration:fresh": "tsx src/db/migrate/cli.ts fresh",+ "db:migration:status": "tsx src/db/migrate/cli.ts status",+ "db:migration:create": "tsx src/db/migrate/cli.ts create"},"dependencies": {"@freellmapi/shared": "*",diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.tsnew file mode 100644index 00000000..5d33ec36--- /dev/null+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts@@ -0,0 +1,193 @@+import Database from 'better-sqlite3';+import { describe, expect, it } from 'vitest';+import { connectDb } from '../../../db/index.js';+import { getMigrationStatuses, runMigrations } from '../../../db/migrate/runner.js';+import { up as runLegacyBaseline } from '../../../db/migrations/20260101_000000_legacy_baseline.js';++const LEGACY_BASELINE_FILENAME = '20260101_000000_legacy_baseline.ts';++interface SchemaRow {+ type: string;+ name: string;+ tbl_name: string;+ sql: string | null;+}++interface DatabaseSnapshot {+ schema: SchemaRow[];+ rows: Record<string, unknown[]>;+}++describe('migration round trip', () => {+ it('connectDb opens a connection without applying migrations', () => {+ const originalNodeEnv = process.env.NODE_ENV;+ process.env.NODE_ENV = 'test';+ const db = connectDb(':memory:');++ try {+ expect(hasTable(db, 'models')).toBe(false);+ expect(hasTable(db, 'migrations')).toBe(false);+ } finally {+ db.close();+ if (originalNodeEnv === undefined) {+ delete process.env.NODE_ENV;+ } else {+ process.env.NODE_ENV = originalNodeEnv;+ }+ }+ });++ it('runs the legacy baseline against existing legacy DBs so rebased legacy changes apply', async () => {+ const db = new Database(':memory:');++ try {+ runLegacyBaseline(db);+ db.prepare(`+ UPDATE models+ SET enabled = 1+ WHERE platform = 'opencode'+ AND model_id IN ('nemotron-3-super-free', 'minimax-m3-free')+ `).run();++ expect(getEnabledZenDeadPromoCount(db)).toBe(2);++ await runMigrations(db, 'up');++ expect(getEnabledZenDeadPromoCount(db)).toBe(0);+ expect(getAppliedMigrationNames(db)).toEqual([LEGACY_BASELINE_FILENAME]);+ } finally {+ db.close();+ }+ });++ it('runs all migrations up, down to baseline, then up to the same schema', async () => {+ const db = new Database(':memory:');++ try {+ await runMigrations(db, 'up');+ expect(getPendingMigrationNames(db)).toEqual([]);++ const fullState = snapshotAppState(db);+ await runDownToBaseline(db);++ expect(getAppliedMigrationNames(db)).toEqual([LEGACY_BASELINE_FILENAME]);++ await runMigrations(db, 'up');+ expect(getPendingMigrationNames(db)).toEqual([]);+ expect(snapshotAppState(db)).toEqual(fullState);+ } finally {+ db.close();+ }+ });+});++async function runDownToBaseline(db: Database.Database): Promise<void> {+ while (getAppliedMigrationNames(db).length > 1) {+ const migrationName = getLatestAppliedMigrationName(db);+ const before = snapshotAppState(db);++ await runMigrations(db, 'down');++ expect(snapshotAppState(db), `${migrationName} down() must alter app DB state or throw irreversible`)+ .not.toEqual(before);+ }+}++function getLatestAppliedMigrationName(db: Database.Database): string {+ const row = db.prepare(`+ SELECT filename+ FROM migrations+ ORDER BY id DESC+ LIMIT 1+ `).get() as { filename: string } | undefined;++ if (!row) throw new Error('No applied migrations found');+ return row.filename;+}++function getAppliedMigrationNames(db: Database.Database): string[] {+ return getMigrationStatuses(db)+ .filter(status => status.status === 'applied')+ .map(status => status.filename);+}++function getPendingMigrationNames(db: Database.Database): string[] {+ return getMigrationStatuses(db)+ .filter(status => status.status === 'pending')+ .map(status => status.filename);+}++function getEnabledZenDeadPromoCount(db: Database.Database): number {+ const row = db.prepare(`+ SELECT COUNT(*) AS count+ FROM models+ WHERE platform = 'opencode'+ AND model_id IN ('nemotron-3-super-free', 'minimax-m3-free')+ AND enabled = 1+ `).get() as { count: number };++ return row.count;+}++function snapshotSchema(db: Database.Database): SchemaRow[] {+ return db.prepare(`+ SELECT type, name, tbl_name, sql+ FROM sqlite_master+ WHERE type IN ('index', 'table', 'trigger', 'view')+ AND name NOT LIKE 'sqlite_%'+ ORDER BY type, name+ `).all() as SchemaRow[];+}++function snapshotAppState(db: Database.Database): DatabaseSnapshot {+ const tableNames = getAppTableNames(db);+ const rows: Record<string, unknown[]> = {};++ for (const tableName of tableNames) {+ rows[tableName] = snapshotTableRows(db, tableName);+ }++ return {+ schema: snapshotSchema(db),+ rows,+ };+}++function getAppTableNames(db: Database.Database): string[] {+ const rows = db.prepare(`+ SELECT name+ FROM sqlite_master+ WHERE type = 'table'+ AND name NOT LIKE 'sqlite_%'+ AND name <> 'migrations'+ ORDER BY name+ `).all() as { name: string }[];++ return rows.map(row => row.name);+}++function snapshotTableRows(db: Database.Database, tableName: string): unknown[] {+ const columns = db.prepare(`PRAGMA table_info(${quoteIdentifier(tableName)})`).all() as { name: string }[];+ const orderBy = columns.map(column => quoteIdentifier(column.name)).join(', ');++ return db.prepare(`+ SELECT *+ FROM ${quoteIdentifier(tableName)}+ ORDER BY ${orderBy}+ `).all() as unknown[];+}++function hasTable(db: Database.Database, tableName: string): boolean {+ const row = db.prepare(`+ SELECT name+ FROM sqlite_master+ WHERE type = 'table'+ AND name = ?+ `).get(tableName);++ return Boolean(row);+}++function quoteIdentifier(identifier: string): string {+ return `"${identifier.replaceAll('"', '""')}"`;+}diff --git a/server/src/__tests__/db/migrate/runner.test.ts b/server/src/__tests__/db/migrate/runner.test.tsnew file mode 100644index 00000000..168d4543--- /dev/null+++ b/server/src/__tests__/db/migrate/runner.test.ts@@ -0,0 +1,258 @@+import fs from 'fs';+import os from 'os';+import path from 'path';+import Database from 'better-sqlite3';+import { afterEach, describe, expect, it } from 'vitest';+import { runMigrations } from '../../../db/migrate/runner.js';++const tempDirs: string[] = [];+const LEGACY_BASELINE_FILENAME = '20260101_000000_legacy_baseline.ts';++afterEach(() => {+ for (const dir of tempDirs.splice(0)) {+ fs.rmSync(dir, { recursive: true, force: true });+ }+});++describe('migration runner', () => {+ it('runs pending migrations up and records applied files', async () => {+ const migrationsDir = createMigrationsDir();+ writeMigration(migrationsDir, '20260608_000001_create_widgets.ts', `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE IF NOT EXISTS widgets (id INTEGER PRIMARY KEY)');+ }++ export function down(db: Database.Database): void {+ db.exec('DROP TABLE IF EXISTS widgets');+ }+ `);+ writeMigration(migrationsDir, '20260608_000002_create_gadgets.ts', `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE IF NOT EXISTS gadgets (id INTEGER PRIMARY KEY)');+ }++ export function down(db: Database.Database): void {+ db.exec('DROP TABLE IF EXISTS gadgets');+ }+ `);++ const db = new Database(':memory:');+ try {+ await runMigrations(db, 'up', { migrationsDir });++ const applied = db.prepare('SELECT filename FROM migrations ORDER BY filename ASC').all() as { filename: string }[];+ expect(applied.map(row => row.filename)).toEqual([+ '20260608_000001_create_widgets.ts',+ '20260608_000002_create_gadgets.ts',+ ]);+ expect(hasTable(db, 'widgets')).toBe(true);+ expect(hasTable(db, 'gadgets')).toBe(true);+ } finally {+ db.close();+ }+ });++ it('runs one down migration and removes its migrations row', async () => {+ const migrationsDir = createMigrationsDir();+ writeMigration(migrationsDir, '20260608_000001_create_widgets.ts', `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE IF NOT EXISTS widgets (id INTEGER PRIMARY KEY)');+ }++ export function down(db: Database.Database): void {+ db.exec('DROP TABLE IF EXISTS widgets');+ }+ `);+ writeMigration(migrationsDir, '20260608_000002_create_gadgets.ts', `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE IF NOT EXISTS gadgets (id INTEGER PRIMARY KEY)');+ }++ export function down(db: Database.Database): void {+ db.exec('DROP TABLE IF EXISTS gadgets');+ }+ `);++ const db = new Database(':memory:');+ try {+ await runMigrations(db, 'up', { migrationsDir });+ await runMigrations(db, 'down', { migrationsDir });++ const applied = db.prepare('SELECT filename FROM migrations ORDER BY filename ASC').all() as { filename: string }[];+ expect(applied.map(row => row.filename)).toEqual(['20260608_000001_create_widgets.ts']);+ expect(hasTable(db, 'widgets')).toBe(true);+ expect(hasTable(db, 'gadgets')).toBe(false);+ } finally {+ db.close();+ }+ });++ it('skips already-applied migrations on later up runs', async () => {+ const migrationsDir = createMigrationsDir();+ writeMigration(migrationsDir, '20260608_000001_insert_counter.ts', `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE IF NOT EXISTS run_counter (value INTEGER NOT NULL)');+ db.prepare('INSERT INTO run_counter (value) VALUES (1)').run();+ }++ export function down(db: Database.Database): void {+ db.exec('DROP TABLE IF EXISTS run_counter');+ }+ `);++ const db = new Database(':memory:');+ try {+ await runMigrations(db, 'up', { migrationsDir });+ await runMigrations(db, 'up', { migrationsDir });++ const runs = db.prepare('SELECT COUNT(*) AS count FROM run_counter').get() as { count: number };+ const applied = db.prepare('SELECT COUNT(*) AS count FROM migrations').get() as { count: number };+ expect(runs.count).toBe(1);+ expect(applied.count).toBe(1);+ } finally {+ db.close();+ }+ });++ it('rolls back a failing up migration and does not record it', async () => {+ const migrationsDir = createMigrationsDir();+ writeMigration(migrationsDir, '20260608_000001_fails.ts', `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE failed_table (id INTEGER PRIMARY KEY)');+ throw new Error('boom');+ }++ export function down(db: Database.Database): void {+ db.exec('DROP TABLE IF EXISTS failed_table');+ }+ `);++ const db = new Database(':memory:');+ try {+ await expect(runMigrations(db, 'up', { migrationsDir })).rejects.toThrow('boom');++ const applied = db.prepare('SELECT COUNT(*) AS count FROM migrations').get() as { count: number };+ expect(applied.count).toBe(0);+ expect(hasTable(db, 'failed_table')).toBe(false);+ } finally {+ db.close();+ }+ });++ it('runs the legacy baseline even when the legacy sentinel schema exists', async () => {+ const migrationsDir = createMigrationsDir();+ writeMigration(migrationsDir, LEGACY_BASELINE_FILENAME, `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE IF NOT EXISTS baseline_ran (id INTEGER PRIMARY KEY)');+ }++ export function down(_db: Database.Database): void {+ throw new Error('baseline is irreversible');+ }+ `);+ writeMigration(migrationsDir, '20260608_000001_after_baseline.ts', `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE IF NOT EXISTS after_baseline (id INTEGER PRIMARY KEY)');+ }++ export function down(db: Database.Database): void {+ db.exec('DROP TABLE IF EXISTS after_baseline');+ }+ `);++ const db = new Database(':memory:');+ try {+ db.exec(`+ CREATE TABLE models (+ id INTEGER PRIMARY KEY,+ supports_tools INTEGER NOT NULL DEFAULT 0+ );+ `);++ await runMigrations(db, 'up', { migrationsDir });++ const applied = db.prepare('SELECT filename FROM migrations ORDER BY id ASC').all() as { filename: string }[];+ expect(applied.map(row => row.filename)).toEqual([+ LEGACY_BASELINE_FILENAME,+ '20260608_000001_after_baseline.ts',+ ]);+ expect(hasTable(db, 'baseline_ran')).toBe(true);+ expect(hasTable(db, 'after_baseline')).toBe(true);+ } finally {+ db.close();+ }+ });++ it('runs the legacy baseline normally for fresh DBs', async () => {+ const migrationsDir = createMigrationsDir();+ writeMigration(migrationsDir, LEGACY_BASELINE_FILENAME, `+ import type Database from 'better-sqlite3';++ export function up(db: Database.Database): void {+ db.exec('CREATE TABLE models (id INTEGER PRIMARY KEY, supports_tools INTEGER NOT NULL DEFAULT 0)');+ }++ export function down(_db: Database.Database): void {+ throw new Error('baseline is irreversible');+ }+ `);++ const db = new Database(':memory:');+ try {+ await runMigrations(db, 'up', { migrationsDir });++ const applied = db.prepare('SELECT filename FROM migrations ORDER BY id ASC').all() as { filename: string }[];+ expect(applied.map(row => row.filename)).toEqual([LEGACY_BASELINE_FILENAME]);+ expect(hasTable(db, 'models')).toBe(true);+ expect(hasColumn(db, 'models', 'supports_tools')).toBe(true);+ } finally {+ db.close();+ }+ });+});++function createMigrationsDir(): string {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'freellmapi-migrations-'));+ tempDirs.push(dir);+ return dir;+}++function writeMigration(migrationsDir: string, filename: string, source: string): void {+ fs.writeFileSync(path.join(migrationsDir, filename), source.trimStart());+}++function hasTable(db: Database.Database, tableName: string): boolean {+ const row = db.prepare(`+ SELECT name+ FROM sqlite_master+ WHERE type = 'table'+ AND name = ?+ `).get(tableName);++ return Boolean(row);+}++function hasColumn(db: Database.Database, tableName: string, columnName: string): boolean {+ const columns = db.prepare(`PRAGMA table_info(${quoteIdentifier(tableName)})`).all() as { name: string }[];+ return columns.some(column => column.name === columnName);+}++function quoteIdentifier(identifier: string): string {+ return `"${identifier.replaceAll('"', '""')}"`;+}diff --git a/server/src/__tests__/services/catalog-sync.test.ts b/server/src/__tests__/services/catalog-sync.test.tsindex b810ed4d..e57c7312 100644--- a/server/src/__tests__/services/catalog-sync.test.ts+++ b/server/src/__tests__/services/catalog-sync.test.ts@@ -1,7 +1,7 @@import { describe, it, expect, beforeAll } from 'vitest';import { initDb, getDb, setSetting, getSetting } from '../../db/index.js';import { applyCatalog, reapplyCachedCatalog, MIN_CATALOG_VERSION } from '../../services/catalog-sync.js';-import { migrateDbSchema } from '../../db/migrations.js';+import { runMigrationsSync } from '../../db/migrate/runner.js';// applyCatalog is the write path between the published catalog and the live// router DB. These tests lock its contract: catalog metadata always wins, the@@ -210,7 +210,8 @@ describe('reapplyCachedCatalog', () => {).toBeUndefined();// Simulate a restart: migrations re-insert the baseline model.- migrateDbSchema(getDb());+ getDb().exec('DROP TABLE migrations');+ runMigrationsSync(getDb(), 'up');expect(getDb().prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?').get(victim.platform, victim.modelId),).toBeDefined();diff --git a/server/src/db/README.md b/server/src/db/README.mdnew file mode 100644index 00000000..4242abfa--- /dev/null+++ b/server/src/db/README.md@@ -0,0 +1,31 @@+# Database Migrations++Schema changes belong in migration files under `src/db/migrations/`.+Create them with:++```sh+npm run db:migration:create --name=add_embedding_index+```++In local development, run pending migrations manually with:++```sh+npm run db:migration:up+```++Production startup applies pending migrations automatically. Tests auto-run+migrations from `initDb()` so each test DB starts with the full schema.++Use `db:migration:fresh` only for local development or tests. It drops app+tables, then runs all migrations so the DB is immediately usable:++```sh+npm run db:migration:fresh+```++Never run `db:migration:fresh` in production.++Built-in model catalog rows live in migrations. Add, retire, or correct default+models with a new forward migration.++Use `db:migration:status` to inspect which migration files have already run.diff --git a/server/src/db/index.ts b/server/src/db/index.tsindex f5861ff6..d800c2c4 100644--- a/server/src/db/index.ts+++ b/server/src/db/index.ts@@ -3,7 +3,7 @@ import Database from 'better-sqlite3';import fs from 'fs';import path from 'path';import { fileURLToPath } from 'url';-import { migrateDbSchema } from './migrations.js';+import { runMigrationsSync } from './migrate/runner.js';const __dirname = path.dirname(fileURLToPath(import.meta.url));const DB_PATH = path.resolve(__dirname, '../../data/freeapi.db');@@ -12,12 +12,12 @@ let db: Database.Database;export function getDb(): Database.Database {if (!db) {- throw new Error('Database not initialized. Call initDb() first.');+ throw new Error('Database not initialized. Call initDb() or connectDb() first.');}return db;}-export function initDb(dbPath?: string): Database.Database {+export function connectDb(dbPath?: string): Database.Database {const resolvedPath = dbPath ?? DB_PATH;const isMemory = resolvedPath === ':memory:';@@ -32,12 +32,33 @@ export function initDb(dbPath?: string): Database.Database {if (!isMemory) db.pragma('journal_mode = WAL');db.pragma('foreign_keys = ON');- migrateDbSchema(db);-console.log(`Database initialized at ${resolvedPath}`);return db;}+export function initDb(dbPath?: string): Database.Database {+ const db = connectDb(dbPath);++ if (process.env.NODE_ENV !== 'development') {+ runMigrationsSync(db, 'up');+ } else {+ // In dev, verify the DB has been initialised. If not, give a clear error.+ const ready = db.prepare(+ "SELECT name FROM sqlite_master WHERE type='table' AND name='migrations'"+ ).get();+ if (!ready) {+ console.error(+ '\n [dev] Database not initialised. Run:\n\n' ++ ' npm run db:migration:up\n\n' ++ ' Then restart the server.\n'+ );+ process.exit(1);+ }+ }++ return db;+}+export function getUnifiedApiKey(): string {const db = getDb();const row = db.prepare("SELECT value FROM settings WHERE key = 'unified_api_key'").get() as { value: string };diff --git a/server/src/db/migrate/TEMPLATE.ts b/server/src/db/migrate/TEMPLATE.tsnew file mode 100644index 00000000..09ad8765--- /dev/null+++ b/server/src/db/migrate/TEMPLATE.ts@@ -0,0 +1,22 @@+// Migration: <short description>+// Created: <YYYY-MM-DD>+//+// DOWN: <reversible | irreversible - reason>++import type Database from 'better-sqlite3';++export function up(db: Database.Database): void {+ db.exec(`+ -- your SQL here+ `);+}++export function down(db: Database.Database): void {+ // If reversible:+ db.exec(`+ -- inverse SQL here+ `);++ // If irreversible:+ // throw new Error('irreversible migration: <reason>');+}diff --git a/server/src/db/migrate/cli.ts b/server/src/db/migrate/cli.tsnew file mode 100644index 00000000..5e8ced29--- /dev/null+++ b/server/src/db/migrate/cli.ts@@ -0,0 +1,218 @@+import '../../env.js';+import fs from 'fs';+import path from 'path';+import { fileURLToPath } from 'url';+import type Database from 'better-sqlite3';+import { connectDb } from '../index.js';+import { getMigrationStatuses, runMigrations } from './runner.js';++type Command = 'up' | 'down' | 'fresh' | 'status' | 'create';++const __dirname = path.dirname(fileURLToPath(import.meta.url));+const MIGRATIONS_DIR = path.resolve(__dirname, '../migrations');+const TEMPLATE_PATH = path.resolve(__dirname, 'TEMPLATE.ts');+const DEFAULTS_PATH = path.resolve(__dirname, 'defaults.ts');++async function main(): Promise<void> {+ const command = process.argv[2] as Command | undefined;++ if (command === 'create') {+ createMigrationFile();+ return;+ }++ const db = connectDb();++ switch (command) {+ case 'up':+ await runMigrations(db, 'up');+ return;+ case 'down':+ await runMigrations(db, 'down');+ return;+ case 'fresh':+ await runFresh(db);+ return;+ case 'status':+ printStatus(db);+ return;+ default:+ console.error('Usage: tsx src/db/migrate/cli.ts <up|down|fresh|status|create>');+ process.exit(1);+ }+}++async function runFresh(db: Database.Database): Promise<void> {+ if (process.env.NODE_ENV === 'production') {+ console.error('db:migration:fresh is not allowed in production');+ process.exit(1);+ }++ dropAllUserTables(db);+ await runMigrations(db, 'up');+}++function printStatus(db: Database.Database): void {+ const statuses = getMigrationStatuses(db);+ console.table(statuses.map(status => ({+ filename: status.filename,+ status: status.status,+ applied_at: status.appliedAt ?? '',+ })));+}++function createMigrationFile(): void {+ const rawName = getMigrationName();+ if (!rawName) {+ console.error('Error: --name is required. Usage: npm run db:migration:create --name=<description>');+ process.exit(1);+ }++ const sanitisedName = sanitiseMigrationName(rawName);+ if (!sanitisedName) {+ console.error('Error: --name must contain at least one letter or digit');+ process.exit(1);+ }++ const now = new Date();+ const filename = `${formatTimestamp(now)}_${sanitisedName}.ts`;+ const outputPath = path.join(MIGRATIONS_DIR, filename);+ const relativeOutputPath = toPosixPath(path.relative(process.cwd(), outputPath));++ if (!fs.existsSync(TEMPLATE_PATH)) {+ console.error(`Error: migration template not found at ${toPosixPath(TEMPLATE_PATH)}`);+ process.exit(1);+ }++ if (fs.existsSync(outputPath)) {+ console.error(`Error: migration already exists at ${relativeOutputPath}`);+ process.exit(1);+ }++ fs.mkdirSync(MIGRATIONS_DIR, { recursive: true });+ const template = fs.readFileSync(TEMPLATE_PATH, 'utf8');+ const content = template+ .replaceAll('<short description>', rawName)+ .replaceAll('<YYYY-MM-DD>', formatDate(now));++ fs.writeFileSync(outputPath, content, { flag: 'wx' });+ updateDefaultMigrationRegistry(filename);+ console.log(`Created ${relativeOutputPath}`);+}++function updateDefaultMigrationRegistry(filename: string): void {+ if (!fs.existsSync(DEFAULTS_PATH)) {+ console.error(`Error: migration registry not found at ${toPosixPath(DEFAULTS_PATH)}`);+ process.exit(1);+ }++ const bindingName = toMigrationBindingName(filename);+ const importPath = `../migrations/${filename.replace(/\.ts$/, '.js')}`;+ const importLine = `import * as ${bindingName} from '${importPath}';`;+ const entryLine = ` { filename: '${filename}', module: ${bindingName} },`;+ const registry = fs.readFileSync(DEFAULTS_PATH, 'utf8');++ if (registry.includes(importLine) || registry.includes(entryLine)) return;++ const withImport = registry.replace(+ '\nexport interface MigrationModule',+ `\n${importLine}\n\nexport interface MigrationModule`,+ );++ if (withImport === registry) {+ console.error('Error: could not update migration registry imports');+ process.exit(1);+ }++ const withEntry = withImport.replace(+ /\n\];\s*$/,+ `\n${entryLine}\n];\n`,+ );++ if (withEntry === withImport) {+ console.error('Error: could not update migration registry entries');+ process.exit(1);+ }++ fs.writeFileSync(DEFAULTS_PATH, withEntry);+}++function getMigrationName(): string | undefined {+ const argvName = process.argv+ .map(arg => arg.match(/^--name=(.+)$/)?.[1]?.trim())+ .find((name): name is string => Boolean(name));++ return argvName || process.env.npm_config_name?.trim();+}++function sanitiseMigrationName(name: string): string {+ return name+ .toLowerCase()+ .replace(/[ -]+/g, '_')+ .replace(/[^a-z0-9_]/g, '')+ .replace(/_+/g, '_')+ .replace(/^_+|_+$/g, '');+}++function formatTimestamp(date: Date): string {+ return [+ date.getUTCFullYear(),+ pad(date.getUTCMonth() + 1),+ pad(date.getUTCDate()),+ '_',+ pad(date.getUTCHours()),+ pad(date.getUTCMinutes()),+ pad(date.getUTCSeconds()),+ ].join('');+}++function formatDate(date: Date): string {+ return [+ date.getUTCFullYear(),+ pad(date.getUTCMonth() + 1),+ pad(date.getUTCDate()),+ ].join('-');+}++function pad(value: number): string {+ return String(value).padStart(2, '0');+}++function toMigrationBindingName(filename: string): string {+ return `migration${filename.replace(/\.ts$/, '').replace(/[^a-zA-Z0-9_]/g, '_')}`;+}++function dropAllUserTables(db: Database.Database): void {+ const tables = db.prepare(`+ SELECT name+ FROM sqlite_master+ WHERE type = 'table'+ AND name NOT LIKE 'sqlite_%'+ ORDER BY name ASC+ `).all() as { name: string }[];++ db.pragma('foreign_keys = OFF');+ try {+ const dropTables = db.transaction(() => {+ for (const { name } of tables) {+ db.exec(`DROP TABLE IF EXISTS ${quoteIdentifier(name)}`);+ }+ });+ dropTables();+ } finally {+ db.pragma('foreign_keys = ON');+ }+}++function quoteIdentifier(identifier: string): string {+ return `"${identifier.replaceAll('"', '""')}"`;+}++function toPosixPath(value: string): string {+ return value.replaceAll(path.sep, '/');+}++main().catch((err: unknown) => {+ console.error(err);+ process.exit(1);+});diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.tsnew file mode 100644index 00000000..0d35b62f--- /dev/null+++ b/server/src/db/migrate/defaults.ts@@ -0,0 +1,18 @@+import type Database from 'better-sqlite3';+import * as legacyBaseline from '../migrations/20260101_000000_legacy_baseline.js';++export interface MigrationModule {+ up(db: Database.Database): void;+ down(db: Database.Database): void;+}++export interface DefaultMigration {+ filename: string;+ module: MigrationModule;+}++export const LEGACY_BASELINE_FILENAME = '20260101_000000_legacy_baseline.ts';++export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [+ { filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },+];diff --git a/server/src/db/migrate/runner.ts b/server/src/db/migrate/runner.tsnew file mode 100644index 00000000..5d5f67a9--- /dev/null+++ b/server/src/db/migrate/runner.ts@@ -0,0 +1,264 @@+import fs from 'fs';+import path from 'path';+import { fileURLToPath, pathToFileURL } from 'url';+import type Database from 'better-sqlite3';+import { DEFAULT_MIGRATIONS, type MigrationModule } from './defaults.js';++export type MigrationDirection = 'up' | 'down';+export type MigrationState = 'applied' | 'pending';++export interface MigrationRunnerOptions {+ migrationsDir?: string;+ migrationFileExtension?: '.ts' | '.js';+}++export interface MigrationStatus {+ filename: string;+ status: MigrationState;+ appliedAt: string | null;+}++interface AppliedMigrationRow {+ filename: string;+ applied_at: string;+}++interface MigrationRecord {+ filename: string;+ module?: MigrationModule;+}++const __dirname = path.dirname(fileURLToPath(import.meta.url));+const DEFAULT_MIGRATIONS_DIR = path.resolve(__dirname, '../migrations');++const CREATE_MIGRATIONS_TABLE_SQL = `+ CREATE TABLE IF NOT EXISTS migrations (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ filename TEXT NOT NULL UNIQUE,+ applied_at TEXT NOT NULL DEFAULT (datetime('now'))+ )+`;++export async function runMigrations(+ db: Database.Database,+ direction: MigrationDirection = 'up',+ options: MigrationRunnerOptions = {},+): Promise<void> {+ initializeMigrationTracking(db);+ const records = getMigrationRecords(options);++ if (direction === 'up') {+ await runPendingMigrations(db, records, options);+ return;+ }++ if (direction === 'down') {+ await runLatestDownMigration(db, records, options);+ return;+ }++ throw new Error(`Unknown migration direction: ${direction}`);+}++export function runMigrationsSync(+ db: Database.Database,+ direction: MigrationDirection = 'up',+): void {+ initializeMigrationTracking(db);+ const records = getDefaultMigrationRecords();++ if (direction === 'up') {+ runPendingMigrationsSync(db, records);+ return;+ }++ if (direction === 'down') {+ runLatestDownMigrationSync(db, records);+ return;+ }++ throw new Error(`Unknown migration direction: ${direction}`);+}++export function getMigrationStatuses(+ db: Database.Database,+ options: MigrationRunnerOptions = {},+): MigrationStatus[] {+ initializeMigrationTracking(db);++ const applied = getAppliedMigrations(db);+ return getMigrationRecords(options).map(record => ({+ filename: record.filename,+ status: applied.has(record.filename) ? 'applied' : 'pending',+ appliedAt: applied.get(record.filename) ?? null,+ }));+}++function initializeMigrationTracking(db: Database.Database): void {+ ensureMigrationsTable(db);+}++function ensureMigrationsTable(db: Database.Database): void {+ db.exec(CREATE_MIGRATIONS_TABLE_SQL);+}++async function runPendingMigrations(+ db: Database.Database,+ records: readonly MigrationRecord[],+ options: MigrationRunnerOptions,+): Promise<void> {+ const applied = getAppliedMigrations(db);++ for (const record of records) {+ if (applied.has(record.filename)) continue;++ const migration = await loadMigrationModule(record, options);+ const applyMigration = db.transaction(() => {+ migration.up(db);+ db.prepare('INSERT INTO migrations (filename) VALUES (?)').run(record.filename);+ });++ applyMigration();+ applied.set(record.filename, new Date().toISOString());+ }+}++async function runLatestDownMigration(+ db: Database.Database,+ records: readonly MigrationRecord[],+ options: MigrationRunnerOptions,+): Promise<void> {+ const row = db.prepare(`+ SELECT filename+ FROM migrations+ ORDER BY id DESC+ LIMIT 1+ `).get() as { filename: string } | undefined;++ if (!row) return;++ const record = records.find(record => record.filename === row.filename);+ if (!record) throw new Error(`Migration file not found: ${row.filename}`);++ const migration = await loadMigrationModule(record, options);+ const revertMigration = db.transaction(() => {+ migration.down(db);+ db.prepare('DELETE FROM migrations WHERE filename = ?').run(row.filename);+ });++ revertMigration();+}++function runPendingMigrationsSync(+ db: Database.Database,+ records: readonly MigrationRecord[],+): void {+ const applied = getAppliedMigrations(db);++ for (const record of records) {+ if (applied.has(record.filename)) continue;+ if (!record.module) throw new Error(`Migration ${record.filename} cannot run synchronously`);++ const applyMigration = db.transaction(() => {+ record.module!.up(db);+ db.prepare('INSERT INTO migrations (filename) VALUES (?)').run(record.filename);+ });++ applyMigration();+ applied.set(record.filename, new Date().toISOString());+ }+}++function runLatestDownMigrationSync(+ db: Database.Database,+ records: readonly MigrationRecord[],+): void {+ const row = db.prepare(`+ SELECT filename+ FROM migrations+ ORDER BY id DESC+ LIMIT 1+ `).get() as { filename: string } | undefined;++ if (!row) return;++ const record = records.find(record => record.filename === row.filename);+ if (!record?.module) throw new Error(`Migration ${row.filename} cannot run synchronously`);++ const revertMigration = db.transaction(() => {+ record.module!.down(db);+ db.prepare('DELETE FROM migrations WHERE filename = ?').run(row.filename);+ });++ revertMigration();+}++function getAppliedMigrations(db: Database.Database): Map<string, string> {+ const rows = db.prepare(`+ SELECT filename, applied_at+ FROM migrations+ ORDER BY filename ASC+ `).all() as AppliedMigrationRow[];++ return new Map(rows.map(row => [row.filename, row.applied_at]));+}++function getMigrationRecords(options: MigrationRunnerOptions): MigrationRecord[] {+ if (isDefaultMigrationSet(options)) return getDefaultMigrationRecords();++ return getMigrationFilenames(options).map(filename => ({ filename }));+}++function getDefaultMigrationRecords(): MigrationRecord[] {+ return DEFAULT_MIGRATIONS.map(migration => ({+ filename: migration.filename,+ module: migration.module,+ }));+}++function getMigrationFilenames(options: MigrationRunnerOptions): string[] {+ const migrationsDir = getMigrationsDir(options);+ if (!fs.existsSync(migrationsDir)) return [];++ const extension = getMigrationFileExtension(options);+ return fs.readdirSync(migrationsDir, { withFileTypes: true })+ .filter(entry => entry.isFile())+ .map(entry => entry.name)+ .filter(filename => filename.endsWith(extension) && !filename.endsWith('.d.ts'))+ .sort((left, right) => left.localeCompare(right));+}++async function loadMigrationModule(+ record: MigrationRecord,+ options: MigrationRunnerOptions,+): Promise<MigrationModule> {+ if (record.module) return record.module;++ const migrationPath = path.join(getMigrationsDir(options), record.filename);+ if (!fs.existsSync(migrationPath)) {+ throw new Error(`Migration file not found: ${record.filename}`);+ }++ const imported = await import(pathToFileURL(migrationPath).href) as Partial<MigrationModule>;+ if (typeof imported.up !== 'function' || typeof imported.down !== 'function') {+ throw new Error(`Migration ${record.filename} must export up(db) and down(db) functions`);+ }++ return {+ up: imported.up,+ down: imported.down,+ };+}++function isDefaultMigrationSet(options: MigrationRunnerOptions): boolean {+ return options.migrationsDir === undefined && options.migrationFileExtension === undefined;+}++function getMigrationsDir(options: MigrationRunnerOptions): string {+ return options.migrationsDir ?? DEFAULT_MIGRATIONS_DIR;+}++function getMigrationFileExtension(options: MigrationRunnerOptions): '.ts' | '.js' {+ if (options.migrationFileExtension) return options.migrationFileExtension;+ return fileURLToPath(import.meta.url).endsWith('.ts') ? '.ts' : '.js';+}diff --git a/server/src/db/migrations.ts b/server/src/db/migrations/20260101_000000_legacy_baseline.tssimilarity index 99%rename from server/src/db/migrations.tsrename to server/src/db/migrations/20260101_000000_legacy_baseline.tsindex abee479f..0593d10c 100644--- a/server/src/db/migrations.ts+++ b/server/src/db/migrations/20260101_000000_legacy_baseline.ts@@ -1,9 +1,9 @@import crypto from 'crypto';import Database from 'better-sqlite3';-import { initEncryptionKey } from '../lib/crypto.js';-import { applyModelPricing } from './model-pricing.js';+import { initEncryptionKey } from '../../lib/crypto.js';+import { applyModelPricing } from '../model-pricing.js';-export function migrateDbSchema(db: Database.Database) {+export function up(db: Database.Database): void {createTables(db);initEncryptionKey(db);seedModels(db);@@ -50,6 +50,10 @@ export function migrateDbSchema(db: Database.Database) {migrateProfilesInit(db);}+export function down(_db: Database.Database): void {+ throw new Error('Legacy baseline is irreversible - restore from backup');+}+function createTables(db: Database.Database) {db.exec(`CREATE TABLE IF NOT EXISTS models (