Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
17.2 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import * as fs from 'node:fs';
import {DatabaseSync, type SQLOutputValue, type StatementSync} from 'node:sqlite';
import type {EngineClock, EngineTimer} from '@electron/engine/EngineClock';
import type {
	AccountEngineSnapshotCursor,
	AccountEngineSnapshotEntries,
} from '@fluxer/desktop_ipc/src/AccountEngineContract';
import {ACCOUNT_ENGINE_CORE_SCHEMA_EPOCH_OFFSET} from '@fluxer/desktop_ipc/src/AccountEngineCoreContract';
import {isRecord} from '@fluxer/record_utils/src/RecordGuards';

const ENGINE_CACHE_EPOCH = 1;
const CACHE_COMMIT_INTERVAL_MS = 250;
const CACHE_COMMIT_MAX_OPS = 4000;
export const CACHE_ACCOUNT_MAX_ROWS = 2_000_000;
export const CACHE_ACCOUNT_MAX_VALUE_BYTES = 1024 * 1024 * 1024;
const MENTION_PRUNE_SLACK = 50;
const SQLITE_PRIMARY_RESULT_MASK = 0xff;
const SQLITE_CORRUPT = 11;
const SQLITE_NOTADB = 26;

const SCHEMA = `
CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT);
CREATE TABLE IF NOT EXISTS rows(account_key TEXT, entity TEXT, key TEXT, value TEXT,
	PRIMARY KEY(account_key, entity, key)) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS cursors(account_key TEXT PRIMARY KEY, session_id TEXT, schema_epoch INTEGER, updated_at INTEGER);
CREATE TABLE IF NOT EXISTS summaries(account_key TEXT PRIMARY KEY, json TEXT, updated_at INTEGER);
CREATE TABLE IF NOT EXISTS mentions(account_key TEXT, message_id TEXT, created_at INTEGER, json TEXT,
	PRIMARY KEY(account_key, message_id)) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS mentions_by_time ON mentions(account_key, created_at DESC, message_id DESC);
`;

interface CacheAccountUsage {
	rows: number;
	bytes: number;
}

interface CacheOpenReport {
	readonly reset: 'none' | 'epoch' | 'corrupt';
	readonly memoryFallback: string | null;
}

function prefixUpperBound(prefix: string): string | null {
	const codePoints = Array.from(prefix);
	const last = codePoints.pop()?.codePointAt(0);
	if (last === undefined || last >= 0x10ffff) {
		return null;
	}
	return codePoints.join('') + String.fromCodePoint(last + 1);
}

function removeDatabaseFiles(path: string): void {
	for (const suffix of ['', '-wal', '-shm']) {
		fs.rmSync(`${path}${suffix}`, {force: true});
	}
}

function closeQuietly(db: DatabaseSync | null): void {
	try {
		db?.close();
	} catch {}
}

function readStoredEpoch(db: DatabaseSync): string | null {
	const row = db.prepare('SELECT value FROM meta WHERE key = ?').get('schema_epoch') as
		| {value: string | null}
		| undefined;
	return row?.value ?? null;
}

function writeStoredEpoch(db: DatabaseSync, schemaEpoch: string): void {
	db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run('schema_epoch', schemaEpoch);
}

function openMemoryDatabase(schemaEpoch: string): DatabaseSync {
	const db = new DatabaseSync(':memory:');
	db.exec(SCHEMA);
	writeStoredEpoch(db, schemaEpoch);
	return db;
}

function ensurePrivateFile(path: string): void {
	const handle = fs.openSync(path, 'a', 0o600);
	fs.closeSync(handle);
	try {
		fs.chmodSync(path, 0o600);
	} catch {}
}

interface Statements {
	readonly getMeta: StatementSync;
	readonly setMeta: StatementSync;
	readonly getRow: StatementSync;
	readonly rowLength: StatementSync;
	readonly upsertRow: StatementSync;
	readonly deleteRow: StatementSync;
	readonly rowsByEntity: StatementSync;
	readonly rowsByRange: StatementSync;
	readonly rowsFrom: StatementSync;
	readonly deleteEntity: StatementSync;
	readonly entityUsage: StatementSync;
	readonly allRows: StatementSync;
	readonly deleteAccountRows: StatementSync;
	readonly getCursor: StatementSync;
	readonly setCursor: StatementSync;
	readonly deleteCursor: StatementSync;
	readonly setSummary: StatementSync;
	readonly allSummaries: StatementSync;
	readonly deleteSummary: StatementSync;
	readonly insertMention: StatementSync;
	readonly pruneMentions: StatementSync;
	readonly listMentions: StatementSync;
	readonly deleteMentions: StatementSync;
	readonly countMentions: StatementSync;
	readonly accountUsage: StatementSync;
}

export class CacheDatabase {
	private readonly path: string;
	private readonly clock: EngineClock;
	private db: DatabaseSync;
	private statements: Statements;
	private readonly usage = new Map<string, CacheAccountUsage>();
	private readonly mentionCounts = new Map<string, number>();
	private inTransaction = false;
	private pendingOps = 0;
	private commitTimer: EngineTimer | null = null;
	private closed = false;
	readonly openReport: CacheOpenReport;

	constructor(path: string, snapshotSchemaEpoch: number, clock: EngineClock) {
		this.path = path;
		this.clock = clock;
		const schemaEpoch = String(ENGINE_CACHE_EPOCH * ACCOUNT_ENGINE_CORE_SCHEMA_EPOCH_OFFSET + snapshotSchemaEpoch);
		try {
			const {db, reset} = this.openFile(schemaEpoch);
			this.db = db;
			this.openReport = {reset, memoryFallback: null};
		} catch (error) {
			this.db = openMemoryDatabase(schemaEpoch);
			this.openReport = {reset: 'none', memoryFallback: error instanceof Error ? error.message : String(error)};
		}
		this.statements = this.prepareStatements();
	}

	getMeta(key: string): string | null {
		const row = this.statements.getMeta.get(key) as {value: string | null} | undefined;
		return row?.value ?? null;
	}

	setMeta(key: string, value: string): void {
		this.write(() => this.statements.setMeta.run(key, value));
	}

	usageFor(accountKey: string): CacheAccountUsage {
		const usage = this.mutableUsage(accountKey);
		return {rows: usage.rows, bytes: usage.bytes};
	}

	getRow(accountKey: string, entity: string, key: string): string | null {
		const row = this.statements.getRow.get(accountKey, entity, key) as {value: string} | undefined;
		return row?.value ?? null;
	}

	rowsByPrefix(accountKey: string, entity: string, prefix: string): Array<{key: string; value: string}> {
		const upper = prefixUpperBound(prefix);
		let rows: Array<Record<string, SQLOutputValue>>;
		if (prefix.length === 0) {
			rows = this.statements.rowsByEntity.all(accountKey, entity);
		} else if (upper === null) {
			rows = this.statements.rowsFrom.all(accountKey, entity, prefix);
		} else {
			rows = this.statements.rowsByRange.all(accountKey, entity, prefix, upper);
		}
		const result: Array<{key: string; value: string}> = [];
		for (const row of rows as Array<{key: string; value: string}>) {
			if (prefix.length === 0 || row.key.startsWith(prefix)) {
				result.push({key: row.key, value: row.value});
			}
		}
		return result;
	}

	upsertRow(accountKey: string, entity: string, key: string, value: string): void {
		const previous = this.statements.rowLength.get(accountKey, entity, key) as {size: number} | undefined;
		const usage = this.mutableUsage(accountKey);
		this.write(() => this.statements.upsertRow.run(accountKey, entity, key, value));
		if (previous === undefined) {
			usage.rows += 1;
			usage.bytes += Buffer.byteLength(value, 'utf8');
		} else {
			usage.bytes += Buffer.byteLength(value, 'utf8') - previous.size;
		}
	}

	deleteRow(accountKey: string, entity: string, key: string): void {
		const previous = this.statements.rowLength.get(accountKey, entity, key) as {size: number} | undefined;
		if (previous === undefined) {
			return;
		}
		const usage = this.mutableUsage(accountKey);
		this.write(() => this.statements.deleteRow.run(accountKey, entity, key));
		usage.rows -= 1;
		usage.bytes -= previous.size;
	}

	deleteRowsByPrefix(accountKey: string, entity: string, prefix: string): void {
		for (const row of this.rowsByPrefix(accountKey, entity, prefix)) {
			this.deleteRow(accountKey, entity, row.key);
		}
	}

	replaceEntity(accountKey: string, entity: string, entries: ReadonlyArray<readonly [string, string]>): void {
		const previous = this.statements.entityUsage.get(accountKey, entity) as {rows: number; bytes: number | null};
		const usage = this.mutableUsage(accountKey);
		this.write(() => {
			this.statements.deleteEntity.run(accountKey, entity);
			for (const [key, value] of entries) {
				this.statements.upsertRow.run(accountKey, entity, key, value);
			}
		}, entries.length + 1);
		usage.rows += entries.length - previous.rows;
		let bytes = 0;
		for (const [, value] of entries) {
			bytes += Buffer.byteLength(value, 'utf8');
		}
		usage.bytes += bytes - (previous.bytes ?? 0);
	}

	readCursor(accountKey: string): AccountEngineSnapshotCursor | null {
		const row = this.statements.getCursor.get(accountKey) as
			| {session_id: string | null; schema_epoch: number | null; updated_at: number}
			| undefined;
		if (row === undefined) {
			return null;
		}
		return {sessionId: row.session_id, schemaEpoch: row.schema_epoch, updatedAt: row.updated_at};
	}

	writeCursor(accountKey: string, cursor: AccountEngineSnapshotCursor): void {
		this.write(() =>
			this.statements.setCursor.run(accountKey, cursor.sessionId, cursor.schemaEpoch, Math.floor(cursor.updatedAt)),
		);
	}

	captureAccount(accountKey: string): AccountEngineSnapshotEntries {
		const entries: AccountEngineSnapshotEntries = {};
		for (const row of this.statements.allRows.iterate(accountKey) as Iterable<{
			entity: string;
			key: string;
			value: string;
		}>) {
			let entity = entries[row.entity];
			if (entity === undefined) {
				entity = {};
				entries[row.entity] = entity;
			}
			entity[row.key] = row.value;
		}
		return entries;
	}

	deleteAccountSnapshot(accountKey: string): void {
		this.write(() => {
			this.statements.deleteAccountRows.run(accountKey);
			this.statements.deleteCursor.run(accountKey);
		}, 2);
		this.usage.delete(accountKey);
	}

	purgeAccount(accountKey: string): void {
		this.write(() => {
			this.statements.deleteAccountRows.run(accountKey);
			this.statements.deleteCursor.run(accountKey);
			this.statements.deleteSummary.run(accountKey);
			this.statements.deleteMentions.run(accountKey);
		}, 4);
		this.usage.delete(accountKey);
		this.mentionCounts.delete(accountKey);
		this.commit();
		try {
			this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
		} catch {}
	}

	writeSummary(accountKey: string, json: string): void {
		this.write(() => this.statements.setSummary.run(accountKey, json, this.clock.now()));
	}

	readSummaries(): Map<string, string> {
		const summaries = new Map<string, string>();
		for (const row of this.statements.allSummaries.all() as Array<{account_key: string; json: string}>) {
			summaries.set(row.account_key, row.json);
		}
		return summaries;
	}

	insertMention(accountKey: string, messageId: string, createdAt: number, json: string, keep: number): void {
		this.write(() => this.statements.insertMention.run(accountKey, messageId, createdAt, json));
		const count = (this.mentionCounts.get(accountKey) ?? this.countMentions(accountKey) - 1) + 1;
		if (count <= keep + MENTION_PRUNE_SLACK) {
			this.mentionCounts.set(accountKey, count);
			return;
		}
		this.write(() => this.statements.pruneMentions.run(accountKey, accountKey, keep));
		this.mentionCounts.set(accountKey, keep);
	}

	listMentions(accountKey: string, limit: number): Array<string> {
		return (this.statements.listMentions.all(accountKey, limit) as Array<{json: string}>).map((row) => row.json);
	}

	private countMentions(accountKey: string): number {
		return (this.statements.countMentions.get(accountKey) as {count: number}).count;
	}

	knownAccountKeys(): Set<string> {
		const keys = new Set<string>();
		for (const query of [
			'SELECT account_key FROM cursors',
			'SELECT account_key FROM summaries',
			'SELECT DISTINCT account_key FROM mentions',
			'SELECT DISTINCT account_key FROM rows',
		]) {
			for (const row of this.db.prepare(query).all() as Array<{account_key: string}>) {
				keys.add(row.account_key);
			}
		}
		return keys;
	}

	commit(): void {
		if (this.commitTimer !== null) {
			this.clock.clearTimeout(this.commitTimer);
			this.commitTimer = null;
		}
		this.pendingOps = 0;
		if (!this.inTransaction) {
			return;
		}
		this.inTransaction = false;
		this.db.exec('COMMIT');
	}

	close(): void {
		if (this.closed) {
			return;
		}
		this.commit();
		this.closed = true;
		try {
			this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
		} catch {}
		this.db.close();
	}

	private write<T>(run: () => T, ops = 1): T {
		if (this.closed) {
			throw new Error('The account engine cache is closed');
		}
		if (!this.inTransaction) {
			this.db.exec('BEGIN');
			this.inTransaction = true;
		}
		const result = run();
		this.pendingOps += ops;
		if (this.pendingOps >= CACHE_COMMIT_MAX_OPS) {
			this.commit();
		} else if (this.commitTimer === null) {
			this.commitTimer = this.clock.setTimeout(() => {
				this.commitTimer = null;
				this.commit();
			}, CACHE_COMMIT_INTERVAL_MS);
		}
		return result;
	}

	private mutableUsage(accountKey: string): CacheAccountUsage {
		let usage = this.usage.get(accountKey);
		if (usage === undefined) {
			const stored = this.statements.accountUsage.get(accountKey) as {rows: number; bytes: number | null};
			usage = {rows: stored.rows, bytes: stored.bytes ?? 0};
			this.usage.set(accountKey, usage);
		}
		return usage;
	}

	private openFile(schemaEpoch: string): {db: DatabaseSync; reset: CacheOpenReport['reset']} {
		let reset: CacheOpenReport['reset'] = 'none';
		let db = this.tryOpen();
		if (db === null) {
			reset = 'corrupt';
			removeDatabaseFiles(this.path);
			db = this.tryOpen();
			if (db === null) {
				throw new Error(`The account engine cache at ${this.path} cannot be opened`);
			}
		}
		try {
			const storedEpoch = readStoredEpoch(db);
			if (storedEpoch !== schemaEpoch) {
				if (storedEpoch !== null) {
					reset = 'epoch';
					db.close();
					removeDatabaseFiles(this.path);
					db = this.tryOpen();
					if (db === null) {
						throw new Error(`The account engine cache at ${this.path} cannot be recreated`);
					}
				}
				writeStoredEpoch(db, schemaEpoch);
			}
		} catch (error) {
			closeQuietly(db);
			throw error;
		}
		return {db, reset};
	}

	private tryOpen(): DatabaseSync | null {
		let db: DatabaseSync | null = null;
		try {
			ensurePrivateFile(this.path);
			db = new DatabaseSync(this.path);
			db.exec('PRAGMA journal_mode=WAL');
			db.exec('PRAGMA synchronous=NORMAL');
			db.exec('PRAGMA cache_size=-65536');
			db.exec('PRAGMA busy_timeout=2000');
			db.exec('PRAGMA secure_delete=ON');
			const check = db.prepare('PRAGMA quick_check').all() as Array<Record<string, unknown>>;
			const verdict = check.length === 1 ? Object.values(check[0])[0] : null;
			if (verdict !== 'ok') {
				db.close();
				return null;
			}
			db.exec(SCHEMA);
			return db;
		} catch (error) {
			closeQuietly(db);
			if (!isRecord(error) || error.code !== 'ERR_SQLITE_ERROR' || typeof error.errcode !== 'number') {
				throw error;
			}
			const primaryCode = error.errcode & SQLITE_PRIMARY_RESULT_MASK;
			if (primaryCode !== SQLITE_CORRUPT && primaryCode !== SQLITE_NOTADB) {
				throw error;
			}
			return null;
		}
	}

	private prepareStatements(): Statements {
		const db = this.db;
		return {
			getMeta: db.prepare('SELECT value FROM meta WHERE key = ?'),
			setMeta: db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)'),
			getRow: db.prepare('SELECT value FROM rows WHERE account_key = ? AND entity = ? AND key = ?'),
			rowLength: db.prepare(
				'SELECT octet_length(value) AS size FROM rows WHERE account_key = ? AND entity = ? AND key = ?',
			),
			upsertRow: db.prepare('INSERT OR REPLACE INTO rows(account_key, entity, key, value) VALUES (?, ?, ?, ?)'),
			deleteRow: db.prepare('DELETE FROM rows WHERE account_key = ? AND entity = ? AND key = ?'),
			rowsByEntity: db.prepare('SELECT key, value FROM rows WHERE account_key = ? AND entity = ?'),
			rowsByRange: db.prepare(
				'SELECT key, value FROM rows WHERE account_key = ? AND entity = ? AND key >= ? AND key < ?',
			),
			rowsFrom: db.prepare('SELECT key, value FROM rows WHERE account_key = ? AND entity = ? AND key >= ?'),
			deleteEntity: db.prepare('DELETE FROM rows WHERE account_key = ? AND entity = ?'),
			entityUsage: db.prepare(
				'SELECT count(*) AS rows, total(octet_length(value)) AS bytes FROM rows WHERE account_key = ? AND entity = ?',
			),
			allRows: db.prepare('SELECT entity, key, value FROM rows WHERE account_key = ?'),
			deleteAccountRows: db.prepare('DELETE FROM rows WHERE account_key = ?'),
			getCursor: db.prepare('SELECT session_id, schema_epoch, updated_at FROM cursors WHERE account_key = ?'),
			setCursor: db.prepare(
				'INSERT OR REPLACE INTO cursors(account_key, session_id, schema_epoch, updated_at) VALUES (?, ?, ?, ?)',
			),
			deleteCursor: db.prepare('DELETE FROM cursors WHERE account_key = ?'),
			setSummary: db.prepare('INSERT OR REPLACE INTO summaries(account_key, json, updated_at) VALUES (?, ?, ?)'),
			allSummaries: db.prepare('SELECT account_key, json FROM summaries'),
			deleteSummary: db.prepare('DELETE FROM summaries WHERE account_key = ?'),
			insertMention: db.prepare(
				'INSERT OR REPLACE INTO mentions(account_key, message_id, created_at, json) VALUES (?, ?, ?, ?)',
			),
			pruneMentions: db.prepare(
				'DELETE FROM mentions WHERE account_key = ? AND message_id NOT IN (SELECT message_id FROM mentions WHERE account_key = ? ORDER BY created_at DESC, message_id DESC LIMIT ?)',
			),
			listMentions: db.prepare(
				'SELECT json FROM mentions WHERE account_key = ? ORDER BY created_at DESC, message_id DESC LIMIT ?',
			),
			deleteMentions: db.prepare('DELETE FROM mentions WHERE account_key = ?'),
			countMentions: db.prepare('SELECT count(*) AS count FROM mentions WHERE account_key = ?'),
			accountUsage: db.prepare(
				'SELECT count(*) AS rows, total(octet_length(value)) AS bytes FROM rows WHERE account_key = ?',
			),
		};
	}
}