Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Load SQLite lazily in edge services and update every consumer (#981)

Dee7946c9804b96
diff --git a/cli/src/engines/cursor/home.spec.ts b/cli/src/engines/cursor/home.spec.ts
index aa0ee3718..2f302b694 100644
--- a/cli/src/engines/cursor/home.spec.ts
+++ b/cli/src/engines/cursor/home.spec.ts
@@ -6,7 +6,7 @@ import { cursorConfigDir, cursorDataDir } from './home.js'
import { CursorTranscriptDiscovery } from './discovery.js'
import { loadCursorReplayTaskLinks } from './subagent.js'
import { cleanupCursorOneShotSession } from '../../lib/oneshot.js'
-import { builtinSqlite } from '../../lib/sqliteRead.js'
+import { builtinSqlite } from '../../lib/sqliteBuiltin.js'
const parent = 'aaaaaaaa-1111-4222-8333-444444444444'
const child = 'bbbbbbbb-1111-4222-8333-444444444444'
diff --git a/cli/src/leanEntry.spec.ts b/cli/src/leanEntry.spec.ts
index 9c3b2d526..6182dc108 100644
--- a/cli/src/leanEntry.spec.ts
+++ b/cli/src/leanEntry.spec.ts
@@ -5,8 +5,9 @@
* none of the CLI's commands, and the master, search and workspaces no zod (`config/env.ts` and `lib/registry.ts` once brought it to every
* one of them, 8 to 19 MiB each, measured 2026-10-05).
*/
-import { execFileSync, spawnSync } from 'node:child_process'
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { execFileSync, spawn, spawnSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { createServer } from 'node:net'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
@@ -53,6 +54,8 @@ describe('the lean bundle a release carries', () => {
}
const runnerOf = (name: string): string | undefined => Object.entries(RUNNER).find(([, runner]) => name.startsWith(`${runner}-`))?.[0]
const hasZod = (names: Set<string>): boolean => [...names].some((name) => files.get(name)!.includes('$ZodType'))
+ /** The binding's loader: the module `lib/sqliteBuiltin.ts` asks Node for, a string only that code holds. */
+ const hasSqlite = (names: Set<string>): boolean => [...names].some((name) => files.get(name)!.includes('"node:sqlite"'))
it('is files of their own for the master, the core and each service, the code it runs and no other role\'s', () => {
for (const role of ROLES) {
@@ -106,6 +109,50 @@ describe('the lean bundle a release carries', () => {
for (const role of ['master', 'search', 'updater', ...SERVICE_HOSTS.edge.services]) expect(hasZod(loads(role)), role).toBe(false)
})
+ it('brings the edge host no node:sqlite until it reads a store some engines keep a conversation in', () => {
+ // A native binding, synchronous and able to take a process down with it, in the host meant to be light.
+ // The handoff, the monitor and projects read such stores for opencode, kilo, hermes and devin alone, and
+ // `lib/sqliteRead.ts` imports the binding's loader (`lib/sqliteBuiltin.ts`) only at the first read.
+ expect(hasSqlite(loads('search')), 'search, the native-sqlite host, still has it').toBe(true)
+ for (const role of SERVICE_HOSTS.edge.services) expect(hasSqlite(loads(role)), role).toBe(false)
+ })
+
+ it('starts the edge host without node:sqlite, where search, whose index is SQLite, has it', async () => {
+ // Short: the services' socket is `<data>/daemon-<port>.sock`, 96 bytes at most (lib/localSocket.ts).
+ const home = mkdtempSync(join(tmpdir(), 'le-'))
+ // Each service dials the core once it has started: a socket that counts them, then asks what loaded.
+ const loadedOnce = (service: string, names: readonly string[]): Promise<string[]> => new Promise((resolve, reject) => {
+ const data = join(home, service)
+ mkdirSync(join(data, 'tmux'), { recursive: true })
+ let dialled = 0
+ const core = createServer((socket) => {
+ socket.on('error', () => {})
+ if (++dialled === names.length) child.kill('SIGUSR2')
+ }).listen(join(data, 'daemon-18999.sock'))
+ const report = 'data:text/javascript,process.on("SIGUSR2",()=>{process.stdout.write("LOADED="+JSON.stringify(process.moduleLoadList.filter((m)=>/sqlite/.test(m)))+"\\n");process.exit(0)})'
+ // Nothing of this machine's: its own home, data folder and tmux server.
+ const child = spawn(process.execPath, ['--import', report, entry, '__service', names.join(',')], {
+ env: { PATH: process.env.PATH, HOME: data, ADAPTER_DATA_DIR: data, PORT: '18999', TMUX_TMPDIR: join(data, 'tmux'), HARNESSD_SERVICE: service },
+ stdio: ['ignore', 'pipe', 'pipe'],
+ })
+ let out = ''
+ child.stdout.on('data', (chunk: Buffer) => { out += chunk.toString() })
+ child.stderr.on('data', (chunk: Buffer) => { out += chunk.toString() })
+ const timer = setTimeout(() => child.kill('SIGKILL'), 60_000)
+ child.on('close', () => {
+ clearTimeout(timer)
+ core.close()
+ const loaded = /^LOADED=(.*)$/m.exec(out)
+ if (loaded) resolve(JSON.parse(loaded[1]) as string[])
+ else reject(new Error(`${service} ended before it said what it loaded (${dialled} of ${names.length} dialled):\n${out}`))
+ })
+ })
+ try {
+ expect(await loadedOnce('search', ['search']), 'search: what tells a loaded binding apart').toContain('NativeModule sqlite')
+ expect(await loadedOnce('edge', SERVICE_HOSTS.edge.services), 'the edge host').toEqual([])
+ } finally { rmSync(home, { recursive: true, force: true }) }
+ }, 150_000)
+
it('passes the release script\'s check, which a lean bundle that breaks a process does not', async () => {
// @ts-expect-error — plain ESM with no declaration file
const { leanBlock } = await import('../scripts/lib/leanBlock.mjs') as { leanBlock: (files: Record<string, string>) => string }
diff --git a/cli/src/lib/agentHandoff.ts b/cli/src/lib/agentHandoff.ts
index 5d4320282..2472afe9b 100644
--- a/cli/src/lib/agentHandoff.ts
+++ b/cli/src/lib/agentHandoff.ts
@@ -40,7 +40,7 @@ import { redactSecretsInText } from './logBundle.js'
import type { AgentEngine } from '../engines/types.js'
import type { LiveEvent } from './normalize.js'
import type { RegisteredSession } from './registry.js'
-import { SQLITE_BACKED_ENGINES } from './sqliteAvailability.js'
+import { SQLITE_BACKED_ENGINES } from './sqliteRead.js'
import { omitLongRuns, readSessionTurns, SessionTurnsError, type TurnSource } from './sessionSearch/sessionTurns.js'
import type { IndexedTurn } from './sessionSearch/turns.js'
import { isSubagentTranscript } from './subagentTranscript.js'
diff --git a/cli/src/lib/handoffDiscovery.ts b/cli/src/lib/handoffDiscovery.ts
index 37f72671c..3dfac7961 100644
--- a/cli/src/lib/handoffDiscovery.ts
+++ b/cli/src/lib/handoffDiscovery.ts
@@ -9,7 +9,7 @@ import type { RecentRecap } from './recapReads.js'
import type { RegisteredSession } from './registry.js'
import type { RepairedSession } from './sessionRepair.js'
import type { TurnSource } from './sessionSearch/sessionTurns.js'
-import { SQLITE_BACKED_ENGINES } from './sqliteAvailability.js'
+import { SQLITE_BACKED_ENGINES } from './sqliteRead.js'
export interface DiscoveryDeps {
/** Engines other than Claude: the born, unique session of a process (sessionRepair findLiveSession). */
diff --git a/cli/src/lib/purgeAgentService.spec.ts b/cli/src/lib/purgeAgentService.spec.ts
index b956d05ac..96468d8d6 100644
--- a/cli/src/lib/purgeAgentService.spec.ts
+++ b/cli/src/lib/purgeAgentService.spec.ts
@@ -7,7 +7,7 @@ import { registry, type RegisteredSession } from './registry.js'
import { StoppedAgentStore } from './stoppedAgents.js'
import { SessionCheckpointStore } from './sessionCheckpoint.js'
import { PurgeAgentService, inspectNativeHistory, eraseNativeHistory, sessionDataBytes, type PurgeRequest } from './purgeAgentService.js'
-import { builtinSqlite } from './sqliteRead.js'
+import { builtinSqlite } from './sqliteBuiltin.js'
import type { StopAgentOptions } from './stopAgentService.js'
import * as worktrees from './worktreeDeletion.js'
diff --git a/cli/src/lib/purgeAgentService.ts b/cli/src/lib/purgeAgentService.ts
index a40ca03bc..7b581b1cd 100644
--- a/cli/src/lib/purgeAgentService.ts
+++ b/cli/src/lib/purgeAgentService.ts
@@ -7,7 +7,7 @@ import { basename, dirname, join, resolve, sep } from 'node:path'
import { env } from '../config/env.js'
import { hermesDbPath } from '../engines/hermes/home.js'
import { engineKeepsTranscriptFile, validTranscriptPath, type RegisteredSession } from './registry.js'
-import { builtinSqlite, sqliteReadAll } from './sqliteRead.js'
+import { sqliteReadAll } from './sqliteRead.js'
import type { StoppedAgentStore } from './stoppedAgents.js'
import type { SessionCheckpointStore } from './sessionCheckpoint.js'
import type { StopAgentOptions } from './stopAgentService.js'
@@ -77,6 +77,8 @@ export async function inspectNativeHistory(s: RegisteredSession): Promise<Histor
sessionTable = 'session_v2'; children = [['session_message', `session_id = ${id}`]]
}
await query(`SELECT id FROM ${sessionTable} WHERE id = ${id}`)
+ // The binding, only for a purge of a store-backed engine's history: the monitor's process imports this module.
+ const { builtinSqlite } = await import('./sqliteBuiltin.js')
if (!builtinSqlite()) await exec('sqlite3', ['--version'], { timeout: 2500, maxBuffer: 4096 })
const allowedTables = new Set([sessionTable, ...children.map(([table]) => table)])
if ((await query("SELECT name, tbl_name FROM sqlite_master WHERE type='trigger'")).some(t => allowedTables.has(String(t.tbl_name)))) {
@@ -130,7 +132,7 @@ export async function eraseNativeHistory(history: History): Promise<number> {
const current = file(history.database)
if (current.dev !== history.databaseFile?.dev || current.ino !== history.databaseFile?.ino) fail('The engine history store changed. Review deletion again.')
const sql = ['PRAGMA busy_timeout=1000;', 'PRAGMA foreign_keys=ON;', 'BEGIN IMMEDIATE;', ...history.statements!, 'COMMIT;'].join('\n')
- const Database = builtinSqlite()
+ const Database = (await import('./sqliteBuiltin.js')).builtinSqlite()
if (Database) {
const db = new Database(history.database, { readOnly: false })
try { db.exec(sql) } finally { db.close() }
diff --git a/cli/src/lib/sessionCheckpoint.sqlite.spec.ts b/cli/src/lib/sessionCheckpoint.sqlite.spec.ts
index a561b5fcb..3ef2c9656 100644
--- a/cli/src/lib/sessionCheckpoint.sqlite.spec.ts
+++ b/cli/src/lib/sessionCheckpoint.sqlite.spec.ts
@@ -12,7 +12,7 @@ beforeEach(() => {
for (const key of ['OPENCODE_DATA_DIR', 'KILO_DATA_DIR', 'HERMES_HOME', 'DEVIN_HOME']) vi.stubEnv(key, root)
})
afterEach(async () => {
- const { closeSqliteHandles } = await import('./sqliteRead.js')
+ const { closeSqliteHandles } = await import('./sqliteBuiltin.js')
closeSqliteHandles()
for (const database of databases.splice(0)) database.close()
vi.unstubAllEnvs()
diff --git a/cli/src/lib/sessionRepair.ts b/cli/src/lib/sessionRepair.ts
index bc4cfc013..6a6af311e 100644
--- a/cli/src/lib/sessionRepair.ts
+++ b/cli/src/lib/sessionRepair.ts
@@ -24,7 +24,6 @@ import { readCodexRolloutMeta, resolveCodexRollout } from '../engines/codex/roll
import { agyConversationForPid, findAgyTranscript } from '../engines/agy/session.js'
import { copilotSessionCwd, copilotSessionForPid, findCopilotTranscript } from '../engines/copilot/session.js'
import { hermesDbPath, listHermesHomes } from '../engines/hermes/home.js'
-import { sqlitePreflightMessage } from './sqliteAvailability.js'
import { sqliteReadAll, type SqliteParam } from './sqliteRead.js'
import { piSessionFolder, readPiHead } from './sessionSearch/externals/pi.js'
import { claudeProjectsRoots, codexHomeRoots } from './engineHomes.js'
@@ -227,9 +226,10 @@ async function fileEngineSession(
let missingSqliteReported = false
/** The store-backed repair branch cannot work without a SQLite reader; warn on the first miss only. */
-function reportMissingSqliteOnce(): void {
+async function reportMissingSqliteOnce(): Promise<void> {
if (missingSqliteReported) return
missingSqliteReported = true
+ const { sqlitePreflightMessage } = await import('./sqliteAvailability.js')
console.warn(sqlitePreflightMessage() ?? '[preflight] no SQLite reader available')
}
@@ -239,7 +239,7 @@ async function dbEngineSession(dbPath: string, sql: string, params: SqliteParam[
if (!result.ok) {
// No reader at all is not a transient DB lock, and repair returning null forever with no signal is
// how "my opencode agents never appear on Ubuntu" looks from the outside. Say it once.
- if (result.reason === 'missing') reportMissingSqliteOnce()
+ if (result.reason === 'missing') await reportMissingSqliteOnce()
return null
}
const rows = result.rows
diff --git a/cli/src/lib/sessionSearch/externals/cursor.spec.ts b/cli/src/lib/sessionSearch/externals/cursor.spec.ts
index 217cac869..5b674c19b 100644
--- a/cli/src/lib/sessionSearch/externals/cursor.spec.ts
+++ b/cli/src/lib/sessionSearch/externals/cursor.spec.ts
@@ -4,7 +4,7 @@ import { basename, dirname, join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
-import { builtinSqlite } from '../../sqliteRead.js'
+import { builtinSqlite } from '../../sqliteBuiltin.js'
import { cursorBucket, cursorProvider, cursorSlug, cursorTurnOpen, readChat, type CursorDatabase } from './cursor.js'
import { entries, scanMemo } from './support.js'
import type { ProcessView, RunningProcess, ScanContext } from './types.js'
diff --git a/cli/src/lib/sessionSearch/externals/cursor.ts b/cli/src/lib/sessionSearch/externals/cursor.ts
index 93abaa35c..dc1cab832 100644
--- a/cli/src/lib/sessionSearch/externals/cursor.ts
+++ b/cli/src/lib/sessionSearch/externals/cursor.ts
@@ -25,7 +25,7 @@ import { basename, dirname, join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { agentCommandOwnershipSnapshot } from '../../engineBin.js'
-import { builtinSqlite } from '../../sqliteRead.js'
+import { builtinSqlite } from '../../sqliteBuiltin.js'
import { argvTokens, engineProcessMatch, resumeSessionId } from '../../tmux.js'
import { absoluteFolder, entries, fileStamp, parseLine, readJson, readTail, record, text, UUID } from './support.js'
import type { ExternalProvider, ExternalSession, OwnerClaim, ProcessView, RunningProcess, ScanContext } from './types.js'
diff --git a/cli/src/lib/sessionSearch/externals/devin.spec.ts b/cli/src/lib/sessionSearch/externals/devin.spec.ts
index c2f56d193..4b2e31822 100644
--- a/cli/src/lib/sessionSearch/externals/devin.spec.ts
+++ b/cli/src/lib/sessionSearch/externals/devin.spec.ts
@@ -11,7 +11,7 @@ import { join } from 'node:path'
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
-import { builtinSqlite, closeSqliteHandles, overrideBuiltinSqlite } from '../../sqliteRead.js'
+import { builtinSqlite, closeSqliteHandles, overrideBuiltinSqlite } from '../../sqliteBuiltin.js'
import { devinListSql, devinProvider, devinTurnOpen, lockPid } from './devin.js'
import { LIST_LIMIT, ownerRecord, readSql, type SqlRead } from './opencode.js'
import { scanMemo } from './support.js'
diff --git a/cli/src/lib/sessionSearch/externals/hermes.spec.ts b/cli/src/lib/sessionSearch/externals/hermes.spec.ts
index f5b0435e1..cb702f6d8 100644
--- a/cli/src/lib/sessionSearch/externals/hermes.spec.ts
+++ b/cli/src/lib/sessionSearch/externals/hermes.spec.ts
@@ -10,7 +10,7 @@ import { join } from 'node:path'
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
-import { builtinSqlite, closeSqliteHandles, overrideBuiltinSqlite } from '../../sqliteRead.js'
+import { builtinSqlite, closeSqliteHandles, overrideBuiltinSqlite } from '../../sqliteBuiltin.js'
import {
activeProfile, argvProfile, bestContinuation, compressionTip, hermesChains, hermesHomes, hermesKind, hermesLeases,
hermesListSql, hermesProvider, hermesRow, hermesTurnOpen, leaseHeldBy, type HermesRow, type HermesSession,
diff --git a/cli/src/lib/sessionSearch/externals/opencode.spec.ts b/cli/src/lib/sessionSearch/externals/opencode.spec.ts
index 6b3a8ac6a..716ef5ee4 100644
--- a/cli/src/lib/sessionSearch/externals/opencode.spec.ts
+++ b/cli/src/lib/sessionSearch/externals/opencode.spec.ts
@@ -11,7 +11,7 @@ import { join } from 'node:path'
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
-import { builtinSqlite, closeSqliteHandles, overrideBuiltinSqlite } from '../../sqliteRead.js'
+import { builtinSqlite, closeSqliteHandles, overrideBuiltinSqlite } from '../../sqliteBuiltin.js'
import {
LIST_LIMIT, ancestry, argvSubcommand, engineProcess, opencodeListSql, opencodeProvider, opencodeTurnOpen,
ownerRecord, parseOwnerRecord, readSql, rowsOf, splitCounts, storeStamp, tableColumns, tally, type SqlRead,
diff --git a/cli/src/lib/sessionSearch/store.ts b/cli/src/lib/sessionSearch/store.ts
index a3cde463b..95e09b8b7 100644
--- a/cli/src/lib/sessionSearch/store.ts
+++ b/cli/src/lib/sessionSearch/store.ts
@@ -13,7 +13,7 @@
import { chmodSync, existsSync, rmSync } from 'node:fs'
-import { builtinSqlite } from '../sqliteRead.js'
+import { builtinSqlite } from '../sqliteBuiltin.js'
import type { IndexedTurn } from './turns.js'
// Rebuild schema 10's external-engine index with Codex app/editor context removed from asks.
diff --git a/cli/src/lib/sqliteAvailability.spec.ts b/cli/src/lib/sqliteAvailability.spec.ts
index 9a9345c12..b3d0c7065 100644
--- a/cli/src/lib/sqliteAvailability.spec.ts
+++ b/cli/src/lib/sqliteAvailability.spec.ts
@@ -3,7 +3,7 @@ import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { delimiter, join } from 'node:path'
import { hasSqliteCli, hasSqliteReader, resetSqliteAvailabilityCache, sqlitePreflightMessage } from './sqliteAvailability.js'
-import { overrideBuiltinSqlite } from './sqliteRead.js'
+import { overrideBuiltinSqlite } from './sqliteBuiltin.js'
const dirs: string[] = []
const realPath = process.env.PATH
diff --git a/cli/src/lib/sqliteAvailability.ts b/cli/src/lib/sqliteAvailability.ts
index 7ae35b4ef..b41e098d7 100644
--- a/cli/src/lib/sqliteAvailability.ts
+++ b/cli/src/lib/sqliteAvailability.ts
@@ -13,10 +13,8 @@
* store rewrite in `sessionModel.ts`, a write the built-in binding is not used for).
*/
import { binaryOnPath } from './binaryOnPath.js'
-import { builtinSqlite } from './sqliteRead.js'
-
-/** Engines that cannot be mirrored without the CLI. Keep in sync with the readers listed above. */
-export const SQLITE_BACKED_ENGINES = ['opencode', 'kilo', 'hermes', 'devin'] as const
+import { builtinSqlite } from './sqliteBuiltin.js'
+import { SQLITE_BACKED_ENGINES } from './sqliteRead.js'
let cached: boolean | null = null
diff --git a/cli/src/lib/sqliteBuiltin.ts b/cli/src/lib/sqliteBuiltin.ts
new file mode 100644
index 000000000..1e52e8edf
--- /dev/null
+++ b/cli/src/lib/sqliteBuiltin.ts
@@ -0,0 +1,109 @@
+/** The native `node:sqlite` binding, apart from `sqliteRead.ts`, which imports it at its first read: the edge host
+ * loads it only to read a store opencode, kilo, hermes or devin keeps, never for Claude Code or Codex. */
+import { statSync } from 'node:fs'
+import { pathToFileURL } from 'node:url'
+import { idleWalStore, type SqliteParam, type SqliteReadResult, type SqliteRow } from './sqliteRead.js'
+
+interface StatementLike { all(...params: SqliteParam[]): SqliteRow[] }
+export interface DatabaseLike {
+ prepare(sql: string): StatementLike
+ exec(sql: string): void
+ close(): void
+}
+export interface DatabaseConstructor { new (path: string | URL, options: { readOnly: boolean }): DatabaseLike }
+
+let builtin: DatabaseConstructor | null | undefined
+
+/**
+ * `node:sqlite` when this Node has it, else null. Resolved through `process.getBuiltinModule` (Node
+ * ≥ 20.16 / 22.3) rather than `import()`: synchronous, and invisible to the bundler, which must not
+ * try to resolve a module that only some runtimes have.
+ *
+ * The binding announces itself with an ExperimentalWarning on first load. That line would land in
+ * `harness.log` at every daemon boot, so it is filtered — only that one warning, and every other
+ * listener is kept.
+ */
+export function builtinSqlite(): DatabaseConstructor | null {
+ if (builtin !== undefined) return builtin
+ builtin = null
+ const get = (process as { getBuiltinModule?: (id: string) => unknown }).getBuiltinModule
+ if (typeof get !== 'function') return builtin
+ const previous = process.listeners('warning')
+ process.removeAllListeners('warning')
+ process.on('warning', (warning) => {
+ if (warning.name === 'ExperimentalWarning' && /SQLite/i.test(warning.message)) return
+ for (const listener of previous) listener.call(process, warning)
+ })
+ try {
+ const mod = get.call(process, 'node:sqlite') as { DatabaseSync?: unknown } | undefined
+ if (mod && typeof mod.DatabaseSync === 'function') builtin = mod.DatabaseSync as DatabaseConstructor
+ } catch {
+ builtin = null
+ }
+ return builtin
+}
+
+interface Handle { db: DatabaseLike; dev: number; ino: number; immutable: boolean; stamp: string }
+
+/** One open handle per store: the readers poll every second, and opening is the expensive part. */
+const handles = new Map<string, Handle>()
+
+/**
+ * The cached handle, or a fresh one. Keyed by path but checked by inode: an engine that rebuilds its
+ * store (a reinstall, a `rm` and restart) leaves a handle on the OLD file, which would keep answering
+ * from a database nobody writes to any more — a poller that never sees another row, with no error.
+ */
+function openHandle(Database: DatabaseConstructor, dbPath: string, busyTimeoutMs: number): DatabaseLike {
+ const { dev, ino, size, mtimeMs, ctimeMs } = statSync(dbPath)
+ const stamp = `${size}:${mtimeMs}:${ctimeMs}`
+ // An idle store is read immutable; once its engine opens it again (a `-wal` appears), it is read live.
+ const immutable = idleWalStore(dbPath)
+ const cached = handles.get(dbPath)
+ // A writer can open, checkpoint, and close between polls, leaving no WAL. An immutable handle
+ // never checks for writes itself, so reuse it only while the main file's fingerprint holds.
+ if (cached && cached.dev === dev && cached.ino === ino && cached.immutable === immutable
+ && (!immutable || cached.stamp === stamp)) return cached.db
+ if (cached) dropHandle(dbPath)
+ const url = pathToFileURL(dbPath)
+ url.search = 'immutable=1'
+ const db = new Database(immutable ? url : dbPath, { readOnly: true })
+ // Not a write: the pragma is per-connection state, accepted on a read-only handle.
+ db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.trunc(busyTimeoutMs))}`)
+ handles.set(dbPath, { db, dev, ino, immutable, stamp })
+ return db
+}
+
+function dropHandle(dbPath: string): void {
+ const handle = handles.get(dbPath)
+ handles.delete(dbPath)
+ try { handle?.db.close() } catch { /* already gone */ }
+}
+
+/** Test seam, and for a store that was replaced on disk: forget every open handle. */
+export function closeSqliteHandles(): void {
+ for (const path of [...handles.keys()]) dropHandle(path)
+}
+
+export function readBuiltin(
+ Database: DatabaseConstructor,
+ dbPath: string,
+ sql: string,
+ params: SqliteParam[],
+ busyTimeoutMs: number,
+): SqliteReadResult {
+ try {
+ const db = openHandle(Database, dbPath, busyTimeoutMs)
+ return { ok: true, rows: db.prepare(sql).all(...params), via: 'builtin' }
+ } catch (err) {
+ // Locked, mid-checkpoint, the file replaced underneath the handle, not a database at all: every one
+ // of these is "try again later" for a poller, and a handle that failed is not trusted again — the
+ // next call reopens from the path.
+ dropHandle(dbPath)
+ return { ok: false, reason: 'transient', error: err instanceof Error ? err : new Error(String(err)) }
+ }
+}
+
+/** Test seam: force the CLI path (or the built-in one) regardless of what this Node has. */
+export function overrideBuiltinSqlite(value: DatabaseConstructor | null | undefined): void {
+ builtin = value
+}
diff --git a/cli/src/lib/sqliteRead.spec.ts b/cli/src/lib/sqliteRead.spec.ts
index 4030c0f8b..1de252cac 100644
--- a/cli/src/lib/sqliteRead.spec.ts
+++ b/cli/src/lib/sqliteRead.spec.ts
@@ -3,9 +3,8 @@ import { execFileSync } from 'node:child_process'
import { chmodSync, existsSync, mkdtempSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
-import {
- builtinSqlite, closeSqliteHandles, idleWalStore, inlineSqlParams, overrideBuiltinSqlite, sqliteReadAll,
-} from './sqliteRead.js'
+import { builtinSqlite, closeSqliteHandles, overrideBuiltinSqlite } from './sqliteBuiltin.js'
+import { idleWalStore, inlineSqlParams, sqliteReadAll } from './sqliteRead.js'
const hasCli = (() => {
try { execFileSync('sqlite3', ['-version'], { stdio: 'ignore' }); return true } catch { return false }
diff --git a/cli/src/lib/sqliteRead.ts b/cli/src/lib/sqliteRead.ts
index 17a885d76..ed663facd 100644
--- a/cli/src/lib/sqliteRead.ts
+++ b/cli/src/lib/sqliteRead.ts
@@ -4,7 +4,7 @@
*
* Two ways in, tried in this order:
*
- * 1. `node:sqlite` — built into the Node the installers ship (22.23, see `runtime/current-node`),
+ * 1. `node:sqlite` (`sqliteBuiltin.ts`, imported at the first read) — built into the Node the installers ship (22.23, see `runtime/current-node`),
* and into any Node ≥ 22.13 without a flag. In-process, parameterised, no JSON round-trip: the
* query that hung the `sqlite3` CLI for good on one machine (an opencode session with a ~7.5 MB
* row) finishes here in tens of milliseconds. The binding is SYNCHRONOUS — it blocks the event
@@ -21,8 +21,7 @@
*/
import { execFile } from 'node:child_process'
-import { closeSync, existsSync, openSync, readSync, statSync } from 'node:fs'
-import { pathToFileURL } from 'node:url'
+import { closeSync, existsSync, openSync, readSync } from 'node:fs'
import { promisify } from 'node:util'
const execFileAsync = promisify(execFile)
@@ -44,20 +43,13 @@ export type SqliteReadResult =
/** `missing`: no way to read SQLite on this machine. `transient`: locked, mid-write, unreadable — retry. */
| { ok: false; reason: 'missing' | 'transient'; error?: Error }
+/** The engines that keep a conversation in a store; here, not beside the binding, for the handoff. */
+export const SQLITE_BACKED_ENGINES = ['opencode', 'kilo', 'hermes', 'devin'] as const
+
const DEFAULT_BUSY_TIMEOUT_MS = 250
const DEFAULT_CLI_TIMEOUT_MS = 5_000
const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024
-// ---- built-in binding ---------------------------------------------------------------------------
-
-interface StatementLike { all(...params: SqliteParam[]): SqliteRow[] }
-interface DatabaseLike {
- prepare(sql: string): StatementLike
- exec(sql: string): void
- close(): void
-}
-interface DatabaseConstructor { new (path: string | URL, options: { readOnly: boolean }): DatabaseLike }
-
/**
* A WAL store its engine is not using right now: its header says WAL (file format versions 2) and no
* `-wal` file is beside it. Opening one read-only would CREATE `-wal` and `-shm` in the engine's
@@ -80,97 +72,6 @@ export function idleWalStore(dbPath: string): boolean {
}
}
-let builtin: DatabaseConstructor | null | undefined
-
-/**
- * `node:sqlite` when this Node has it, else null. Resolved through `process.getBuiltinModule` (Node
- * ≥ 20.16 / 22.3) rather than `import()`: synchronous, and invisible to the bundler, which must not
- * try to resolve a module that only some runtimes have.
- *
- * The binding announces itself with an ExperimentalWarning on first load. That line would land in
- * `harness.log` at every daemon boot, so it is filtered — only that one warning, and every other
- * listener is kept.
- */
-export function builtinSqlite(): DatabaseConstructor | null {
- if (builtin !== undefined) return builtin
- builtin = null
- const get = (process as { getBuiltinModule?: (id: string) => unknown }).getBuiltinModule
- if (typeof get !== 'function') return builtin
- const previous = process.listeners('warning')
- process.removeAllListeners('warning')
- process.on('warning', (warning) => {
- if (warning.name === 'ExperimentalWarning' && /SQLite/i.test(warning.message)) return
- for (const listener of previous) listener.call(process, warning)
- })
- try {
- const mod = get.call(process, 'node:sqlite') as { DatabaseSync?: unknown } | undefined
- if (mod && typeof mod.DatabaseSync === 'function') builtin = mod.DatabaseSync as DatabaseConstructor
- } catch {
- builtin = null
- }
- return builtin
-}
-
-interface Handle { db: DatabaseLike; dev: number; ino: number; immutable: boolean; stamp: string }
-
-/** One open handle per store: the readers poll every second, and opening is the expensive part. */
-const handles = new Map<string, Handle>()
-
-/**
- * The cached handle, or a fresh one. Keyed by path but checked by inode: an engine that rebuilds its
- * store (a reinstall, a `rm` and restart) leaves a handle on the OLD file, which would keep answering
- * from a database nobody writes to any more — a poller that never sees another row, with no error.
- */
-function openHandle(Database: DatabaseConstructor, dbPath: string, busyTimeoutMs: number): DatabaseLike {
- const { dev, ino, size, mtimeMs, ctimeMs } = statSync(dbPath)
- const stamp = `${size}:${mtimeMs}:${ctimeMs}`
- // An idle store is read immutable; once its engine opens it again (a `-wal` appears), it is read live.
- const immutable = idleWalStore(dbPath)
- const cached = handles.get(dbPath)
- // A writer can open, checkpoint, and close between polls, leaving no WAL. An immutable handle
- // never checks for writes itself, so reuse it only while the main file's fingerprint holds.
- if (cached && cached.dev === dev && cached.ino === ino && cached.immutable === immutable
- && (!immutable || cached.stamp === stamp)) return cached.db
- if (cached) dropHandle(dbPath)
- const url = pathToFileURL(dbPath)
- url.search = 'immutable=1'
- const db = new Database(immutable ? url : dbPath, { readOnly: true })
- // Not a write: the pragma is per-connection state, accepted on a read-only handle.
- db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.trunc(busyTimeoutMs))}`)
- handles.set(dbPath, { db, dev, ino, immutable, stamp })
- return db
-}
-
-function dropHandle(dbPath: string): void {
- const handle = handles.get(dbPath)
- handles.delete(dbPath)
- try { handle?.db.close() } catch { /* already gone */ }
-}
-
-/** Test seam, and for a store that was replaced on disk: forget every open handle. */
-export function closeSqliteHandles(): void {
- for (const path of [...handles.keys()]) dropHandle(path)
-}
-
-function readBuiltin(
- Database: DatabaseConstructor,
- dbPath: string,
- sql: string,
- params: SqliteParam[],
- busyTimeoutMs: number,
-): SqliteReadResult {
- try {
- const db = openHandle(Database, dbPath, busyTimeoutMs)
- return { ok: true, rows: db.prepare(sql).all(...params), via: 'builtin' }
- } catch (err) {
- // Locked, mid-checkpoint, the file replaced underneath the handle, not a database at all: every one
- // of these is "try again later" for a poller, and a handle that failed is not trusted again — the
- // next call reopens from the path.
- dropHandle(dbPath)
- return { ok: false, reason: 'transient', error: err instanceof Error ? err : new Error(String(err)) }
- }
-}
-
// ---- CLI fallback -------------------------------------------------------------------------------
/**
@@ -252,6 +153,8 @@ export async function sqliteReadAll(
params: SqliteParam[] = [],
options: SqliteReadOptions = {},
): Promise<SqliteReadResult> {
+ // At the first read, not with this module: the edge host's handoff, monitor and projects import this one.
+ const { builtinSqlite, readBuiltin } = await import('./sqliteBuiltin.js')
const Database = builtinSqlite()
if (Database) {
return readBuiltin(Database, dbPath, sql, params, options.busyTimeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS)
@@ -264,8 +167,3 @@ export async function sqliteReadAll(
options.maxBuffer ?? DEFAULT_MAX_BUFFER,
)
}
-
-/** Test seam: force the CLI path (or the built-in one) regardless of what this Node has. */
-export function overrideBuiltinSqlite(value: DatabaseConstructor | null | undefined): void {
- builtin = value
-}
diff --git a/cli/src/services/AGENTS.md b/cli/src/services/AGENTS.md
index 14a16075a..42fc67961 100644
--- a/cli/src/services/AGENTS.md
+++ b/cli/src/services/AGENTS.md
@@ -91,6 +91,9 @@ master too old to run it starts it beside itself, still in its own process (`src
pulled in for a constant brought zod to search and workspaces, 8 MiB each (`src/dsh/id.ts`). A
failing import fails the service's start, loudly, and the master parks it. `src/leanEntry.spec.ts`
holds each process to its own code, and the master, search, the updater and the edge host to no zod.
+ It also keeps the native `node:sqlite` binding out of the edge host until a service there reads a store
+ that opencode, kilo, hermes or devin keeps a conversation in. `lib/sqliteRead.ts` imports the binding
+ (`lib/sqliteBuiltin.ts`) only at the first read, so import `sqliteReadAll` and never the binding itself.
- `src/services/process.ts` is the process's side: `hostServices` beats to the master and stops every
service before the process exits; `runServiceProcess` is each service's own connection to the core,
with the master's token, reconnecting after core restarts. `<host>.crash` and `<host>.leak`, or a
diff --git a/cli/src/services/handoffProcess.spec.ts b/cli/src/services/handoffProcess.spec.ts
index dbc9ec409..d6e72f8bd 100644
--- a/cli/src/services/handoffProcess.spec.ts
+++ b/cli/src/services/handoffProcess.spec.ts
@@ -1,10 +1,32 @@
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
+import { env } from '../config/env.js'
import { HANDOFF_REQUESTS, type CoreApi } from '../core/api.js'
import { databaseHistory } from '../lib/databaseHistory.js'
+import { startHandoff } from './handoff.js'
import { handoffCoreApi, runHandoffService } from './handoffProcess.js'
import { runServiceProcess, type ServiceProcessOptions } from './process.js'
vi.mock('./process.js', () => ({ runServiceProcess: vi.fn(() => ({ stop: vi.fn() })) }))
+// The SQLite binding, faked: an OpenCode store with one turn in it, and a count of the times it was loaded.
+const binding = vi.hoisted(() => ({ loads: 0, reads: [] as Array<{ path: string; sql: string; params: unknown[] }> }))
+vi.mock('../lib/sqliteBuiltin.js', () => {
+ binding.loads++
+ const part = (type: string, text: string) => JSON.stringify({ type, text })
+ return {
+ builtinSqlite: () => class {},
+ readBuiltin: (_database: unknown, path: string, sql: string, params: unknown[]) => {
+ binding.reads.push({ path, sql, params })
+ return { ok: true, via: 'builtin', rows: [
+ { mid: 'msg_1', mtc: 1, mdata: JSON.stringify({ role: 'user' }), pid: 'prt_1', pdata: part('text', 'add a login page') },
+ { mid: 'msg_2', mtc: 2, mdata: JSON.stringify({ role: 'assistant' }), pid: 'prt_2', pdata: part('text', 'Added the login page.') },
+ ] }
+ },
+ }
+})
afterEach(() => vi.clearAllMocks())
describe('the handoff in the edge host', () => {
@@ -76,4 +98,32 @@ describe('the handoff in the edge host', () => {
runHandoffService({ dataDir: '/data', socketPath: '/data/daemon-1.sock', machineId: 'm', token: 't' })
expect(Object.keys(vi.mocked(runServiceProcess).mock.calls.at(-1)![0].requests)).toEqual([...HANDOFF_REQUESTS])
})
+
+ it('loads the SQLite binding only for an engine that keeps its conversation in a store, and hands that one over', async () => {
+ const ws = mkdtempSync(join(tmpdir(), 'handoff-store-'))
+ try {
+ execFileSync('git', ['init', '-q'], { cwd: ws })
+ const agents: Record<string, Record<string, unknown>> = {
+ claude: { agentId: 'claude-1', sessionId: 's1', engine: 'claude', cwd: ws, transcriptPath: null, registeredAt: Date.now() - 60_000 },
+ opencode: { agentId: 'opencode-1', sessionId: 'ses_handoff1', engine: 'opencode', cwd: ws, transcriptPath: null, registeredAt: Date.now() - 60_000 },
+ }
+ const core = handoffCoreApi('/data', async (query, payload) => ({
+ value: query === 'resolve' ? Object.values(agents).find((agent) => agent.agentId === payload.id) ?? null
+ : query === 'recentAsks' || query === 'recaps' ? [] : null,
+ }))
+ // Claude Code and Codex keep a transcript file: nothing to read from a store, and the binding stays out.
+ expect(core.transcripts.databaseHistory(agents.claude as never)).toBeUndefined()
+ expect(core.transcripts.databaseHistory({ ...agents.claude, engine: 'codex' } as never)).toBeUndefined()
+ const prepare = startHandoff(core).agent_handoff_prepare!
+ const asker = { local: true, owner: true }
+ await prepare({ agentId: 'claude-1', changeId: 'c'.repeat(32), targetEngine: 'codex' }, asker)
+ expect(binding.loads).toBe(0)
+ // OpenCode: read through the binding, loaded for that read, and the turn it holds handed over.
+ const handed = await prepare({ agentId: 'opencode-1', changeId: 'a'.repeat(32), targetEngine: 'claude' }, asker) as { file: string | null }
+ expect(binding.loads).toBe(1)
+ expect(binding.reads).toEqual([{ path: join(env.OPENCODE_DATA_DIR, 'opencode.db'), sql: expect.stringContaining('FROM message m'), params: ['ses_handoff1'] }])
+ expect(handed.file).toEqual(expect.any(String))
+ expect(readFileSync(join(ws, handed.file!), 'utf8')).toContain('add a login page')
+ } finally { rmSync(ws, { recursive: true, force: true }) }
+ })
})
diff --git a/companions/scripts/memory-native-opencode-source-probe.mjs b/companions/scripts/memory-native-opencode-source-probe.mjs
index 17291b5e8..c30c4222e 100644
--- a/companions/scripts/memory-native-opencode-source-probe.mjs
+++ b/companions/scripts/memory-native-opencode-source-probe.mjs
@@ -54,7 +54,7 @@ try {
`export { NativeMemoryCapture } from ${JSON.stringify(join(cli, 'src/memory/capture.ts'))};`,
`export { CodingMemoryStore } from ${JSON.stringify(join(cli, 'src/memory/store.ts'))};`,
`export { QUEUE_OPERATIONS } from ${JSON.stringify(join(cli, 'src/memory/operations.ts'))};`,
- `export { closeSqliteHandles } from ${JSON.stringify(join(cli, 'src/lib/sqliteRead.ts'))};`,
+ `export { closeSqliteHandles } from ${JSON.stringify(join(cli, 'src/lib/sqliteBuiltin.ts'))};`,
`export { opencodeMemoryPluginSource } from ${JSON.stringify(join(cli, 'src/lib/opencodeMemoryPlugin.ts'))};`,
].join('\n'), resolveDir: cli }, outfile: bundle, bundle: true, platform: 'node', format: 'esm', target: 'node22',
banner: { js: "import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);" } })
diff --git a/companions/src/memory/activity.spec.ts b/companions/src/memory/activity.spec.ts
index 5e89a92bb..ffc05de9d 100644
--- a/companions/src/memory/activity.spec.ts
+++ b/companions/src/memory/activity.spec.ts
@@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { CodingMemoryStore } from './store.js'
-import { builtinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { builtinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import type { Database } from './database.js'
import type { MemoryRecord, MemoryDraft, SourceEvent } from './types.js'
import type { ActivityReceiver } from './receipts.js'
diff --git a/companions/src/memory/feedback.spec.ts b/companions/src/memory/feedback.spec.ts
index afd9bbd7b..da2d7f6a6 100644
--- a/companions/src/memory/feedback.spec.ts
+++ b/companions/src/memory/feedback.spec.ts
@@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { CodingMemoryStore } from './store.js'
-import { builtinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { builtinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import type { Database } from './database.js'
import type { LibraryCommand } from './library.js'
import type { MemoryDeliveryBinding } from './receipts.js'
diff --git a/companions/src/memory/notebook.spec.ts b/companions/src/memory/notebook.spec.ts
index 4dbb296cd..8dc630a96 100644
--- a/companions/src/memory/notebook.spec.ts
+++ b/companions/src/memory/notebook.spec.ts
@@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { CodingMemoryStore } from './store.js'
-import { builtinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { builtinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import type { Database as MemoryDatabase } from './database.js'
import type { NotebookLease } from './notebook.js'
import type { MemoryAccess, MemoryDraft, MemoryRecord, SourceEvent } from './types.js'
diff --git a/companions/src/memory/opencodeCapture.spec.ts b/companions/src/memory/opencodeCapture.spec.ts
index 836fef747..017bfed8a 100644
--- a/companions/src/memory/opencodeCapture.spec.ts
+++ b/companions/src/memory/opencodeCapture.spec.ts
@@ -4,7 +4,7 @@ import { mkdtempSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
-import { closeSqliteHandles, overrideBuiltinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { closeSqliteHandles, overrideBuiltinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import { NativeMemoryCapture, type CaptureSession } from './capture.js'
import { CodingMemoryStore } from './store.js'
import type { Database } from './database.js'
diff --git a/companions/src/memory/queue.spec.ts b/companions/src/memory/queue.spec.ts
index 1b7af678f..f5c4391e7 100644
--- a/companions/src/memory/queue.spec.ts
+++ b/companions/src/memory/queue.spec.ts
@@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { CodingMemoryStore } from './store.js'
-import { builtinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { builtinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import type { CaptureBatch, LearningLease } from './queue.js'
import type { MemoryAccess, MemoryDraft, SourceEvent } from './types.js'
diff --git a/companions/src/memory/store.spec.ts b/companions/src/memory/store.spec.ts
index 0f056c9c3..662651bba 100644
--- a/companions/src/memory/store.spec.ts
+++ b/companions/src/memory/store.spec.ts
@@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { CodingMemoryStore } from './store.js'
-import { builtinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { builtinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import type { MemoryAccess, MemoryDraft, SourceEvent } from './types.js'
const access: MemoryAccess = { profileId: 'owner', projectIds: ['project_a'], includeProfile: true }
diff --git a/companions/src/memory/store.ts b/companions/src/memory/store.ts
index 5ce2e9ac7..e2248a746 100644
--- a/companions/src/memory/store.ts
+++ b/companions/src/memory/store.ts
@@ -5,7 +5,7 @@
import { randomUUID } from 'node:crypto'
import { chmodSync, existsSync, lstatSync, mkdirSync } from 'node:fs'
import { basename, dirname, isAbsolute, join } from 'node:path'
-import { builtinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { builtinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import { redact } from '../shared/guard.js'
import { admission, assertSafe, canonical, digest, proposalFingerprint } from './admission.js'
import type { ProjectLocator } from './project.js'
diff --git a/companions/src/memory/usefulness.spec.ts b/companions/src/memory/usefulness.spec.ts
index 520e4e777..70dc7a59f 100644
--- a/companions/src/memory/usefulness.spec.ts
+++ b/companions/src/memory/usefulness.spec.ts
@@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, readFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
-import { builtinSqlite } from '../../../cli/src/lib/sqliteRead.js'
+import { builtinSqlite } from '../../../cli/src/lib/sqliteBuiltin.js'
import type { Database } from './database.js'
import { CodingMemoryStore } from './store.js'
import type { MemoryDeliveryBinding } from './receipts.js'