The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(state-store): address contention and pragma review findings
Yann Roberto688e939b2976
diff --git a/docs/architecture/hud-status-session-control.md b/docs/architecture/hud-status-session-control.mdindex 7225018d4..8518a1008 100644--- a/docs/architecture/hud-status-session-control.md+++ b/docs/architecture/hud-status-session-control.md@@ -94,3 +94,10 @@ a paused process may still be writing. After an abnormal exit, stop all ECCprocesses using that database, inspect the PID and hostname in the lock file,and remove only the leftover `.ecc-state.lock` file before retrying. Do notremove the database itself. In-memory stores do not create lock files.++The control-pane HTTP server runs board claims and moves in one-shot workers,+so waiting for another writer does not stall health checks or snapshots. A+timed-out mutation returns HTTP 503 with `code: STATE_STORE_BUSY` and+`Retry-After: 1`; invalid mutations continue to return HTTP 400. Each worker+closes its store before reporting the result and exits naturally, including+when the requesting browser disconnects, to avoid interrupting a write.diff --git a/scripts/lib/control-pane/server.js b/scripts/lib/control-pane/server.jsindex 8391ac796..cf4d73168 100644--- a/scripts/lib/control-pane/server.js+++ b/scripts/lib/control-pane/server.js@@ -11,20 +11,7 @@ const { renderControlPaneHtml } = require('./ui');const { renderProximityVizHtml } = require('./proximity-viz');const { renderControlPlaneViewHtml } = require('./control-plane-view-ui');const { createControlPlaneViewSource } = require('./control-plane-view');-const { claimWorkItem, moveWorkItem } = require('./work-item-mutations');--// Run a single write against the local work-item store, then close it. Kept-// thin so the loopback-only server can mutate the JIT board without holding a-// long-lived handle.-async function withStateStore(stateDbPath, fn) {- const { createStateStore } = require('../state-store');- const store = await createStateStore({ dbPath: stateDbPath });- try {- return await fn(store);- } finally {- store.close();- }-}+const { runWorkItemMutation } = require('./work-item-runner');// Host/Origin gating lives in scripts/lib/loopback-guard.js so every ECC// loopback server shares one hardened implementation; re-exported below to@@ -349,19 +336,25 @@ function createControlPaneServer(options = {}) {const id = decodeURIComponent((claimMatch || moveMatch)[1]);const body = await readRequestJson(req);try {- const result = await withStateStore(resolvedConfig.stateDbPath, store =>+ const result = await runWorkItemMutation(resolvedConfig.stateDbPath,+ claimMatch ? 'claim' : 'move',claimMatch- ? claimWorkItem(store, {+ ? {id,owner: body.owner,assigneeKind: body.as || body.assigneeKind,sessionId: body.sessionId- })- : moveWorkItem(store, { id, lane: body.lane })+ }+ : { id, lane: body.lane });sendJson(res, 200, { ok: true, ...result });} catch (mutationError) {- sendJson(res, 400, { ok: false, error: mutationError.message });+ if (mutationError.code === 'STATE_STORE_BUSY') {+ res.setHeader('Retry-After', '1');+ sendJson(res, 503, { ok: false, code: mutationError.code, error: mutationError.message });+ } else {+ sendJson(res, 400, { ok: false, error: mutationError.message });+ }}return;}diff --git a/scripts/lib/control-pane/work-item-runner.js b/scripts/lib/control-pane/work-item-runner.jsnew file mode 100644index 000000000..ab4e16b30--- /dev/null+++ b/scripts/lib/control-pane/work-item-runner.js@@ -0,0 +1,27 @@+'use strict';++const path = require('path');+const { Worker } = require('worker_threads');++/** Await a one-shot worker without blocking unrelated HTTP requests on a lock. */+function runWorkItemMutation(dbPath, action, args) {+ return new Promise((resolve, reject) => {+ const worker = new Worker(path.join(__dirname, 'work-item-worker.js'), {+ workerData: { dbPath, action, args }+ });+ let responded = false;+ worker.once('message', message => {+ responded = true;+ if (message.ok) resolve(message.result);+ else reject(Object.assign(new Error(message.error.message), { code: message.error.code }));+ });+ worker.once('error', reject);+ worker.once('exit', code => {+ if (!responded) reject(new Error(`Work-item worker exited without a result (${code})`));+ });+ // Let the worker exit naturally after closing the store. Terminating it on+ // a client disconnect could interrupt a write and abandon its file lock.+ });+}++module.exports = { runWorkItemMutation };diff --git a/scripts/lib/control-pane/work-item-worker.js b/scripts/lib/control-pane/work-item-worker.jsnew file mode 100644index 000000000..72c808eb5--- /dev/null+++ b/scripts/lib/control-pane/work-item-worker.js@@ -0,0 +1,23 @@+'use strict';++const { parentPort, workerData } = require('worker_threads');+const { createStateStore } = require('../state-store');+const { claimWorkItem, moveWorkItem } = require('./work-item-mutations');++/** Complete one board mutation away from the HTTP event loop, then close. */+async function mutate() {+ const { dbPath, action, args } = workerData;+ const mutation = action === 'claim' ? claimWorkItem : action === 'move' ? moveWorkItem : null;+ if (!mutation) throw new Error('Unknown work-item mutation');+ const store = await createStateStore({ dbPath });+ try {+ return store._database.transaction(() => mutation(store, args))();+ } finally {+ store.close();+ }+}++mutate().then(+ result => parentPort.postMessage({ ok: true, result }),+ error => parentPort.postMessage({ ok: false, error: { message: error.message, code: error.code } })+);diff --git a/scripts/lib/state-store/index.js b/scripts/lib/state-store/index.jsindex 3d60fc579..3b4d8f33d 100644--- a/scripts/lib/state-store/index.js+++ b/scripts/lib/state-store/index.js@@ -208,15 +208,20 @@ function wrapSqlJsDatabase(SQL, dbPath) {let inSnapshot = false;let dirty = false;let inTransaction = false;+ let pragmaExecuted = false;+ let snapshotBytes = null;function reload() {if (dbPath === ':memory:' && rawDb) return;const bytes = dbPath !== ':memory:' && assertSafeDatabaseFile(dbPath)? readDatabaseFile(dbPath) : undefined;+ // sql.js can use its input buffer as writable backing storage.+ const originalBytes = bytes ? Buffer.from(bytes) : null;const latest = new SQL.Database(bytes);latest.run('PRAGMA foreign_keys = ON');if (rawDb) rawDb.close();rawDb = latest;+ snapshotBytes = originalBytes;}// Hold one lock from reload through commit. Nested statements and public@@ -228,18 +233,27 @@ function wrapSqlJsDatabase(SQL, dbPath) {reload();inSnapshot = true;dirty = false;+ pragmaExecuted = false;try {const result = callback();if (result && typeof result.then === 'function') {throw new Error('State-store operations must be synchronous');}- if (dirty && dbPath !== ':memory:') {- writeDatabaseFileAtomic(dbPath, Buffer.from(rawDb.export()));+ if ((dirty || pragmaExecuted) && dbPath !== ':memory:') {+ // SQLite PRAGMAs include reads, connection settings and persisted+ // changes. Compare the resulting database instead of parsing their+ // SQL syntax. Export only here: exporting inside a transaction would+ // implicitly end it before our commit/rollback boundary.+ const data = Buffer.from(rawDb.export());+ if (dirty || !snapshotBytes || !data.equals(snapshotBytes)) {+ writeDatabaseFileAtomic(dbPath, data);+ }}return result;} finally {inSnapshot = false;dirty = false;+ pragmaExecuted = false;}};return dbPath === ':memory:' ? execute() : withStateStoreLock(dbPath, execute);@@ -319,7 +333,10 @@ function wrapSqlJsDatabase(SQL, dbPath) {},pragma(pragmaStr) {- return withSnapshot(() => rawDb.run(`PRAGMA ${pragmaStr}`));+ return withSnapshot(() => {+ rawDb.run(`PRAGMA ${pragmaStr}`);+ pragmaExecuted = true;+ });},prepare(sql) {diff --git a/tests/lib/control-pane-contention.test.js b/tests/lib/control-pane-contention.test.jsnew file mode 100644index 000000000..ce7408023--- /dev/null+++ b/tests/lib/control-pane-contention.test.js@@ -0,0 +1,204 @@+'use strict';++const assert = require('assert');+const fs = require('fs');+const http = require('http');+const os = require('os');+const path = require('path');+const { fork } = require('child_process');+const { createStateStore } = require('../../scripts/lib/state-store');++const FIXTURE = path.join(__dirname, 'helpers', 'control-pane-contention-server.js');++function request(url, method = 'GET', body, timeoutMs = 1000) {+ return new Promise((resolve, reject) => {+ const req = http.request(url, { method, agent: false, headers: { 'Content-Type': 'application/json' } }, res => {+ let text = '';+ res.setEncoding('utf8');+ res.on('data', data => { text += data; });+ res.on('end', () => {+ try {+ resolve({ status: res.statusCode, headers: res.headers, body: JSON.parse(text) });+ } catch (error) {+ reject(error);+ }+ });+ });+ req.setTimeout(timeoutMs, () => req.destroy(new Error(`HTTP ${method} timed out after ${timeoutMs}ms`)));+ req.on('error', reject);+ req.end(body === undefined ? undefined : JSON.stringify(body));+ });+}++async function startServer(dbPath, readOnly = false) {+ const child = fork(FIXTURE, [dbPath, readOnly ? 'read-only' : 'editable'], {+ stdio: ['ignore', 'pipe', 'pipe', 'ipc']+ });+ const closed = new Promise(resolve => child.once('close', resolve));+ child.stdout.resume();+ let stderr = '';+ child.stderr.on('data', data => { stderr += data; });+ let ready;+ try {+ ready = await new Promise((resolve, reject) => {+ const timer = setTimeout(() => reject(new Error(`Server startup timed out: ${stderr}`)), 10000);+ child.once('error', error => { clearTimeout(timer); reject(error); });+ child.on('message', message => {+ if (message.type === 'ready') { clearTimeout(timer); resolve(message); }+ });+ child.once('exit', code => { clearTimeout(timer); reject(new Error(`Server exited ${code}: ${stderr}`)); });+ });+ } catch (error) {+ child.kill();+ await closed;+ throw error;+ }+ return {+ url: ready.url,+ mutationReceived() {+ return new Promise((resolve, reject) => {+ const timer = setTimeout(() => { child.off('message', onMessage); reject(new Error('Mutation never reached server')); }, 5000);+ function onMessage(message) {+ if (message.type !== 'mutation-received') return;+ clearTimeout(timer);+ child.off('message', onMessage);+ resolve();+ }+ child.on('message', onMessage);+ });+ },+ async close() {+ if (child.exitCode === null && child.signalCode === null) child.kill();+ await closed;+ }+ };+}++async function withServer(readOnly, fn) {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-http-contention-'));+ const dbPath = path.join(dir, 'state.db');+ let server;+ try {+ const store = await createStateStore({ dbPath });+ try {+ store.upsertWorkItem({ id: 'task', title: 'Synthetic board task', source: 'manual', status: 'open' });+ } finally {+ store.close();+ }+ server = await startServer(dbPath, readOnly);+ return await fn(server, dbPath);+ } finally {+ if (server) await server.close();+ fs.rmSync(dir, { recursive: true, force: true });+ }+}++async function contentionScenario() {+ return withServer(false, async (server, dbPath) => {+ const bytesBefore = fs.readFileSync(dbPath);+ const lockPath = `${dbPath}.ecc-state.lock`;+ const fd = fs.openSync(lockPath, 'wx', 0o600);+ fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, hostname: os.hostname() }));+ let outcome;+ try {+ const received = server.mutationReceived();+ let settled = false;+ const mutation = request(`${server.url}/api/work-items/task/claim`, 'POST', { owner: 'alice' }, 12000)+ .then(result => { settled = true; return result; }, error => { settled = true; return { transportError: error }; });+ await received;+ let healthError = null;+ let healthChecks = 0;+ let snapshotChecks = 0;+ // Probe continuously after actual HTTP receipt, including asynchronous+ // database initialization. No timing guess about when lock wait begins.+ while (!settled) {+ try {+ const health = await request(`${server.url}/api/health`);+ assert.strictEqual(health.status, 200);+ healthChecks += 1;+ const snapshot = await request(`${server.url}/api/snapshot`, 'GET', undefined, 3000);+ assert.strictEqual(snapshot.status, 200);+ assert.ok(snapshot.body.workItems.items.some(item => item.id === 'task'));+ snapshotChecks += 1;+ } catch (error) {+ healthError = error.message;+ break;+ }+ }+ const result = await mutation;+ if (result.transportError) throw result.transportError;+ outcome = { result, healthError, healthChecks, snapshotChecks };+ } finally {+ fs.closeSync(fd);+ fs.unlinkSync(lockPath);+ }+ assert.deepStrictEqual(fs.readFileSync(dbPath), bytesBefore, 'Timed-out mutation must not change database bytes');+ const store = await createStateStore({ dbPath });+ try {+ const item = store.getWorkItemById('task');+ assert.strictEqual(item.status, 'open');+ assert.strictEqual(item.owner, null);+ } finally {+ store.close();+ }+ return outcome;+ });+}++async function run() {+ let passed = 0;+ let failed = 0;+ async function test(name, fn) {+ try {+ await fn();+ console.log(` PASS ${name}`);+ passed += 1;+ } catch (error) {+ console.log(` FAIL ${name}\n ${error.message}`);+ failed += 1;+ }+ }+ console.log('\n=== Testing control-pane mutation contention ===\n');+ const contention = await contentionScenario();+ await test('health and snapshots remain responsive while a board mutation waits for another database writer', () => {+ assert.strictEqual(contention.healthError, null, contention.healthError || undefined);+ assert.ok(contention.healthChecks > 0);+ assert.ok(contention.snapshotChecks > 0);+ });+ await test('lock timeout is retryable HTTP 503 with Retry-After', () => {+ assert.strictEqual(contention.result.status, 503);+ assert.ok(Number(contention.result.headers['retry-after']) > 0, 'Retry-After must give a positive delay');+ assert.strictEqual(contention.result.body.ok, false);+ assert.strictEqual(contention.result.body.code, 'STATE_STORE_BUSY');+ });+ await test('claim and move still save their changes, while invalid edits return 400', () => withServer(false, async (server, dbPath) => {+ const claim = await request(`${server.url}/api/work-items/task/claim`, 'POST', { owner: 'alice', as: 'human' }, 5000);+ assert.strictEqual(claim.status, 200);+ assert.strictEqual(claim.body.item.owner, 'alice');+ assert.strictEqual(claim.body.item.status, 'running');+ const move = await request(`${server.url}/api/work-items/task/move`, 'POST', { lane: 'blocked' }, 5000);+ assert.strictEqual(move.status, 200);+ assert.strictEqual(move.body.item.status, 'blocked');+ const invalid = await request(`${server.url}/api/work-items/task/move`, 'POST', { lane: 'imaginary' }, 5000);+ assert.strictEqual(invalid.status, 400);+ assert.strictEqual(invalid.body.ok, false);+ const store = await createStateStore({ dbPath });+ try {+ const item = store.getWorkItemById('task');+ assert.strictEqual(item.owner, 'alice');+ assert.strictEqual(item.status, 'blocked');+ assert.strictEqual(item.metadata.assigneeKind, 'human');+ } finally {+ store.close();+ }+ }));+ await test('read-only mode rejects board edits before database work', () => withServer(true, async server => {+ for (const action of ['claim', 'move']) {+ const result = await request(`${server.url}/api/work-items/task/${action}`, 'POST', { owner: 'alice', lane: 'done' });+ assert.strictEqual(result.status, 403);+ }+ }));+ console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);+ process.exitCode = failed ? 1 : 0;+}+run().catch(error => { console.error(error); process.exitCode = 1; });diff --git a/tests/lib/helpers/control-pane-contention-server.js b/tests/lib/helpers/control-pane-contention-server.jsnew file mode 100644index 000000000..24f51cdc5--- /dev/null+++ b/tests/lib/helpers/control-pane-contention-server.js@@ -0,0 +1,26 @@+'use strict';++const { createControlPaneServer } = require('../../../scripts/lib/control-pane/server');++async function main() {+ const app = createControlPaneServer({+ host: '127.0.0.1', port: 0, stateDbPath: process.argv[2],+ dbPath: `${process.argv[2]}.ecc2`, allowActions: process.argv[3] !== 'read-only'+ });+ app.server.prependListener('request', req => {+ if (req.method === 'POST') process.send({ type: 'mutation-received' });+ });+ await app.listen();+ process.send({ type: 'ready', url: app.url });+ process.on('message', async message => {+ if (message === 'close') {+ await app.close();+ process.disconnect();+ }+ });+}+main().catch(error => {+ console.error(error);+ process.exitCode = 1;+ if (process.connected) process.disconnect();+});diff --git a/tests/lib/state-store-persistence.test.js b/tests/lib/state-store-persistence.test.jsindex 00edf7ab4..ce7c702ca 100644--- a/tests/lib/state-store-persistence.test.js+++ b/tests/lib/state-store-persistence.test.js@@ -50,6 +50,71 @@ async function run() {assert.deepStrictEqual(fs.readFileSync(dbPath), before);});+ await test('persistent PRAGMA changes survive snapshot reload and reopening', async dbPath => {+ const store = await createStateStore({ dbPath });+ try {+ assert.strictEqual(store._database.pragma('user_version = 3252'), undefined);+ assert.strictEqual(store._database.prepare('PRAGMA user_version').get().user_version, 3252);+ store._database.pragma('application_id(2468)');+ } finally { store.close(); }+ const reopened = await createStateStore({ dbPath });+ try {+ assert.strictEqual(reopened._database.prepare('PRAGMA user_version').get().user_version, 3252);+ assert.strictEqual(reopened._database.prepare('PRAGMA application_id').get().application_id, 2468);+ } finally { reopened.close(); }+ });++ await test('read-only and connection-local PRAGMAs never rewrite database bytes', async dbPath => {+ const store = await createStateStore({ dbPath });+ const before = fs.readFileSync(dbPath);+ const rename = fs.renameSync;+ let writes = 0;+ fs.renameSync = (from, to) => {+ if (to === dbPath) writes += 1;+ return rename(from, to);+ };+ try {+ store._database.pragma('user_version');+ store._database.pragma('table_info(work_items)');+ store._database.pragma('integrity_check');+ store._database.transaction(() => {+ store._database.pragma('user_version');+ store._database.pragma('table_info(work_items)');+ })();+ store._database.withSnapshot(() => {+ store._database.pragma('cache_size = 256');+ assert.strictEqual(store._database.prepare('PRAGMA cache_size').get().cache_size, 256);+ });+ assert.strictEqual(writes, 0);+ assert.deepStrictEqual(fs.readFileSync(dbPath), before);+ } finally {+ fs.renameSync = rename;+ store.close();+ }+ });++ await test('PRAGMA persistence waits for commit and never commits a rolled-back transaction', async dbPath => {+ const store = await createStateStore({ dbPath });+ try {+ store._database.transaction(() => {+ store.upsertWorkItem(item('committed-pragma'));+ store._database.pragma('table_info(work_items)');+ store._database.pragma('user_version = 3252');+ })();+ store._database.transaction(() => store._database.pragma('application_id(2468)'))();+ assert.strictEqual(store._database.prepare('PRAGMA application_id').get().application_id, 2468);+ const before = fs.readFileSync(dbPath);+ assert.throws(() => store._database.transaction(() => {+ store.upsertWorkItem(item('rolled-back-pragma'));+ store._database.pragma('user_version = 9999');+ throw new Error('abort PRAGMA transaction');+ })(), /abort PRAGMA transaction/);+ assert.deepStrictEqual(fs.readFileSync(dbPath), before);+ assert.strictEqual(store._database.prepare('PRAGMA user_version').get().user_version, 3252);+ assert.deepStrictEqual(store.listWorkItems().items.map(row => row.id), ['committed-pragma']);+ } finally { store.close(); }+ });+await test('failed persistence is discarded, releases the lock and cannot overwrite a later writer', async dbPath => {const first = await createStateStore({ dbPath });const second = await createStateStore({ dbPath });diff --git a/tests/scripts/npm-publish-surface.test.js b/tests/scripts/npm-publish-surface.test.jsindex 02ca28943..c053151d1 100644--- a/tests/scripts/npm-publish-surface.test.js+++ b/tests/scripts/npm-publish-surface.test.js@@ -213,6 +213,9 @@ function main() {"docs/design/context-carriers.md","docs/design/context-profile-delivery.md","scripts/control-pane.js",+ "scripts/lib/control-pane/work-item-runner.js",+ "scripts/lib/control-pane/work-item-worker.js",+ "scripts/lib/state-store/file-lock.js","scripts/feedback.js","scripts/ito.js","scripts/memory.js",