Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(state-store): address contention and pragma review findings

Yann Roberto688e939b2976
diff --git a/docs/architecture/hud-status-session-control.md b/docs/architecture/hud-status-session-control.md
index 7225018d4..8518a1008 100644
--- a/docs/architecture/hud-status-session-control.md
+++ b/docs/architecture/hud-status-session-control.md
@@ -94,3 +94,10 @@ a paused process may still be writing. After an abnormal exit, stop all ECC
processes using that database, inspect the PID and hostname in the lock file,
and remove only the leftover `.ecc-state.lock` file before retrying. Do not
remove the database itself. In-memory stores do not create lock files.
+
+The control-pane HTTP server runs board claims and moves in one-shot workers,
+so waiting for another writer does not stall health checks or snapshots. A
+timed-out mutation returns HTTP 503 with `code: STATE_STORE_BUSY` and
+`Retry-After: 1`; invalid mutations continue to return HTTP 400. Each worker
+closes its store before reporting the result and exits naturally, including
+when the requesting browser disconnects, to avoid interrupting a write.
diff --git a/scripts/lib/control-pane/server.js b/scripts/lib/control-pane/server.js
index 8391ac796..cf4d73168 100644
--- a/scripts/lib/control-pane/server.js
+++ b/scripts/lib/control-pane/server.js
@@ -11,20 +11,7 @@ const { renderControlPaneHtml } = require('./ui');
const { renderProximityVizHtml } = require('./proximity-viz');
const { renderControlPlaneViewHtml } = require('./control-plane-view-ui');
const { createControlPlaneViewSource } = require('./control-plane-view');
-const { claimWorkItem, moveWorkItem } = require('./work-item-mutations');
-
-// Run a single write against the local work-item store, then close it. Kept
-// thin so the loopback-only server can mutate the JIT board without holding a
-// long-lived handle.
-async function withStateStore(stateDbPath, fn) {
- const { createStateStore } = require('../state-store');
- const store = await createStateStore({ dbPath: stateDbPath });
- try {
- return await fn(store);
- } finally {
- store.close();
- }
-}
+const { runWorkItemMutation } = require('./work-item-runner');
// Host/Origin gating lives in scripts/lib/loopback-guard.js so every ECC
// loopback server shares one hardened implementation; re-exported below to
@@ -349,19 +336,25 @@ function createControlPaneServer(options = {}) {
const id = decodeURIComponent((claimMatch || moveMatch)[1]);
const body = await readRequestJson(req);
try {
- const result = await withStateStore(resolvedConfig.stateDbPath, store =>
+ const result = await runWorkItemMutation(resolvedConfig.stateDbPath,
+ claimMatch ? 'claim' : 'move',
claimMatch
- ? claimWorkItem(store, {
+ ? {
id,
owner: body.owner,
assigneeKind: body.as || body.assigneeKind,
sessionId: body.sessionId
- })
- : moveWorkItem(store, { id, lane: body.lane })
+ }
+ : { id, lane: body.lane }
);
sendJson(res, 200, { ok: true, ...result });
} catch (mutationError) {
- sendJson(res, 400, { ok: false, error: mutationError.message });
+ if (mutationError.code === 'STATE_STORE_BUSY') {
+ res.setHeader('Retry-After', '1');
+ sendJson(res, 503, { ok: false, code: mutationError.code, error: mutationError.message });
+ } else {
+ sendJson(res, 400, { ok: false, error: mutationError.message });
+ }
}
return;
}
diff --git a/scripts/lib/control-pane/work-item-runner.js b/scripts/lib/control-pane/work-item-runner.js
new file mode 100644
index 000000000..ab4e16b30
--- /dev/null
+++ b/scripts/lib/control-pane/work-item-runner.js
@@ -0,0 +1,27 @@
+'use strict';
+
+const path = require('path');
+const { Worker } = require('worker_threads');
+
+/** Await a one-shot worker without blocking unrelated HTTP requests on a lock. */
+function runWorkItemMutation(dbPath, action, args) {
+ return new Promise((resolve, reject) => {
+ const worker = new Worker(path.join(__dirname, 'work-item-worker.js'), {
+ workerData: { dbPath, action, args }
+ });
+ let responded = false;
+ worker.once('message', message => {
+ responded = true;
+ if (message.ok) resolve(message.result);
+ else reject(Object.assign(new Error(message.error.message), { code: message.error.code }));
+ });
+ worker.once('error', reject);
+ worker.once('exit', code => {
+ if (!responded) reject(new Error(`Work-item worker exited without a result (${code})`));
+ });
+ // Let the worker exit naturally after closing the store. Terminating it on
+ // a client disconnect could interrupt a write and abandon its file lock.
+ });
+}
+
+module.exports = { runWorkItemMutation };
diff --git a/scripts/lib/control-pane/work-item-worker.js b/scripts/lib/control-pane/work-item-worker.js
new file mode 100644
index 000000000..72c808eb5
--- /dev/null
+++ b/scripts/lib/control-pane/work-item-worker.js
@@ -0,0 +1,23 @@
+'use strict';
+
+const { parentPort, workerData } = require('worker_threads');
+const { createStateStore } = require('../state-store');
+const { claimWorkItem, moveWorkItem } = require('./work-item-mutations');
+
+/** Complete one board mutation away from the HTTP event loop, then close. */
+async function mutate() {
+ const { dbPath, action, args } = workerData;
+ const mutation = action === 'claim' ? claimWorkItem : action === 'move' ? moveWorkItem : null;
+ if (!mutation) throw new Error('Unknown work-item mutation');
+ const store = await createStateStore({ dbPath });
+ try {
+ return store._database.transaction(() => mutation(store, args))();
+ } finally {
+ store.close();
+ }
+}
+
+mutate().then(
+ result => parentPort.postMessage({ ok: true, result }),
+ error => parentPort.postMessage({ ok: false, error: { message: error.message, code: error.code } })
+);
diff --git a/scripts/lib/state-store/index.js b/scripts/lib/state-store/index.js
index 3d60fc579..3b4d8f33d 100644
--- a/scripts/lib/state-store/index.js
+++ b/scripts/lib/state-store/index.js
@@ -208,15 +208,20 @@ function wrapSqlJsDatabase(SQL, dbPath) {
let inSnapshot = false;
let dirty = false;
let inTransaction = false;
+ let pragmaExecuted = false;
+ let snapshotBytes = null;
function reload() {
if (dbPath === ':memory:' && rawDb) return;
const bytes = dbPath !== ':memory:' && assertSafeDatabaseFile(dbPath)
? readDatabaseFile(dbPath) : undefined;
+ // sql.js can use its input buffer as writable backing storage.
+ const originalBytes = bytes ? Buffer.from(bytes) : null;
const latest = new SQL.Database(bytes);
latest.run('PRAGMA foreign_keys = ON');
if (rawDb) rawDb.close();
rawDb = latest;
+ snapshotBytes = originalBytes;
}
// Hold one lock from reload through commit. Nested statements and public
@@ -228,18 +233,27 @@ function wrapSqlJsDatabase(SQL, dbPath) {
reload();
inSnapshot = true;
dirty = false;
+ pragmaExecuted = false;
try {
const result = callback();
if (result && typeof result.then === 'function') {
throw new Error('State-store operations must be synchronous');
}
- if (dirty && dbPath !== ':memory:') {
- writeDatabaseFileAtomic(dbPath, Buffer.from(rawDb.export()));
+ if ((dirty || pragmaExecuted) && dbPath !== ':memory:') {
+ // SQLite PRAGMAs include reads, connection settings and persisted
+ // changes. Compare the resulting database instead of parsing their
+ // SQL syntax. Export only here: exporting inside a transaction would
+ // implicitly end it before our commit/rollback boundary.
+ const data = Buffer.from(rawDb.export());
+ if (dirty || !snapshotBytes || !data.equals(snapshotBytes)) {
+ writeDatabaseFileAtomic(dbPath, data);
+ }
}
return result;
} finally {
inSnapshot = false;
dirty = false;
+ pragmaExecuted = false;
}
};
return dbPath === ':memory:' ? execute() : withStateStoreLock(dbPath, execute);
@@ -319,7 +333,10 @@ function wrapSqlJsDatabase(SQL, dbPath) {
},
pragma(pragmaStr) {
- return withSnapshot(() => rawDb.run(`PRAGMA ${pragmaStr}`));
+ return withSnapshot(() => {
+ rawDb.run(`PRAGMA ${pragmaStr}`);
+ pragmaExecuted = true;
+ });
},
prepare(sql) {
diff --git a/tests/lib/control-pane-contention.test.js b/tests/lib/control-pane-contention.test.js
new file mode 100644
index 000000000..ce7408023
--- /dev/null
+++ b/tests/lib/control-pane-contention.test.js
@@ -0,0 +1,204 @@
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const http = require('http');
+const os = require('os');
+const path = require('path');
+const { fork } = require('child_process');
+const { createStateStore } = require('../../scripts/lib/state-store');
+
+const FIXTURE = path.join(__dirname, 'helpers', 'control-pane-contention-server.js');
+
+function request(url, method = 'GET', body, timeoutMs = 1000) {
+ return new Promise((resolve, reject) => {
+ const req = http.request(url, { method, agent: false, headers: { 'Content-Type': 'application/json' } }, res => {
+ let text = '';
+ res.setEncoding('utf8');
+ res.on('data', data => { text += data; });
+ res.on('end', () => {
+ try {
+ resolve({ status: res.statusCode, headers: res.headers, body: JSON.parse(text) });
+ } catch (error) {
+ reject(error);
+ }
+ });
+ });
+ req.setTimeout(timeoutMs, () => req.destroy(new Error(`HTTP ${method} timed out after ${timeoutMs}ms`)));
+ req.on('error', reject);
+ req.end(body === undefined ? undefined : JSON.stringify(body));
+ });
+}
+
+async function startServer(dbPath, readOnly = false) {
+ const child = fork(FIXTURE, [dbPath, readOnly ? 'read-only' : 'editable'], {
+ stdio: ['ignore', 'pipe', 'pipe', 'ipc']
+ });
+ const closed = new Promise(resolve => child.once('close', resolve));
+ child.stdout.resume();
+ let stderr = '';
+ child.stderr.on('data', data => { stderr += data; });
+ let ready;
+ try {
+ ready = await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error(`Server startup timed out: ${stderr}`)), 10000);
+ child.once('error', error => { clearTimeout(timer); reject(error); });
+ child.on('message', message => {
+ if (message.type === 'ready') { clearTimeout(timer); resolve(message); }
+ });
+ child.once('exit', code => { clearTimeout(timer); reject(new Error(`Server exited ${code}: ${stderr}`)); });
+ });
+ } catch (error) {
+ child.kill();
+ await closed;
+ throw error;
+ }
+ return {
+ url: ready.url,
+ mutationReceived() {
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { child.off('message', onMessage); reject(new Error('Mutation never reached server')); }, 5000);
+ function onMessage(message) {
+ if (message.type !== 'mutation-received') return;
+ clearTimeout(timer);
+ child.off('message', onMessage);
+ resolve();
+ }
+ child.on('message', onMessage);
+ });
+ },
+ async close() {
+ if (child.exitCode === null && child.signalCode === null) child.kill();
+ await closed;
+ }
+ };
+}
+
+async function withServer(readOnly, fn) {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-http-contention-'));
+ const dbPath = path.join(dir, 'state.db');
+ let server;
+ try {
+ const store = await createStateStore({ dbPath });
+ try {
+ store.upsertWorkItem({ id: 'task', title: 'Synthetic board task', source: 'manual', status: 'open' });
+ } finally {
+ store.close();
+ }
+ server = await startServer(dbPath, readOnly);
+ return await fn(server, dbPath);
+ } finally {
+ if (server) await server.close();
+ fs.rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+async function contentionScenario() {
+ return withServer(false, async (server, dbPath) => {
+ const bytesBefore = fs.readFileSync(dbPath);
+ const lockPath = `${dbPath}.ecc-state.lock`;
+ const fd = fs.openSync(lockPath, 'wx', 0o600);
+ fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, hostname: os.hostname() }));
+ let outcome;
+ try {
+ const received = server.mutationReceived();
+ let settled = false;
+ const mutation = request(`${server.url}/api/work-items/task/claim`, 'POST', { owner: 'alice' }, 12000)
+ .then(result => { settled = true; return result; }, error => { settled = true; return { transportError: error }; });
+ await received;
+ let healthError = null;
+ let healthChecks = 0;
+ let snapshotChecks = 0;
+ // Probe continuously after actual HTTP receipt, including asynchronous
+ // database initialization. No timing guess about when lock wait begins.
+ while (!settled) {
+ try {
+ const health = await request(`${server.url}/api/health`);
+ assert.strictEqual(health.status, 200);
+ healthChecks += 1;
+ const snapshot = await request(`${server.url}/api/snapshot`, 'GET', undefined, 3000);
+ assert.strictEqual(snapshot.status, 200);
+ assert.ok(snapshot.body.workItems.items.some(item => item.id === 'task'));
+ snapshotChecks += 1;
+ } catch (error) {
+ healthError = error.message;
+ break;
+ }
+ }
+ const result = await mutation;
+ if (result.transportError) throw result.transportError;
+ outcome = { result, healthError, healthChecks, snapshotChecks };
+ } finally {
+ fs.closeSync(fd);
+ fs.unlinkSync(lockPath);
+ }
+ assert.deepStrictEqual(fs.readFileSync(dbPath), bytesBefore, 'Timed-out mutation must not change database bytes');
+ const store = await createStateStore({ dbPath });
+ try {
+ const item = store.getWorkItemById('task');
+ assert.strictEqual(item.status, 'open');
+ assert.strictEqual(item.owner, null);
+ } finally {
+ store.close();
+ }
+ return outcome;
+ });
+}
+
+async function run() {
+ let passed = 0;
+ let failed = 0;
+ async function test(name, fn) {
+ try {
+ await fn();
+ console.log(` PASS ${name}`);
+ passed += 1;
+ } catch (error) {
+ console.log(` FAIL ${name}\n ${error.message}`);
+ failed += 1;
+ }
+ }
+ console.log('\n=== Testing control-pane mutation contention ===\n');
+ const contention = await contentionScenario();
+ await test('health and snapshots remain responsive while a board mutation waits for another database writer', () => {
+ assert.strictEqual(contention.healthError, null, contention.healthError || undefined);
+ assert.ok(contention.healthChecks > 0);
+ assert.ok(contention.snapshotChecks > 0);
+ });
+ await test('lock timeout is retryable HTTP 503 with Retry-After', () => {
+ assert.strictEqual(contention.result.status, 503);
+ assert.ok(Number(contention.result.headers['retry-after']) > 0, 'Retry-After must give a positive delay');
+ assert.strictEqual(contention.result.body.ok, false);
+ assert.strictEqual(contention.result.body.code, 'STATE_STORE_BUSY');
+ });
+ await test('claim and move still save their changes, while invalid edits return 400', () => withServer(false, async (server, dbPath) => {
+ const claim = await request(`${server.url}/api/work-items/task/claim`, 'POST', { owner: 'alice', as: 'human' }, 5000);
+ assert.strictEqual(claim.status, 200);
+ assert.strictEqual(claim.body.item.owner, 'alice');
+ assert.strictEqual(claim.body.item.status, 'running');
+ const move = await request(`${server.url}/api/work-items/task/move`, 'POST', { lane: 'blocked' }, 5000);
+ assert.strictEqual(move.status, 200);
+ assert.strictEqual(move.body.item.status, 'blocked');
+ const invalid = await request(`${server.url}/api/work-items/task/move`, 'POST', { lane: 'imaginary' }, 5000);
+ assert.strictEqual(invalid.status, 400);
+ assert.strictEqual(invalid.body.ok, false);
+ const store = await createStateStore({ dbPath });
+ try {
+ const item = store.getWorkItemById('task');
+ assert.strictEqual(item.owner, 'alice');
+ assert.strictEqual(item.status, 'blocked');
+ assert.strictEqual(item.metadata.assigneeKind, 'human');
+ } finally {
+ store.close();
+ }
+ }));
+ await test('read-only mode rejects board edits before database work', () => withServer(true, async server => {
+ for (const action of ['claim', 'move']) {
+ const result = await request(`${server.url}/api/work-items/task/${action}`, 'POST', { owner: 'alice', lane: 'done' });
+ assert.strictEqual(result.status, 403);
+ }
+ }));
+ console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
+ process.exitCode = failed ? 1 : 0;
+}
+run().catch(error => { console.error(error); process.exitCode = 1; });
diff --git a/tests/lib/helpers/control-pane-contention-server.js b/tests/lib/helpers/control-pane-contention-server.js
new file mode 100644
index 000000000..24f51cdc5
--- /dev/null
+++ b/tests/lib/helpers/control-pane-contention-server.js
@@ -0,0 +1,26 @@
+'use strict';
+
+const { createControlPaneServer } = require('../../../scripts/lib/control-pane/server');
+
+async function main() {
+ const app = createControlPaneServer({
+ host: '127.0.0.1', port: 0, stateDbPath: process.argv[2],
+ dbPath: `${process.argv[2]}.ecc2`, allowActions: process.argv[3] !== 'read-only'
+ });
+ app.server.prependListener('request', req => {
+ if (req.method === 'POST') process.send({ type: 'mutation-received' });
+ });
+ await app.listen();
+ process.send({ type: 'ready', url: app.url });
+ process.on('message', async message => {
+ if (message === 'close') {
+ await app.close();
+ process.disconnect();
+ }
+ });
+}
+main().catch(error => {
+ console.error(error);
+ process.exitCode = 1;
+ if (process.connected) process.disconnect();
+});
diff --git a/tests/lib/state-store-persistence.test.js b/tests/lib/state-store-persistence.test.js
index 00edf7ab4..ce7c702ca 100644
--- a/tests/lib/state-store-persistence.test.js
+++ b/tests/lib/state-store-persistence.test.js
@@ -50,6 +50,71 @@ async function run() {
assert.deepStrictEqual(fs.readFileSync(dbPath), before);
});
+ await test('persistent PRAGMA changes survive snapshot reload and reopening', async dbPath => {
+ const store = await createStateStore({ dbPath });
+ try {
+ assert.strictEqual(store._database.pragma('user_version = 3252'), undefined);
+ assert.strictEqual(store._database.prepare('PRAGMA user_version').get().user_version, 3252);
+ store._database.pragma('application_id(2468)');
+ } finally { store.close(); }
+ const reopened = await createStateStore({ dbPath });
+ try {
+ assert.strictEqual(reopened._database.prepare('PRAGMA user_version').get().user_version, 3252);
+ assert.strictEqual(reopened._database.prepare('PRAGMA application_id').get().application_id, 2468);
+ } finally { reopened.close(); }
+ });
+
+ await test('read-only and connection-local PRAGMAs never rewrite database bytes', async dbPath => {
+ const store = await createStateStore({ dbPath });
+ const before = fs.readFileSync(dbPath);
+ const rename = fs.renameSync;
+ let writes = 0;
+ fs.renameSync = (from, to) => {
+ if (to === dbPath) writes += 1;
+ return rename(from, to);
+ };
+ try {
+ store._database.pragma('user_version');
+ store._database.pragma('table_info(work_items)');
+ store._database.pragma('integrity_check');
+ store._database.transaction(() => {
+ store._database.pragma('user_version');
+ store._database.pragma('table_info(work_items)');
+ })();
+ store._database.withSnapshot(() => {
+ store._database.pragma('cache_size = 256');
+ assert.strictEqual(store._database.prepare('PRAGMA cache_size').get().cache_size, 256);
+ });
+ assert.strictEqual(writes, 0);
+ assert.deepStrictEqual(fs.readFileSync(dbPath), before);
+ } finally {
+ fs.renameSync = rename;
+ store.close();
+ }
+ });
+
+ await test('PRAGMA persistence waits for commit and never commits a rolled-back transaction', async dbPath => {
+ const store = await createStateStore({ dbPath });
+ try {
+ store._database.transaction(() => {
+ store.upsertWorkItem(item('committed-pragma'));
+ store._database.pragma('table_info(work_items)');
+ store._database.pragma('user_version = 3252');
+ })();
+ store._database.transaction(() => store._database.pragma('application_id(2468)'))();
+ assert.strictEqual(store._database.prepare('PRAGMA application_id').get().application_id, 2468);
+ const before = fs.readFileSync(dbPath);
+ assert.throws(() => store._database.transaction(() => {
+ store.upsertWorkItem(item('rolled-back-pragma'));
+ store._database.pragma('user_version = 9999');
+ throw new Error('abort PRAGMA transaction');
+ })(), /abort PRAGMA transaction/);
+ assert.deepStrictEqual(fs.readFileSync(dbPath), before);
+ assert.strictEqual(store._database.prepare('PRAGMA user_version').get().user_version, 3252);
+ assert.deepStrictEqual(store.listWorkItems().items.map(row => row.id), ['committed-pragma']);
+ } finally { store.close(); }
+ });
+
await test('failed persistence is discarded, releases the lock and cannot overwrite a later writer', async dbPath => {
const first = await createStateStore({ dbPath });
const second = await createStateStore({ dbPath });
diff --git a/tests/scripts/npm-publish-surface.test.js b/tests/scripts/npm-publish-surface.test.js
index 02ca28943..c053151d1 100644
--- a/tests/scripts/npm-publish-surface.test.js
+++ b/tests/scripts/npm-publish-surface.test.js
@@ -213,6 +213,9 @@ function main() {
"docs/design/context-carriers.md",
"docs/design/context-profile-delivery.md",
"scripts/control-pane.js",
+ "scripts/lib/control-pane/work-item-runner.js",
+ "scripts/lib/control-pane/work-item-worker.js",
+ "scripts/lib/state-store/file-lock.js",
"scripts/feedback.js",
"scripts/ito.js",
"scripts/memory.js",