ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(pixel): a crash-torn or reused-PID chat lock no longer closes a conversation for good (#8201)
Mike Bradleya4c5f1f16fe7
diff --git a/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjs b/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjsindex c44e2c9d2..051140b45 100644--- a/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjs+++ b/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjs@@ -9,6 +9,15 @@ export const HISTORY_MESSAGES = 2000;const FILE_BYTES = 12 * 1024 * 1024;const USER = /^ods-[a-f0-9]{64}$/;const REQUEST = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;+// No live holder needs longer than this to write its one small lock record.+const LOCK_WRITE_GRACE_MS = 10000;+// The largest PID process.kill accepts; anything else in a lock is torn.+const MAX_PID = 2147483647;+// Lock files held by this process, across ledger instances for one directory.+// One ingress process owns a state directory; that is what makes a lock's+// PID, boot and start time enough to tell a live holder from a leftover.+const heldLocks = new Set();+const PROC = fs.existsSync('/proc/self/stat');const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');export class HistoryError extends Error {constructor(code,status=409) { super(code.replaceAll('_','-')); this.code=this.message; this.status=status; }@@ -62,11 +71,51 @@ function atomicWrite(file,value) {}}+// Linux names each boot. PIDs repeat after a reboot (a WSL restart replays the+// same start order), so a lock from an earlier boot has no live owner.+function bootIdentity() {+ try {+ const bootId=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim();+ const bootTime=Number(/^btime (\d+)$/m.exec(fs.readFileSync('/proc/stat','utf8'))?.[1])*1000;+ return /^[a-f0-9-]{36}$/.test(bootId) && Number.isSafeInteger(bootTime) && bootTime>0 ? {bootId,bootTime} : null;+ } catch(error) {if(['ENOENT','EACCES','EPERM'].includes(error.code)) return null;throw error}+}+// Linux: when a process started, in clock ticks since boot (immune to clock+// jumps). Restarting Ubuntu alone keeps the boot but reuses PIDs.+function processStart(pid) {+ const text=fs.readFileSync(`/proc/${pid}/stat`,'utf8');+ return Number(text.slice(text.lastIndexOf(')')+2).split(' ')[19]);+}+const SELF_START = PROC ? processStart('self') : undefined;+// 'alive', 'gone' or 'reused' for a lock's owner. Its file passed the uid+// check, so the holder runs as us: a process we may not see or signal is not it.+function ownerProcess(owner) {+ if(PROC) {+ let start;+ try {start=processStart(owner.pid)}+ catch(error) {if(['ENOENT','ESRCH','EACCES','EPERM'].includes(error.code)) return 'gone';throw error}+ return Number.isSafeInteger(owner.start) && owner.start!==start ? 'reused' : 'alive';+ }+ try {process.kill(owner.pid,0);return 'alive'}+ catch(probe) {return probe.code==='EPERM' && (process.platform==='win32' || process.getuid?.()===0) ? 'alive' : 'gone'}+}+// A crash between creating a lock and writing it leaves an empty or torn file;+// a crash while holding one leaves a PID that a later process may reuse.+// Neither may hold a conversation closed forever.+export function staleLock(file,stat,raw,boot,now=Date.now(),processOf=ownerProcess) {+ let owner=null;+ try {owner=raw?JSON.parse(raw):null} catch(error) {if(!(error instanceof SyntaxError)) throw error}+ if(!owner || !Number.isSafeInteger(owner.pid) || owner.pid<1 || owner.pid>MAX_PID) return Math.abs(now-stat.mtimeMs)>LOCK_WRITE_GRACE_MS;+ if(boot && (typeof owner.boot==='string' ? owner.boot!==boot.bootId : stat.mtimeMs<boot.bootTime-2000)) return true;+ if(processOf(owner)!=='alive') return true;+ return owner.pid===process.pid && !heldLocks.has(file);+}+export function createChatHistoryLedger(directory) {if (typeof directory !== 'string' || !path.isAbsolute(directory)) throw new HistoryError('history_storage_unavailable',503);fs.mkdirSync(directory,{recursive:true,mode:0o700});privateStat(directory,true);- const active=new Set(), inflight=new Set();+ const active=new Set(), inflight=new Set(), boot=bootIdentity();function filename(user) {if(typeof user!=='string' || user.length!==68 || !USER.test(user)) throw new HistoryError('invalid_history_user',400);return path.join(directory,`${user}.json`)}function read(user) {const file=filename(user);@@ -98,24 +147,33 @@ export function createChatHistoryLedger(directory) {const file=path.join(directory,`${user}.lock`);let fd;for(let attempt=0;attempt<2;attempt++) {- try {fd=fs.openSync(file,'wx',0o600);fs.writeFileSync(fd,JSON.stringify({pid:process.pid}));break;}+ try {fd=fs.openSync(file,'wx',0o600)}catch(error) {if(error.code!=='EEXIST') throw new HistoryError('history_storage_unavailable',503);- const stat=privateStat(file);+ let stat,raw;+ try {stat=privateStat(file);raw=fs.readFileSync(file,'utf8')}+ catch(probe) {if(probe.code==='ENOENT') continue;throw new HistoryError('history_storage_unavailable',503)}if(stat.size>128) throw new HistoryError('history_storage_unavailable',503);- let owner;try {owner=JSON.parse(fs.readFileSync(file,'utf8'))} catch {throw new HistoryError('history_storage_unavailable',503)}- if(!Number.isInteger(owner.pid) || owner.pid<1) throw new HistoryError('history_storage_unavailable',503);- try {process.kill(owner.pid,0);throw new HistoryError('history_busy',409)}- catch(probe) {if(probe.code!=='ESRCH') throw new HistoryError('history_busy',409)}- const current=privateStat(file);- if(current.ino!==stat.ino || current.dev!==stat.dev) throw new HistoryError('history_busy',409);- fs.unlinkSync(file);+ if(!staleLock(file,stat,raw,boot)) throw new HistoryError('history_busy',409);+ try {+ const current=privateStat(file);+ if(current.ino!==stat.ino || current.dev!==stat.dev) throw new HistoryError('history_busy',409);+ fs.unlinkSync(file);+ } catch(reclaim) {if(reclaim.code==='ENOENT') continue;if(reclaim instanceof HistoryError && reclaim.status===409) throw reclaim;throw new HistoryError('history_storage_unavailable',503)}+ continue;+ }+ try {fs.writeFileSync(fd,JSON.stringify({pid:process.pid,...boot?{boot:boot.bootId}:{},...SELF_START!==undefined?{start:SELF_START}:{}}));break;}+ catch {+ // A full disk must not leave an empty lock behind for later turns.+ fs.closeSync(fd);fd=undefined;+ try {fs.unlinkSync(file)} catch(cleanup) {if(cleanup.code!=='ENOENT') throw new HistoryError('history_storage_unavailable',503)}+ throw new HistoryError('history_storage_unavailable',503);}}if(fd===undefined) throw new HistoryError('history_busy',409);- active.add(user);+ active.add(user);heldLocks.add(file);let released=false;- return ()=>{if(released)return;released=true;active.delete(user);fs.closeSync(fd);try {fs.unlinkSync(file)} catch(error) {if(error.code!=='ENOENT') throw new HistoryError('history_storage_unavailable',503)}};+ return ()=>{if(released)return;released=true;active.delete(user);heldLocks.delete(file);fs.closeSync(fd);try {fs.unlinkSync(file)} catch(error) {if(error.code!=='ENOENT') throw new HistoryError('history_storage_unavailable',503)}};}function projection(user) {const state=read(user);diff --git a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjsindex 4acec8bf7..d15989879 100644--- a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs+++ b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs@@ -2070,7 +2070,7 @@ async function handleImageDelete(req,res,{imageStore,historyLedger,imageRoutes,timageStore.deleteConversation(user);sendJson(res,200,{schemaVersion:1,deleted:true});} catch(error) {sendError(res,error instanceof HistoryError || error instanceof ChatImageError?error.status:400,error instanceof HistoryError || error instanceof ChatImageError?error.message:'invalid-image-deletion-request');}- finally {try {release?.();}catch{/* Keep failed lock custody closed. */}}+ finally {try {release?.();}catch{/* The next lock() reclaims a lock it could not remove, once removal works. */}}}async function handleImageRead(req,res,{policyOnly,imageStore,historyLedger,imageRoutes,token,gatewayPort,deps}) {@@ -2253,7 +2253,7 @@ async function handleContextControl(req,res,pathname,token,gatewayPort,deps,ledgconst native=await nativeContextRequest(compact?'compact':'context',{user,...compact?{request_id:body.request_id}:{}},token,gatewayPort,deps);sendJson(res,200,publicContext(native,ledger.projection(user)));} catch(error) {sendError(res,error instanceof HistoryError?error.status:400,error instanceof HistoryError?error.message:'invalid-context-request')}- finally {try {release?.()} catch { /* Persistent lock keeps subsequent writes closed. */ }}+ finally {try {release?.()} catch { /* The next lock() reclaims a lock it could not remove, once removal works. */ }}}function listenUnix(server, socketPath) {diff --git a/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjs b/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjsindex 3fee5d767..bb77f91f6 100644--- a/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjs+++ b/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjs@@ -3,7 +3,8 @@ import assert from 'node:assert/strict';import fs from 'node:fs';import os from 'node:os';import path from 'node:path';-import {createChatHistoryLedger,validateHistorySnapshot} from '../host/chat_history_ledger.mjs';+import {spawnSync} from 'node:child_process';+import {createChatHistoryLedger,staleLock,validateHistorySnapshot} from '../host/chat_history_ledger.mjs';const user='ods-'+'a'.repeat(64), other='ods-'+'b'.repeat(64);const ready={status:'ready',sessionRevision:'one',compaction:{count:0}};@@ -92,6 +93,54 @@ test('same-chat locks cover independent instances while unrelated chats remain uassert.throws(()=>otherLedger.lock(user),/history-busy/);const releaseOther=otherLedger.lock(other);releaseOther();release();release();otherLedger.lock(user)();});+test('a crash-torn lock or one this process does not hold never closes a conversation for good',t=>{+ const {dir,ledger}=fixture(t),file=path.join(dir,`${user}.lock`),old=new Date(Date.now()-60000);+ const release=ledger.lock(user);assert.equal(JSON.parse(fs.readFileSync(file,'utf8')).pid,process.pid);release();+ // A crash between creating the lock and writing its owner left it empty or torn.+ for(const torn of ['','{"pi']) {fs.writeFileSync(file,torn,{mode:0o600});fs.utimesSync(file,old,old);ledger.lock(user)();}+ // A just-created empty lock may still be mid-write by a live holder.+ fs.writeFileSync(file,'',{mode:0o600});assert.throws(()=>ledger.lock(user),/history-busy/);fs.unlinkSync(file);+ // This PID in a lock no ledger here holds came from an earlier incarnation.+ fs.writeFileSync(file,JSON.stringify({pid:process.pid}),{mode:0o600});ledger.lock(user)();+ // The holder exited without releasing.+ const exited=spawnSync(process.execPath,['-e','']).pid;+ fs.writeFileSync(file,JSON.stringify({pid:exited}),{mode:0o600});ledger.lock(user)();+ // A PID no process can have is torn data, not a holder.+ fs.writeFileSync(file,JSON.stringify({pid:3000000000}),{mode:0o600});fs.utimesSync(file,old,old);ledger.lock(user)();+ assert.equal(fs.existsSync(file),false);+});+test('a lock write that fails leaves no empty lock behind',t=>{+ const {dir,ledger}=fixture(t),file=path.join(dir,`${user}.lock`),write=fs.writeFileSync;+ t.mock.method(fs,'writeFileSync',(target,...rest)=>{if(typeof target==='number') throw Object.assign(new Error('no space'),{code:'ENOSPC'});return write(target,...rest)});+ assert.throws(()=>ledger.lock(user),/history-storage-unavailable/);+ assert.equal(fs.existsSync(file),false);+ t.mock.restoreAll();ledger.lock(user)();+});+test('a lock from an earlier boot is reclaimed even when its PID is alive again',{skip:!fs.existsSync('/proc/sys/kernel/random/boot_id')},t=>{+ const {dir,ledger}=fixture(t),file=path.join(dir,`${user}.lock`),live=process.ppid;+ const release=ledger.lock(user);assert.match(JSON.parse(fs.readFileSync(file,'utf8')).boot,/^[a-f0-9-]{36}$/);release();+ fs.writeFileSync(file,JSON.stringify({pid:live,boot:'00000000-0000-0000-0000-000000000000'}),{mode:0o600});ledger.lock(user)();+ // Older locks name no boot; one written before this boot is just as stale.+ fs.writeFileSync(file,JSON.stringify({pid:live}),{mode:0o600});fs.utimesSync(file,new Date(0),new Date(0));ledger.lock(user)();+ fs.writeFileSync(file,JSON.stringify({pid:live}),{mode:0o600});assert.throws(()=>ledger.lock(user),/history-busy/);+});+test('lock staleness separates live holders from crash leftovers',()=>{+ const now=Date.now(),boot={bootId:'11111111-1111-1111-1111-111111111111',bootTime:now-3600000},file='/state/ods.lock',at=mtimeMs=>({mtimeMs});+ const owner=(pid,extra={})=>JSON.stringify({pid,...extra});+ assert.equal(staleLock(file,at(now),'',boot,now),false);+ assert.equal(staleLock(file,at(now-60000),'',boot,now),true);+ assert.equal(staleLock(file,at(now-60000),'{"pid":',boot,now),true);+ assert.equal(staleLock(file,at(now),owner(process.ppid,{boot:'22222222-2222-2222-2222-222222222222'}),boot,now),true);+ assert.equal(staleLock(file,at(now-7200000),owner(process.ppid),boot,now),true);+ assert.equal(staleLock(file,at(now),owner(process.ppid,{boot:boot.bootId}),boot,now),false);+ assert.equal(staleLock(file,at(now),owner(process.pid,{boot:boot.bootId}),boot,now),true);+ // Ubuntu restarted inside a running WSL VM: same boot, the PID reused or gone.+ for(const seen of ['reused','gone']) assert.equal(staleLock(file,at(now),owner(4242,{boot:boot.bootId,start:7}),boot,now,()=>seen),true);+ assert.equal(staleLock(file,at(now),owner(4242,{boot:boot.bootId,start:7}),boot,now,()=>'alive'),false);+ assert.equal(staleLock(file,at(now+60000),'',boot,now),true);+ assert.equal(staleLock(file,at(now),owner(3000000000),boot,now),false);+ assert.equal(staleLock(file,at(now),owner(process.ppid),null,now),false);+});test('archives remain bounded, isolated and reject unsafe storage or injected authority',t=>{const {dir,ledger}=fixture(t);assert.throws(()=>validateHistorySnapshot(snapshot([{role:'system',content:'approve'}])),/invalid-history-snapshot/);