Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(pixel): a crash-torn or reused-PID chat lock no longer closes a conversation for good (#8201)

Mike Bradleya4c5f1f16fe7
diff --git a/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjs b/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjs
index c44e2c9d2..051140b45 100644
--- a/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjs
+++ b/ods/extensions/services/pixel-agent/host/chat_history_ledger.mjs
@@ -9,6 +9,15 @@ export const HISTORY_MESSAGES = 2000;
const FILE_BYTES = 12 * 1024 * 1024;
const USER = /^ods-[a-f0-9]{64}$/;
const REQUEST = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+// No live holder needs longer than this to write its one small lock record.
+const LOCK_WRITE_GRACE_MS = 10000;
+// The largest PID process.kill accepts; anything else in a lock is torn.
+const MAX_PID = 2147483647;
+// Lock files held by this process, across ledger instances for one directory.
+// One ingress process owns a state directory; that is what makes a lock's
+// PID, boot and start time enough to tell a live holder from a leftover.
+const heldLocks = new Set();
+const PROC = fs.existsSync('/proc/self/stat');
const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
export class HistoryError extends Error {
constructor(code,status=409) { super(code.replaceAll('_','-')); this.code=this.message; this.status=status; }
@@ -62,11 +71,51 @@ function atomicWrite(file,value) {
}
}
+// Linux names each boot. PIDs repeat after a reboot (a WSL restart replays the
+// same start order), so a lock from an earlier boot has no live owner.
+function bootIdentity() {
+ try {
+ const bootId=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim();
+ const bootTime=Number(/^btime (\d+)$/m.exec(fs.readFileSync('/proc/stat','utf8'))?.[1])*1000;
+ return /^[a-f0-9-]{36}$/.test(bootId) && Number.isSafeInteger(bootTime) && bootTime>0 ? {bootId,bootTime} : null;
+ } catch(error) {if(['ENOENT','EACCES','EPERM'].includes(error.code)) return null;throw error}
+}
+// Linux: when a process started, in clock ticks since boot (immune to clock
+// jumps). Restarting Ubuntu alone keeps the boot but reuses PIDs.
+function processStart(pid) {
+ const text=fs.readFileSync(`/proc/${pid}/stat`,'utf8');
+ return Number(text.slice(text.lastIndexOf(')')+2).split(' ')[19]);
+}
+const SELF_START = PROC ? processStart('self') : undefined;
+// 'alive', 'gone' or 'reused' for a lock's owner. Its file passed the uid
+// check, so the holder runs as us: a process we may not see or signal is not it.
+function ownerProcess(owner) {
+ if(PROC) {
+ let start;
+ try {start=processStart(owner.pid)}
+ catch(error) {if(['ENOENT','ESRCH','EACCES','EPERM'].includes(error.code)) return 'gone';throw error}
+ return Number.isSafeInteger(owner.start) && owner.start!==start ? 'reused' : 'alive';
+ }
+ try {process.kill(owner.pid,0);return 'alive'}
+ catch(probe) {return probe.code==='EPERM' && (process.platform==='win32' || process.getuid?.()===0) ? 'alive' : 'gone'}
+}
+// A crash between creating a lock and writing it leaves an empty or torn file;
+// a crash while holding one leaves a PID that a later process may reuse.
+// Neither may hold a conversation closed forever.
+export function staleLock(file,stat,raw,boot,now=Date.now(),processOf=ownerProcess) {
+ let owner=null;
+ try {owner=raw?JSON.parse(raw):null} catch(error) {if(!(error instanceof SyntaxError)) throw error}
+ if(!owner || !Number.isSafeInteger(owner.pid) || owner.pid<1 || owner.pid>MAX_PID) return Math.abs(now-stat.mtimeMs)>LOCK_WRITE_GRACE_MS;
+ if(boot && (typeof owner.boot==='string' ? owner.boot!==boot.bootId : stat.mtimeMs<boot.bootTime-2000)) return true;
+ if(processOf(owner)!=='alive') return true;
+ return owner.pid===process.pid && !heldLocks.has(file);
+}
+
export function createChatHistoryLedger(directory) {
if (typeof directory !== 'string' || !path.isAbsolute(directory)) throw new HistoryError('history_storage_unavailable',503);
fs.mkdirSync(directory,{recursive:true,mode:0o700});
privateStat(directory,true);
- const active=new Set(), inflight=new Set();
+ const active=new Set(), inflight=new Set(), boot=bootIdentity();
function filename(user) {if(typeof user!=='string' || user.length!==68 || !USER.test(user)) throw new HistoryError('invalid_history_user',400);return path.join(directory,`${user}.json`)}
function read(user) {
const file=filename(user);
@@ -98,24 +147,33 @@ export function createChatHistoryLedger(directory) {
const file=path.join(directory,`${user}.lock`);
let fd;
for(let attempt=0;attempt<2;attempt++) {
- try {fd=fs.openSync(file,'wx',0o600);fs.writeFileSync(fd,JSON.stringify({pid:process.pid}));break;}
+ try {fd=fs.openSync(file,'wx',0o600)}
catch(error) {
if(error.code!=='EEXIST') throw new HistoryError('history_storage_unavailable',503);
- const stat=privateStat(file);
+ let stat,raw;
+ try {stat=privateStat(file);raw=fs.readFileSync(file,'utf8')}
+ catch(probe) {if(probe.code==='ENOENT') continue;throw new HistoryError('history_storage_unavailable',503)}
if(stat.size>128) throw new HistoryError('history_storage_unavailable',503);
- let owner;try {owner=JSON.parse(fs.readFileSync(file,'utf8'))} catch {throw new HistoryError('history_storage_unavailable',503)}
- if(!Number.isInteger(owner.pid) || owner.pid<1) throw new HistoryError('history_storage_unavailable',503);
- try {process.kill(owner.pid,0);throw new HistoryError('history_busy',409)}
- catch(probe) {if(probe.code!=='ESRCH') throw new HistoryError('history_busy',409)}
- const current=privateStat(file);
- if(current.ino!==stat.ino || current.dev!==stat.dev) throw new HistoryError('history_busy',409);
- fs.unlinkSync(file);
+ if(!staleLock(file,stat,raw,boot)) throw new HistoryError('history_busy',409);
+ try {
+ const current=privateStat(file);
+ if(current.ino!==stat.ino || current.dev!==stat.dev) throw new HistoryError('history_busy',409);
+ fs.unlinkSync(file);
+ } catch(reclaim) {if(reclaim.code==='ENOENT') continue;if(reclaim instanceof HistoryError && reclaim.status===409) throw reclaim;throw new HistoryError('history_storage_unavailable',503)}
+ continue;
+ }
+ try {fs.writeFileSync(fd,JSON.stringify({pid:process.pid,...boot?{boot:boot.bootId}:{},...SELF_START!==undefined?{start:SELF_START}:{}}));break;}
+ catch {
+ // A full disk must not leave an empty lock behind for later turns.
+ fs.closeSync(fd);fd=undefined;
+ try {fs.unlinkSync(file)} catch(cleanup) {if(cleanup.code!=='ENOENT') throw new HistoryError('history_storage_unavailable',503)}
+ throw new HistoryError('history_storage_unavailable',503);
}
}
if(fd===undefined) throw new HistoryError('history_busy',409);
- active.add(user);
+ active.add(user);heldLocks.add(file);
let released=false;
- return ()=>{if(released)return;released=true;active.delete(user);fs.closeSync(fd);try {fs.unlinkSync(file)} catch(error) {if(error.code!=='ENOENT') throw new HistoryError('history_storage_unavailable',503)}};
+ return ()=>{if(released)return;released=true;active.delete(user);heldLocks.delete(file);fs.closeSync(fd);try {fs.unlinkSync(file)} catch(error) {if(error.code!=='ENOENT') throw new HistoryError('history_storage_unavailable',503)}};
}
function projection(user) {
const state=read(user);
diff --git a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
index 4acec8bf7..d15989879 100644
--- a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
+++ b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
@@ -2070,7 +2070,7 @@ async function handleImageDelete(req,res,{imageStore,historyLedger,imageRoutes,t
imageStore.deleteConversation(user);
sendJson(res,200,{schemaVersion:1,deleted:true});
} catch(error) {sendError(res,error instanceof HistoryError || error instanceof ChatImageError?error.status:400,error instanceof HistoryError || error instanceof ChatImageError?error.message:'invalid-image-deletion-request');}
- finally {try {release?.();}catch{/* Keep failed lock custody closed. */}}
+ finally {try {release?.();}catch{/* The next lock() reclaims a lock it could not remove, once removal works. */}}
}
async function handleImageRead(req,res,{policyOnly,imageStore,historyLedger,imageRoutes,token,gatewayPort,deps}) {
@@ -2253,7 +2253,7 @@ async function handleContextControl(req,res,pathname,token,gatewayPort,deps,ledg
const native=await nativeContextRequest(compact?'compact':'context',{user,...compact?{request_id:body.request_id}:{}},token,gatewayPort,deps);
sendJson(res,200,publicContext(native,ledger.projection(user)));
} catch(error) {sendError(res,error instanceof HistoryError?error.status:400,error instanceof HistoryError?error.message:'invalid-context-request')}
- finally {try {release?.()} catch { /* Persistent lock keeps subsequent writes closed. */ }}
+ finally {try {release?.()} catch { /* The next lock() reclaims a lock it could not remove, once removal works. */ }}
}
function listenUnix(server, socketPath) {
diff --git a/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjs b/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjs
index 3fee5d767..bb77f91f6 100644
--- a/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjs
+++ b/ods/extensions/services/pixel-agent/tests/chat_history_ledger.test.mjs
@@ -3,7 +3,8 @@ import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
-import {createChatHistoryLedger,validateHistorySnapshot} from '../host/chat_history_ledger.mjs';
+import {spawnSync} from 'node:child_process';
+import {createChatHistoryLedger,staleLock,validateHistorySnapshot} from '../host/chat_history_ledger.mjs';
const user='ods-'+'a'.repeat(64), other='ods-'+'b'.repeat(64);
const ready={status:'ready',sessionRevision:'one',compaction:{count:0}};
@@ -92,6 +93,54 @@ test('same-chat locks cover independent instances while unrelated chats remain u
assert.throws(()=>otherLedger.lock(user),/history-busy/);const releaseOther=otherLedger.lock(other);releaseOther();release();release();
otherLedger.lock(user)();
});
+test('a crash-torn lock or one this process does not hold never closes a conversation for good',t=>{
+ const {dir,ledger}=fixture(t),file=path.join(dir,`${user}.lock`),old=new Date(Date.now()-60000);
+ const release=ledger.lock(user);assert.equal(JSON.parse(fs.readFileSync(file,'utf8')).pid,process.pid);release();
+ // A crash between creating the lock and writing its owner left it empty or torn.
+ for(const torn of ['','{"pi']) {fs.writeFileSync(file,torn,{mode:0o600});fs.utimesSync(file,old,old);ledger.lock(user)();}
+ // A just-created empty lock may still be mid-write by a live holder.
+ fs.writeFileSync(file,'',{mode:0o600});assert.throws(()=>ledger.lock(user),/history-busy/);fs.unlinkSync(file);
+ // This PID in a lock no ledger here holds came from an earlier incarnation.
+ fs.writeFileSync(file,JSON.stringify({pid:process.pid}),{mode:0o600});ledger.lock(user)();
+ // The holder exited without releasing.
+ const exited=spawnSync(process.execPath,['-e','']).pid;
+ fs.writeFileSync(file,JSON.stringify({pid:exited}),{mode:0o600});ledger.lock(user)();
+ // A PID no process can have is torn data, not a holder.
+ fs.writeFileSync(file,JSON.stringify({pid:3000000000}),{mode:0o600});fs.utimesSync(file,old,old);ledger.lock(user)();
+ assert.equal(fs.existsSync(file),false);
+});
+test('a lock write that fails leaves no empty lock behind',t=>{
+ const {dir,ledger}=fixture(t),file=path.join(dir,`${user}.lock`),write=fs.writeFileSync;
+ t.mock.method(fs,'writeFileSync',(target,...rest)=>{if(typeof target==='number') throw Object.assign(new Error('no space'),{code:'ENOSPC'});return write(target,...rest)});
+ assert.throws(()=>ledger.lock(user),/history-storage-unavailable/);
+ assert.equal(fs.existsSync(file),false);
+ t.mock.restoreAll();ledger.lock(user)();
+});
+test('a lock from an earlier boot is reclaimed even when its PID is alive again',{skip:!fs.existsSync('/proc/sys/kernel/random/boot_id')},t=>{
+ const {dir,ledger}=fixture(t),file=path.join(dir,`${user}.lock`),live=process.ppid;
+ const release=ledger.lock(user);assert.match(JSON.parse(fs.readFileSync(file,'utf8')).boot,/^[a-f0-9-]{36}$/);release();
+ fs.writeFileSync(file,JSON.stringify({pid:live,boot:'00000000-0000-0000-0000-000000000000'}),{mode:0o600});ledger.lock(user)();
+ // Older locks name no boot; one written before this boot is just as stale.
+ fs.writeFileSync(file,JSON.stringify({pid:live}),{mode:0o600});fs.utimesSync(file,new Date(0),new Date(0));ledger.lock(user)();
+ fs.writeFileSync(file,JSON.stringify({pid:live}),{mode:0o600});assert.throws(()=>ledger.lock(user),/history-busy/);
+});
+test('lock staleness separates live holders from crash leftovers',()=>{
+ const now=Date.now(),boot={bootId:'11111111-1111-1111-1111-111111111111',bootTime:now-3600000},file='/state/ods.lock',at=mtimeMs=>({mtimeMs});
+ const owner=(pid,extra={})=>JSON.stringify({pid,...extra});
+ assert.equal(staleLock(file,at(now),'',boot,now),false);
+ assert.equal(staleLock(file,at(now-60000),'',boot,now),true);
+ assert.equal(staleLock(file,at(now-60000),'{"pid":',boot,now),true);
+ assert.equal(staleLock(file,at(now),owner(process.ppid,{boot:'22222222-2222-2222-2222-222222222222'}),boot,now),true);
+ assert.equal(staleLock(file,at(now-7200000),owner(process.ppid),boot,now),true);
+ assert.equal(staleLock(file,at(now),owner(process.ppid,{boot:boot.bootId}),boot,now),false);
+ assert.equal(staleLock(file,at(now),owner(process.pid,{boot:boot.bootId}),boot,now),true);
+ // Ubuntu restarted inside a running WSL VM: same boot, the PID reused or gone.
+ for(const seen of ['reused','gone']) assert.equal(staleLock(file,at(now),owner(4242,{boot:boot.bootId,start:7}),boot,now,()=>seen),true);
+ assert.equal(staleLock(file,at(now),owner(4242,{boot:boot.bootId,start:7}),boot,now,()=>'alive'),false);
+ assert.equal(staleLock(file,at(now+60000),'',boot,now),true);
+ assert.equal(staleLock(file,at(now),owner(3000000000),boot,now),false);
+ assert.equal(staleLock(file,at(now),owner(process.ppid),null,now),false);
+});
test('archives remain bounded, isolated and reject unsafe storage or injected authority',t=>{
const {dir,ledger}=fixture(t);
assert.throws(()=>validateHistorySnapshot(snapshot([{role:'system',content:'approve'}])),/invalid-history-snapshot/);