Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(pi): recover cache locks after release failures

xuzini-xzn487eba9f8b8e
diff --git a/apps/desktop/src/main/maker-host/__tests__/pi-native-compat-overrides-store.test.ts b/apps/desktop/src/main/maker-host/__tests__/pi-native-compat-overrides-store.test.ts
index 5968bb8ece..231d85232f 100644
--- a/apps/desktop/src/main/maker-host/__tests__/pi-native-compat-overrides-store.test.ts
+++ b/apps/desktop/src/main/maker-host/__tests__/pi-native-compat-overrides-store.test.ts
@@ -90,7 +90,7 @@ describe('learned cache compatibility', () => {
// Real child processes execute the production bridge wrapper. A controlled
// rename barrier holds one writer after its disk read, reproducing lost updates
// without relying on scheduler timing; the other writer must observe the lock.
-function cacheProcess(id: string, options: { pause?: boolean; failWrite?: boolean } = {}) {
+function cacheProcess(id: string, options: { pause?: boolean; failWrite?: boolean; failRelease?: 'once' | 'held' } = {}) {
const release = path.join(root.path, 'release-' + id);
const script = `
import * as fs from 'node:fs';
@@ -108,6 +108,7 @@ const handlers = new Map();
const registry = { getProvider() {}, getRegisteredNativeProvider() {} };
let pendingTimers = 0;
let calls = 0, pause = ${Boolean(options.pause)}, failWrite = ${Boolean(options.failWrite)};
+let releaseFailure = ${JSON.stringify(options.failRelease ?? '')};
const api = runInNewContext(${JSON.stringify(PI_CACHE_RECOVERY_SOURCE + '\ninstallCindyCacheRecovery(pi)')}, {
...fs, path, process: { pid: process.pid, uptime: () => process.uptime(), kill: (...a) => process.kill(...a),
env: { PI_CODING_AGENT_DIR: '/fixture', CINDY_PI_CACHE_COMPAT_FILE: file } },
@@ -126,6 +127,13 @@ const api = runInNewContext(${JSON.stringify(PI_CACHE_RECOVERY_SOURCE + '\ninsta
if (failWrite && typeof name === 'string' && name.endsWith('.tmp')) throw new Error('fixture write failure');
return fs.writeFileSync(name, ...a);
},
+ rmSync: (name, ...args) => {
+ if (name === file + '.lock' && releaseFailure) {
+ if (releaseFailure === 'once') releaseFailure = '';
+ throw Object.assign(new Error('fixture release failure'), { code: 'EPERM' });
+ }
+ return fs.rmSync(name, ...args);
+ },
renameSync: (from, to) => {
if (to === file && pause) {
pause = false; process.send({ event: 'before-rename' });
@@ -159,7 +167,7 @@ async function request() {
createInterface({ input: process.stdin }).on('line', async command => {
try {
if (command === 'start') { start(); process.send({ event: 'success', ...await request() }); }
- if (command === 'next') { failWrite = false; start(); process.send({ event: 'next', ...await request() }); }
+ if (command === 'next') { failWrite = false; releaseFailure = ''; start(); process.send({ event: 'next', ...await request() }); }
if (command === 'stop') { process.stdin.destroy(); }
} catch (error) { process.send({ event: 'error', message: String(error) }); process.exitCode = 1; process.stdin.destroy(); }
});
@@ -245,6 +253,29 @@ describe('native and Desktop cache writers share one lock', () => {
} finally { await native.stop(); }
});
+ it.each(['once', 'held'] as const)('lets Desktop persist while Pi stays alive after %s release failure', async failRelease => {
+ const native = cacheProcess('release', { failRelease });
+ try {
+ await native.event('ready'); native.send('start');
+ const result = await native.event('success');
+ expect(result.events).toEqual(['done']); expect(result.calls).toBe(2);
+ if (failRelease === 'held') {
+ await native.event('persistence-idle');
+ expect(fs.existsSync(file + '.lock')).toBe(true);
+ // The successful learning is already durable, so there is no memory
+ // write to incidentally clean up this owner's abandoned lock.
+ native.send('next');
+ const next = await native.event('next');
+ expect(next.events).toEqual(['done']); expect(next.calls).toBe(3);
+ expect(next.sent).toEqual([{ model: 'release' }]);
+ }
+ await vi.waitFor(() => expect(fs.existsSync(file + '.lock')).toBe(false));
+ expect(native.child.exitCode).toBeNull();
+ expect(await record('desktop', 'after-release', fix, b)).toBe(true);
+ expect(Object.keys(JSON.parse(fs.readFileSync(file, 'utf8')).routes)).toHaveLength(2);
+ } finally { await native.stop(); fs.rmSync(file + '.lock', { force: true }); }
+ });
+
it.each(['busy', 'write', 'corrupt'])('keeps success and memory learning after %s persistence failure', async failure => {
if (failure === 'busy') fs.writeFileSync(file + '.lock', JSON.stringify({ pid: process.pid, nonce: 'fixture-owner', startedAt: Date.now() }));
if (failure === 'corrupt') fs.writeFileSync(file, '{bad-json');
diff --git a/packages/maker-core/src/agents/pi/__tests__/cache-recovery.test.ts b/packages/maker-core/src/agents/pi/__tests__/cache-recovery.test.ts
index 6201246778..5c3df0bcec 100644
--- a/packages/maker-core/src/agents/pi/__tests__/cache-recovery.test.ts
+++ b/packages/maker-core/src/agents/pi/__tests__/cache-recovery.test.ts
@@ -22,6 +22,8 @@ function fixture(legacy = false) {
const rules = [{ ...model, providerId: 'connection', routeKey }];
files.set('/agent/pi-cache-compatibility.json', JSON.stringify(rules));
let writeFails = false;
+ const removalFailures = new Map<string, (name: string) => string | undefined>();
+ const deadPids = new Set<number>();
const fs = {
mkdirSync() {}, rmdirSync() {}, closeSync() {}, readdirSync: () => [],
openSync: (name: string) => {
@@ -37,11 +39,17 @@ function fixture(legacy = false) {
readFileSync: (name: string) => { fs.statSync(name); return files.get(name); },
writeFileSync: (name: string, text: string) => { if (writeFails) throw new Error('read-only'); files.set(name, text); },
renameSync: (from: string, to: string) => { files.set(to, files.get(from)!); files.delete(from); },
- rmSync: (name: string) => { files.delete(name); },
+ rmSync: (name: string) => {
+ const code = (removalFailures.get(name) ?? removalFailures.get('*'))?.(name);
+ if (code) throw Object.assign(new Error('fixture removal failure'), { code });
+ files.delete(name);
+ },
};
const api = runInNewContext(PI_CACHE_RECOVERY_SOURCE + '\ninstallCindyCacheRecovery(pi)', {
pi: { on: (name: string, fn: (...args: any[]) => void) => handlers.set(name, fn) },
- process: { pid: 123, uptime: () => 1, kill() {}, env: { PI_CODING_AGENT_DIR: '/agent', CINDY_PI_CACHE_COMPAT_FILE: '/compat.json' } },
+ process: { pid: 123, uptime: () => 1, kill(pid: number) {
+ if (deadPids.has(pid)) throw Object.assign(new Error('dead'), { code: 'ESRCH' });
+ }, env: { PI_CODING_AGENT_DIR: '/agent', CINDY_PI_CACHE_COMPAT_FILE: '/compat.json' } },
// The in-memory filesystem uses POSIX keys on every host, including Windows.
...fs, path: path.posix, createHash, randomUUID, Buffer, setTimeout, clearTimeout, Date: { now: () => now }, homedir: () => '/home/fixture',
});
@@ -75,11 +83,73 @@ function fixture(legacy = false) {
for await (const event of api.wrap(selectedModel, options, invoke)) output.push(event);
return output;
};
- return { api, handlers, fresh, startUser, interrupt, request, files, sent, clock: (value: number) => { now = value; },
+ return { api, handlers, fresh, startUser, interrupt, request, files, sent, failRemovals: (name: string, failure: (name: string) => string | undefined) => removalFailures.set(name, failure), deadPid: (pid: number) => deadPids.add(pid), clock: (value: number) => { now = value; },
failWrites: () => { writeFails = true; }, record: () => JSON.parse(files.get('/compat.json') ?? '{"routes":{}}').routes[key] };
}
describe('Pi native cache recovery', () => {
+ it.each(['EPERM', 'EBUSY', 'EACCES'])('releases a live-owner lock after transient %s without replaying generation', async code => {
+ vi.useFakeTimers();
+ try {
+ const h = fixture();
+ let failures = 1;
+ h.failRemovals('/compat.json.lock', () => failures-- > 0 ? code : undefined);
+ h.fresh();
+ expect(await h.request()).toEqual([success]);
+ expect(h.files.has('/compat.json.lock')).toBe(true);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(h.files.has('/compat.json.lock')).toBe(false);
+ h.fresh(); await h.request([success]);
+ expect(h.sent).toHaveLength(3);
+ expect(h.sent[2]).not.toHaveProperty('prompt_cache_key');
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('recovers an exhausted release on the next input even after learning was persisted', async () => {
+ vi.useFakeTimers();
+ try {
+ const h = fixture(); let denied = true;
+ h.failRemovals('/compat.json.lock', () => denied ? 'EPERM' : undefined);
+ h.fresh(); expect(await h.request()).toEqual([success]);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(h.files.has('/compat.json.lock')).toBe(true);
+ expect(h.record()).toBeDefined();
+ expect(vi.getTimerCount()).toBe(0);
+ denied = false; h.fresh();
+ expect(h.files.has('/compat.json.lock')).toBe(false);
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('retries removal of its reclaim gate without leaving a live-owner gate behind', async () => {
+ vi.useFakeTimers();
+ try {
+ const h = fixture(); let failures = 1;
+ h.deadPid(42);
+ h.files.set('/compat.json.lock', JSON.stringify({ pid: 42, nonce: 'dead-owner' }));
+ h.failRemovals('*', name => name.includes('.reclaim.d/') && failures-- > 0 ? 'EBUSY' : undefined);
+ h.fresh(); expect(await h.request()).toEqual([success]);
+ expect([...h.files.keys()].some(name => name.includes('.reclaim.d/'))).toBe(true);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect([...h.files.keys()].some(name => name.includes('.reclaim.d/'))).toBe(false);
+ expect(h.files.has('/compat.json.lock')).toBe(false);
+ expect(h.record()).toBeDefined();
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('does not remove a successor nonce during delayed release or next-input recovery', async () => {
+ vi.useFakeTimers();
+ try {
+ const h = fixture();
+ h.failRemovals('/compat.json.lock', () => 'EPERM');
+ h.fresh(); expect(await h.request()).toEqual([success]);
+ const successor = JSON.stringify({ pid: 123, nonce: 'successor', startedAt: 1000000000 });
+ h.files.set('/compat.json.lock', successor);
+ h.failRemovals('/compat.json.lock', () => undefined);
+ await vi.advanceTimersByTimeAsync(1000); h.fresh();
+ expect(h.files.get('/compat.json.lock')).toBe(successor);
+ } finally { vi.useRealTimers(); }
+ });
+
it.each(['openai-completions', 'openai-responses'])('keeps learned payload recovery without taking provider ownership on an older %s registry', async api => {
const h = fixture(true);
const selected = { ...model, api };
diff --git a/packages/maker-core/src/agents/pi/cache-recovery-source.ts b/packages/maker-core/src/agents/pi/cache-recovery-source.ts
index fcc0135f86..67686fc002 100644
--- a/packages/maker-core/src/agents/pi/cache-recovery-source.ts
+++ b/packages/maker-core/src/agents/pi/cache-recovery-source.ts
@@ -12,6 +12,7 @@ function installCindyCacheRecovery(pi) {
let pendingInput;
let directInput = false;
let memory = Object.create(null);
+ const pendingReleases = new Map();
let routeRules = new Map();
function rules() {
try {
@@ -112,6 +113,39 @@ function installCindyCacheRecovery(pi) {
|| a.name.localeCompare(b.name));
return active.length > 0 && active[0].name !== ownGate;
}
+ async function releaseOwned(name, nonce) {
+ pendingReleases.set(name, nonce);
+ for (let attempt = 0; attempt < 5; attempt++) {
+ try {
+ const snapshot = lockSnapshot(name);
+ if (!snapshot) {
+ // An unreadable record is not proof that our lock disappeared.
+ try { lstatSync(name); } catch (error) {
+ if (error?.code === 'ENOENT' && pendingReleases.get(name) === nonce) pendingReleases.delete(name);
+ }
+ return;
+ }
+ if (snapshot.record?.nonce !== nonce) {
+ if (pendingReleases.get(name) === nonce) pendingReleases.delete(name);
+ return;
+ }
+ // The ownership check and removal stay synchronous. A delayed retry
+ // must never remove a successor belonging to this same live PID.
+ if (name !== file + '.lock' || !gateBusy(name)) {
+ rmSync(name);
+ if (pendingReleases.get(name) === nonce) pendingReleases.delete(name);
+ return;
+ }
+ } catch (error) {
+ if (!['EPERM', 'EBUSY', 'EACCES'].includes(error?.code)) return;
+ }
+ if (attempt < 4) await new Promise(resolve => setTimeout(resolve, 40 * (attempt + 1)));
+ }
+ // Keep only the nonce we can prove we own, for the next input/write.
+ }
+ function recoverReleases() {
+ for (const [name, nonce] of pendingReleases) void releaseOwned(name, nonce);
+ }
function reclaim(lock) {
const candidate = lockSnapshot(lock);
if (!stale(candidate)) return;
@@ -129,9 +163,9 @@ function installCindyCacheRecovery(pi) {
if (sameLock(lockSnapshot(isolated), candidate)) rmSync(isolated);
else { try { linkSync(isolated, lock); rmSync(isolated); } catch {} }
} finally {
- // Unique nonce names are never reused; a failed cleanup is recoverable
- // after process exit, and cannot remove a different owner's gate.
- try { rmSync(gate); } catch {}
+ // Unique gate names are never reused; retry cleanup without leaving a
+ // live-owner gate behind or removing a different owner's gate.
+ void releaseOwned(gate, nonce);
try { rmdirSync(directory); } catch {}
}
}
@@ -143,6 +177,7 @@ function installCindyCacheRecovery(pi) {
let held = false;
let temporary;
try {
+ recoverReleases();
mkdirSync(path.dirname(file), { recursive: true });
do {
try {
@@ -177,9 +212,7 @@ function installCindyCacheRecovery(pi) {
} catch { /* Storage failure must never undo a successful response. */ }
finally {
if (temporary) { try { rmSync(temporary, { force: true }); } catch {} }
- if (held) {
- try { if (lockSnapshot(lock)?.record?.nonce === nonce) rmSync(lock); } catch {}
- }
+ if (held) void releaseOwned(lock, nonce);
}
}
function remember(key, evidence, startedAt) {
@@ -294,6 +327,7 @@ function installCindyCacheRecovery(pi) {
// Pi represents both steer and followUp as user messages. Snapshot now,
// but decide ownership only after the supervisor sees the public event.
invalidate();
+ recoverReleases();
for (const [key, record] of Object.entries(memory)) void persist(key, record);
const inputRules = readRules();
const records = load()?.routes ?? Object.create(null);