Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
Bound metadata checkpoint scanning without reading file bodies
wyt01019a41f6a4
diff --git a/scripts/test-checkpoint.mjs b/scripts/test-checkpoint.mjsindex 917417ab..fa57ff5d 100644--- a/scripts/test-checkpoint.mjs+++ b/scripts/test-checkpoint.mjs@@ -65,7 +65,7 @@ async function stderrMatches(file, expected) {// Metadata, not file bodies: tracked private configuration must never be read.// ctime also invalidates restored timestamps. Scan once at each batch boundary,// never per case; output directories are outside the source inventory.-export function inputFingerprint(repoRoot, env, selection, inventory, executable, extraRoots = []) {+export async function inputFingerprint(repoRoot, env, selection, inventory, executable, extraRoots = []) {const git = spawnSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard', '--deduplicate'], {cwd: repoRoot, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,});@@ -76,14 +76,30 @@ export function inputFingerprint(repoRoot, env, selection, inventory, executableversions: process.versions, executable, cwd: repoRoot, selection, inventory,env: Object.entries(env).sort(([a], [b]) => a.localeCompare(b)) });let entries = 0;- const record = (file, knownStat) => {+ // Bound both outstanding IO and retained results. Preserve sorted visitation+ // order so completion order cannot change the fingerprint. Settle the whole+ // window before reporting a failure or starting recursive directory work.+ const inspect = async (files, visit) => {+ for (let offset = 0; offset < files.length; offset += 32) {+ const window = files.slice(offset, offset + 32);+ const results = await Promise.allSettled(window.map(file => fs.promises.lstat(file, { bigint: true })));+ if (results.some(result => result.status === 'rejected' && result.reason.code !== 'ENOENT')) {+ throw new Error('Cannot inspect a required checkpoint input');+ }+ for (let i = 0; i < window.length; i++) {+ const result = results[i];+ await visit(window[i], result.status === 'fulfilled' ? result.value : null);+ }+ }+ };+ const record = async (file, stat) => {if (++entries > 500_000) throw new Error('Input inventory exceeds the checkpoint safety limit');try {- const stat = knownStat || fs.lstatSync(file, { bigint: true });+ if (!stat) { add([file, 'missing']); return; }add([file, stat.mode.toString(), stat.size.toString(), stat.mtimeNs.toString(), stat.ctimeNs.toString(), stat.ino.toString()]);if (stat.isSymbolicLink()) {- const target = fs.realpathSync(file);- const resolved = fs.statSync(target, { bigint: true });+ const target = await fs.promises.realpath(file);+ const resolved = await fs.promises.stat(target, { bigint: true });add([target, resolved.size.toString(), resolved.mtimeNs.toString(), resolved.ctimeNs.toString()]);}} catch (error) {@@ -91,34 +107,39 @@ export function inputFingerprint(repoRoot, env, selection, inventory, executableadd([file, 'missing']);}};- for (const file of git.stdout.split('\0').filter(Boolean).sort()) record(path.join(repoRoot, file));+ await inspect(git.stdout.split('\0').filter(Boolean).sort().map(file => path.join(repoRoot, file)), record);const visited = new Set();- const walk = file => {- let stat;- try { stat = fs.lstatSync(file, { bigint: true }); }- catch (error) { if (error.code === 'ENOENT') { record(file); return; } throw error; }+ const walk = async (file, stat) => {+ if (!stat) { await record(file, stat); return; }if (stat.isSymbolicLink()) {- record(file, stat);+ await record(file, stat);// Follow dependency links as well as recording the link itself.- if (fs.existsSync(file)) walk(fs.realpathSync(file));+ let target;+ try { target = await fs.promises.realpath(file); }+ catch (error) {+ if (error.code === 'ENOENT') return;+ throw new Error('Cannot inspect a required checkpoint input');+ }+ await inspect([target], walk);return;}- if (!stat.isDirectory()) { record(file, stat); return; }+ if (!stat.isDirectory()) { await record(file, stat); return; }if (visited.has(file)) return;visited.add(file);- for (const entry of fs.readdirSync(file).sort()) {+ let children;+ try { children = await fs.promises.readdir(file); }+ catch { throw new Error('Cannot inspect a required checkpoint input'); }+ await inspect(children.sort().filter(entry => {// Vitest writes these caches while collecting/executing unchanged inputs.- if (path.basename(file) === 'node_modules' && ['.vite', '.vite-temp', '.cache'].includes(entry)) continue;- walk(path.join(file, entry));- }+ return !(path.basename(file) === 'node_modules' && ['.vite', '.vite-temp', '.cache'].includes(entry));+ }).map(entry => path.join(file, entry)), walk);};for (const root of ['node_modules', 'src/core-agent/node_modules', 'venv','resources/runtime', 'resources/embedding-model', 'resources/officecli', 'src/main/native/build', 'out', ...extraRoots]) {- walk(path.resolve(repoRoot, root));+ await inspect([path.resolve(repoRoot, root)], walk);}- record(process.execPath);- record(executable);- if (env.ORKAS_TEST_PYTHON && path.isAbsolute(env.ORKAS_TEST_PYTHON)) record(env.ORKAS_TEST_PYTHON);+ await inspect([process.execPath, executable,+ ...(env.ORKAS_TEST_PYTHON && path.isAbsolute(env.ORKAS_TEST_PYTHON) ? [env.ORKAS_TEST_PYTHON] : [])], record);return hash.digest('hex');}@@ -199,13 +220,13 @@ export async function runCheckpoint({ directory, resume, batches, args, executabif (new Set(inventory).size !== inventory.length) throw new Error('Duplicate test discovery entries');batches = Math.min(batches, adapter ? discoveryReport.batchUnits : inventory.length);if (!Number.isInteger(batches) || batches < 1) throw new Error('Invalid checkpoint batch inventory');- const fingerprint = () => {+ const fingerprint = async () => {const scanStarted = performance.now();- const result = inputFingerprint(repoRoot, env, { args, batches, ...(adapter ? { lane: adapter.lane } : {}) }, inventory, executable, adapter?.extraRoots);+ const result = await inputFingerprint(repoRoot, env, { args, batches, ...(adapter ? { lane: adapter.lane } : {}) }, inventory, executable, adapter?.extraRoots);inputScanMs += performance.now() - scanStarted;return result;};- const initial = fingerprint();+ const initial = await fingerprint();const statePath = path.join(directory, 'checkpoint.json');const previous = resume ? readJson(statePath) : null;const reusable = previous?.schema === 1 && previous.fingerprint === initial@@ -254,7 +275,7 @@ export async function runCheckpoint({ directory, resume, batches, args, executabreport = readJson(path.join(directory, reportFile));const stderrGap = attempt.exitCode === 0 && !await stderrMatches(path.join(directory, attempt.stderr), report?.stderr);const certified = attempt.exitCode === 0 && validReport(report) && !stderrGap;- if (fingerprint() !== initial) changed = true;+ if (await fingerprint() !== initial) changed = true;entry = { ...attempt, report: reportFile, reportHash: report ? await hashFile(path.join(directory, reportFile)) : '', certified, stderrGap };if (certified && !changed) state.completed[batch] = entry;if (changed) state.completed = {};@@ -266,7 +287,7 @@ export async function runCheckpoint({ directory, resume, batches, args, executab// A cached or newly executed shard may never hide an omitted/new file.const executed = results.flatMap(result => result.files.map(file => file.file)).sort();const complete = JSON.stringify(executed) === JSON.stringify(inventory);- changed ||= fingerprint() !== initial;+ changed ||= await fingerprint() !== initial;const exitCode = !complete || changed || results.some(result => !result.certified) ? 1 : 0;// Missing evidence for one batch must not discard independently certified// siblings. A fully certified but inconsistent union invalidates all shards.diff --git a/test/main/quality/test-checkpoint.test.ts b/test/main/quality/test-checkpoint.test.tsindex ace70a28..a389c507 100644--- a/test/main/quality/test-checkpoint.test.ts+++ b/test/main/quality/test-checkpoint.test.ts@@ -2,14 +2,86 @@ import fs from 'node:fs';import os from 'node:os';import path from 'node:path';import { spawn, spawnSync } from 'node:child_process';-import { afterEach, expect, it } from 'vitest';+import { afterEach, expect, it, vi } from 'vitest';import { createStderrReceipt, combineStderrReceipts } from '../../../scripts/test-stderr-receipt.mjs';+import { inputFingerprint } from '../../../scripts/test-checkpoint.mjs';const pcRoot = path.resolve(__dirname, '../../..');const node = process.env.ORKAS_TEST_NODE || process.execPath;const roots: string[] = [];afterEach(() => roots.splice(0).forEach(root => fs.rmSync(root, { recursive: true, force: true })));+it('checkpoint input scanning yields for cancellation and bounds concurrent metadata work', async () => {+ const f = fixture();+ for (let i = 0; i < 100; i++) f.write(`node_modules/fixture/${i}.js`, 'fixture');+ const original = fs.promises.lstat.bind(fs.promises);+ let active = 0;+ let peak = 0;+ let yielded = false;+ const pending = new Promise<void>(resolve => setImmediate(() => { yielded = true; resolve(); }));+ const spy = vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => {+ peak = Math.max(peak, ++active);+ try { return await original(...args); } finally { active--; }+ });+ try {+ const result = await inputFingerprint(f.root, {}, {}, [], node);+ expect(result).toMatch(/^[a-f0-9]{64}$/);+ expect(yielded).toBe(true);+ expect(peak).toBeGreaterThan(1);+ expect(peak).toBeLessThanOrEqual(32);+ expect(active).toBe(0);+ } finally { spy.mockRestore(); await pending; }+});++it('checkpoint input scanning detects linked dependency edits, additions and deletions without reading bodies', async () => {+ const f = fixture();+ const library = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-checkpoint-library-'));+ roots.push(library);+ const source = path.join(library, 'dependency.js');+ fs.writeFileSync(source, 'first');+ const link = path.join(f.root, 'node_modules/linked');+ fs.symlinkSync(library, link, process.platform === 'win32' ? 'junction' : 'dir');+ const scan = () => inputFingerprint(f.root, {}, {}, [], node, [library]);+ try {+ const first = await scan();+ expect(await scan()).toBe(first);+ const stat = fs.statSync(source);+ fs.writeFileSync(source, 'other');+ fs.utimesSync(source, stat.atime, stat.mtime);+ const edited = await scan();+ expect(edited).not.toBe(first);+ const added = path.join(library, 'new.js');+ fs.writeFileSync(added, 'new dependency');+ const expanded = await scan();+ expect(expanded).not.toBe(edited);+ fs.unlinkSync(added);+ const deleted = await scan();+ expect(deleted).not.toBe(expanded);+ const read = vi.spyOn(fs.promises, 'readFile').mockRejectedValue(new Error('File bodies are private'));+ const readSync = vi.spyOn(fs, 'readFileSync').mockImplementation(() => { throw new Error('File bodies are private'); });+ try { expect(await scan()).toBe(deleted); }+ finally { read.mockRestore(); readSync.mockRestore(); }+ } finally { fs.unlinkSync(link); }+});++it('checkpoint input scanning fails closed on inaccessible metadata and settles pending IO', async () => {+ const f = fixture();+ const denied = path.join(f.root, 'test/a.test.ts');+ const original = fs.promises.lstat.bind(fs.promises);+ let active = 0;+ const spy = vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => {+ active++;+ try {+ if (String(args[0]) === denied) throw Object.assign(new Error('denied'), { code: 'EACCES' });+ return await original(...args);+ } finally { active--; }+ });+ try {+ await expect(inputFingerprint(f.root, {}, {}, [], node)).rejects.toThrow('Cannot inspect a required checkpoint input');+ expect(active).toBe(0);+ } finally { spy.mockRestore(); }+});+it('attributes the exact stderr bytes across chunk boundaries and concurrent worker line ordering', () => {const first = createStderrReceipt();first.ingest('progress α\n');