Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

Bound metadata checkpoint scanning without reading file bodies

wyt01019a41f6a4
diff --git a/scripts/test-checkpoint.mjs b/scripts/test-checkpoint.mjs
index 917417ab..fa57ff5d 100644
--- a/scripts/test-checkpoint.mjs
+++ b/scripts/test-checkpoint.mjs
@@ -65,7 +65,7 @@ async function stderrMatches(file, expected) {
// Metadata, not file bodies: tracked private configuration must never be read.
// ctime also invalidates restored timestamps. Scan once at each batch boundary,
// never per case; output directories are outside the source inventory.
-export function inputFingerprint(repoRoot, env, selection, inventory, executable, extraRoots = []) {
+export async function inputFingerprint(repoRoot, env, selection, inventory, executable, extraRoots = []) {
const git = spawnSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard', '--deduplicate'], {
cwd: repoRoot, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
});
@@ -76,14 +76,30 @@ export function inputFingerprint(repoRoot, env, selection, inventory, executable
versions: process.versions, executable, cwd: repoRoot, selection, inventory,
env: Object.entries(env).sort(([a], [b]) => a.localeCompare(b)) });
let entries = 0;
- const record = (file, knownStat) => {
+ // Bound both outstanding IO and retained results. Preserve sorted visitation
+ // order so completion order cannot change the fingerprint. Settle the whole
+ // window before reporting a failure or starting recursive directory work.
+ const inspect = async (files, visit) => {
+ for (let offset = 0; offset < files.length; offset += 32) {
+ const window = files.slice(offset, offset + 32);
+ const results = await Promise.allSettled(window.map(file => fs.promises.lstat(file, { bigint: true })));
+ if (results.some(result => result.status === 'rejected' && result.reason.code !== 'ENOENT')) {
+ throw new Error('Cannot inspect a required checkpoint input');
+ }
+ for (let i = 0; i < window.length; i++) {
+ const result = results[i];
+ await visit(window[i], result.status === 'fulfilled' ? result.value : null);
+ }
+ }
+ };
+ const record = async (file, stat) => {
if (++entries > 500_000) throw new Error('Input inventory exceeds the checkpoint safety limit');
try {
- const stat = knownStat || fs.lstatSync(file, { bigint: true });
+ if (!stat) { add([file, 'missing']); return; }
add([file, stat.mode.toString(), stat.size.toString(), stat.mtimeNs.toString(), stat.ctimeNs.toString(), stat.ino.toString()]);
if (stat.isSymbolicLink()) {
- const target = fs.realpathSync(file);
- const resolved = fs.statSync(target, { bigint: true });
+ const target = await fs.promises.realpath(file);
+ const resolved = await fs.promises.stat(target, { bigint: true });
add([target, resolved.size.toString(), resolved.mtimeNs.toString(), resolved.ctimeNs.toString()]);
}
} catch (error) {
@@ -91,34 +107,39 @@ export function inputFingerprint(repoRoot, env, selection, inventory, executable
add([file, 'missing']);
}
};
- for (const file of git.stdout.split('\0').filter(Boolean).sort()) record(path.join(repoRoot, file));
+ await inspect(git.stdout.split('\0').filter(Boolean).sort().map(file => path.join(repoRoot, file)), record);
const visited = new Set();
- const walk = file => {
- let stat;
- try { stat = fs.lstatSync(file, { bigint: true }); }
- catch (error) { if (error.code === 'ENOENT') { record(file); return; } throw error; }
+ const walk = async (file, stat) => {
+ if (!stat) { await record(file, stat); return; }
if (stat.isSymbolicLink()) {
- record(file, stat);
+ await record(file, stat);
// Follow dependency links as well as recording the link itself.
- if (fs.existsSync(file)) walk(fs.realpathSync(file));
+ let target;
+ try { target = await fs.promises.realpath(file); }
+ catch (error) {
+ if (error.code === 'ENOENT') return;
+ throw new Error('Cannot inspect a required checkpoint input');
+ }
+ await inspect([target], walk);
return;
}
- if (!stat.isDirectory()) { record(file, stat); return; }
+ if (!stat.isDirectory()) { await record(file, stat); return; }
if (visited.has(file)) return;
visited.add(file);
- for (const entry of fs.readdirSync(file).sort()) {
+ let children;
+ try { children = await fs.promises.readdir(file); }
+ catch { throw new Error('Cannot inspect a required checkpoint input'); }
+ await inspect(children.sort().filter(entry => {
// Vitest writes these caches while collecting/executing unchanged inputs.
- if (path.basename(file) === 'node_modules' && ['.vite', '.vite-temp', '.cache'].includes(entry)) continue;
- walk(path.join(file, entry));
- }
+ return !(path.basename(file) === 'node_modules' && ['.vite', '.vite-temp', '.cache'].includes(entry));
+ }).map(entry => path.join(file, entry)), walk);
};
for (const root of ['node_modules', 'src/core-agent/node_modules', 'venv',
'resources/runtime', 'resources/embedding-model', 'resources/officecli', 'src/main/native/build', 'out', ...extraRoots]) {
- walk(path.resolve(repoRoot, root));
+ await inspect([path.resolve(repoRoot, root)], walk);
}
- record(process.execPath);
- record(executable);
- if (env.ORKAS_TEST_PYTHON && path.isAbsolute(env.ORKAS_TEST_PYTHON)) record(env.ORKAS_TEST_PYTHON);
+ await inspect([process.execPath, executable,
+ ...(env.ORKAS_TEST_PYTHON && path.isAbsolute(env.ORKAS_TEST_PYTHON) ? [env.ORKAS_TEST_PYTHON] : [])], record);
return hash.digest('hex');
}
@@ -199,13 +220,13 @@ export async function runCheckpoint({ directory, resume, batches, args, executab
if (new Set(inventory).size !== inventory.length) throw new Error('Duplicate test discovery entries');
batches = Math.min(batches, adapter ? discoveryReport.batchUnits : inventory.length);
if (!Number.isInteger(batches) || batches < 1) throw new Error('Invalid checkpoint batch inventory');
- const fingerprint = () => {
+ const fingerprint = async () => {
const scanStarted = performance.now();
- const result = inputFingerprint(repoRoot, env, { args, batches, ...(adapter ? { lane: adapter.lane } : {}) }, inventory, executable, adapter?.extraRoots);
+ const result = await inputFingerprint(repoRoot, env, { args, batches, ...(adapter ? { lane: adapter.lane } : {}) }, inventory, executable, adapter?.extraRoots);
inputScanMs += performance.now() - scanStarted;
return result;
};
- const initial = fingerprint();
+ const initial = await fingerprint();
const statePath = path.join(directory, 'checkpoint.json');
const previous = resume ? readJson(statePath) : null;
const reusable = previous?.schema === 1 && previous.fingerprint === initial
@@ -254,7 +275,7 @@ export async function runCheckpoint({ directory, resume, batches, args, executab
report = readJson(path.join(directory, reportFile));
const stderrGap = attempt.exitCode === 0 && !await stderrMatches(path.join(directory, attempt.stderr), report?.stderr);
const certified = attempt.exitCode === 0 && validReport(report) && !stderrGap;
- if (fingerprint() !== initial) changed = true;
+ if (await fingerprint() !== initial) changed = true;
entry = { ...attempt, report: reportFile, reportHash: report ? await hashFile(path.join(directory, reportFile)) : '', certified, stderrGap };
if (certified && !changed) state.completed[batch] = entry;
if (changed) state.completed = {};
@@ -266,7 +287,7 @@ export async function runCheckpoint({ directory, resume, batches, args, executab
// A cached or newly executed shard may never hide an omitted/new file.
const executed = results.flatMap(result => result.files.map(file => file.file)).sort();
const complete = JSON.stringify(executed) === JSON.stringify(inventory);
- changed ||= fingerprint() !== initial;
+ changed ||= await fingerprint() !== initial;
const exitCode = !complete || changed || results.some(result => !result.certified) ? 1 : 0;
// Missing evidence for one batch must not discard independently certified
// siblings. A fully certified but inconsistent union invalidates all shards.
diff --git a/test/main/quality/test-checkpoint.test.ts b/test/main/quality/test-checkpoint.test.ts
index ace70a28..a389c507 100644
--- a/test/main/quality/test-checkpoint.test.ts
+++ b/test/main/quality/test-checkpoint.test.ts
@@ -2,14 +2,86 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawn, spawnSync } from 'node:child_process';
-import { afterEach, expect, it } from 'vitest';
+import { afterEach, expect, it, vi } from 'vitest';
import { createStderrReceipt, combineStderrReceipts } from '../../../scripts/test-stderr-receipt.mjs';
+import { inputFingerprint } from '../../../scripts/test-checkpoint.mjs';
const pcRoot = path.resolve(__dirname, '../../..');
const node = process.env.ORKAS_TEST_NODE || process.execPath;
const roots: string[] = [];
afterEach(() => roots.splice(0).forEach(root => fs.rmSync(root, { recursive: true, force: true })));
+it('checkpoint input scanning yields for cancellation and bounds concurrent metadata work', async () => {
+ const f = fixture();
+ for (let i = 0; i < 100; i++) f.write(`node_modules/fixture/${i}.js`, 'fixture');
+ const original = fs.promises.lstat.bind(fs.promises);
+ let active = 0;
+ let peak = 0;
+ let yielded = false;
+ const pending = new Promise<void>(resolve => setImmediate(() => { yielded = true; resolve(); }));
+ const spy = vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => {
+ peak = Math.max(peak, ++active);
+ try { return await original(...args); } finally { active--; }
+ });
+ try {
+ const result = await inputFingerprint(f.root, {}, {}, [], node);
+ expect(result).toMatch(/^[a-f0-9]{64}$/);
+ expect(yielded).toBe(true);
+ expect(peak).toBeGreaterThan(1);
+ expect(peak).toBeLessThanOrEqual(32);
+ expect(active).toBe(0);
+ } finally { spy.mockRestore(); await pending; }
+});
+
+it('checkpoint input scanning detects linked dependency edits, additions and deletions without reading bodies', async () => {
+ const f = fixture();
+ const library = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-checkpoint-library-'));
+ roots.push(library);
+ const source = path.join(library, 'dependency.js');
+ fs.writeFileSync(source, 'first');
+ const link = path.join(f.root, 'node_modules/linked');
+ fs.symlinkSync(library, link, process.platform === 'win32' ? 'junction' : 'dir');
+ const scan = () => inputFingerprint(f.root, {}, {}, [], node, [library]);
+ try {
+ const first = await scan();
+ expect(await scan()).toBe(first);
+ const stat = fs.statSync(source);
+ fs.writeFileSync(source, 'other');
+ fs.utimesSync(source, stat.atime, stat.mtime);
+ const edited = await scan();
+ expect(edited).not.toBe(first);
+ const added = path.join(library, 'new.js');
+ fs.writeFileSync(added, 'new dependency');
+ const expanded = await scan();
+ expect(expanded).not.toBe(edited);
+ fs.unlinkSync(added);
+ const deleted = await scan();
+ expect(deleted).not.toBe(expanded);
+ const read = vi.spyOn(fs.promises, 'readFile').mockRejectedValue(new Error('File bodies are private'));
+ const readSync = vi.spyOn(fs, 'readFileSync').mockImplementation(() => { throw new Error('File bodies are private'); });
+ try { expect(await scan()).toBe(deleted); }
+ finally { read.mockRestore(); readSync.mockRestore(); }
+ } finally { fs.unlinkSync(link); }
+});
+
+it('checkpoint input scanning fails closed on inaccessible metadata and settles pending IO', async () => {
+ const f = fixture();
+ const denied = path.join(f.root, 'test/a.test.ts');
+ const original = fs.promises.lstat.bind(fs.promises);
+ let active = 0;
+ const spy = vi.spyOn(fs.promises, 'lstat').mockImplementation(async (...args) => {
+ active++;
+ try {
+ if (String(args[0]) === denied) throw Object.assign(new Error('denied'), { code: 'EACCES' });
+ return await original(...args);
+ } finally { active--; }
+ });
+ try {
+ await expect(inputFingerprint(f.root, {}, {}, [], node)).rejects.toThrow('Cannot inspect a required checkpoint input');
+ expect(active).toBe(0);
+ } finally { spy.mockRestore(); }
+});
+
it('attributes the exact stderr bytes across chunk boundaries and concurrent worker line ordering', () => {
const first = createStderrReceipt();
first.ingest('progress α\n');