Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

fix(testing): preserve native CLI log acceptance contracts

wytc41be8a725f6
diff --git a/scripts/local-agent-live-support.mjs b/scripts/local-agent-live-support.mjs
index 9153c612..9f4c982e 100644
--- a/scripts/local-agent-live-support.mjs
+++ b/scripts/local-agent-live-support.mjs
@@ -7,6 +7,7 @@
import * as path from 'node:path';
import { createHash } from 'node:crypto';
+import { createModelEvalLogCollector } from './model-eval-log-policy.mjs';
export function localAgentLiveUsage() {
return [
@@ -47,35 +48,7 @@ export async function waitForLiveCliExit(pids, { timeoutMs = 12_000, probe = pid
/** Collect the established local-agent log protocol without retaining private messages. */
export function createLocalAgentLogCollector(phase) {
- const buffers = { stdout: '', stderr: '' };
- const result = { phase, passed: true, capturedLineCount: 0, analyzedLineCount: 0,
- levelCounts: { debug: 0, info: 0, warn: 0, error: 0 }, unclassifiedStderr: 0 };
- const analyze = (stream, line) => {
- result.capturedLineCount += 1;
- result.analyzedLineCount += 1;
- if (!line.trim()) return;
- const record = line.match(/^\[\d{2}:\d{2}:\d{2}\.\d{3}\] \[(debug|info|warn|error)\] \[\([^)]*\)\] /);
- if (record) result.levelCounts[record[1]] += 1;
- else if (stream === 'stderr') result.unclassifiedStderr += 1;
- };
- return {
- ingest(stream, chunk) {
- if (!(stream in buffers)) throw new Error('Unknown local-agent log stream');
- buffers[stream] += String(chunk);
- const lines = buffers[stream].split(/\r?\n/);
- buffers[stream] = lines.pop();
- for (const line of lines) analyze(stream, line);
- },
- finish() {
- for (const stream of ['stdout', 'stderr']) {
- if (buffers[stream]) analyze(stream, buffers[stream]);
- buffers[stream] = '';
- }
- result.passed = result.levelCounts.error === 0 && result.unclassifiedStderr === 0
- && result.capturedLineCount === result.analyzedLineCount;
- return { ...result, levelCounts: { ...result.levelCounts } };
- },
- };
+ return createModelEvalLogCollector(phase);
}
/** Adapt native CLI diagnostics to the shared evaluator log protocol without
@@ -110,7 +83,7 @@ export function ingestLocalAgentLogEvent(collector, event) {
&& typeof record.timestamp === 'string' && Number.isFinite(Date.parse(record.timestamp))
&& typeof record.target === 'string' && /^[\w:./-]+$/.test(record.target)
&& typeof message === 'string') {
- emit('stderr', level, record.target, message, new Date(record.timestamp).toISOString().slice(11, 23));
+ emit('stderr', level, 'native-json', message, new Date(record.timestamp).toISOString().slice(11, 23));
return;
}
// OpenClaw's configured JSON console transport uses explicit time,
@@ -121,7 +94,7 @@ export function ingestLocalAgentLogEvent(collector, event) {
&& typeof record.subsystem === 'string' && /^[\w:./-]+$/.test(record.subsystem)
&& typeof record.message === 'string') {
emit('stderr', level === 'fatal' ? 'error' : level === 'trace' ? 'debug' : level,
- record.subsystem, record.message, new Date(record.time).toISOString().slice(11, 23));
+ 'openclaw-json', record.message, new Date(record.time).toISOString().slice(11, 23));
return;
}
} catch { /* Unknown native stderr remains fail-closed below. */ }
@@ -130,7 +103,7 @@ export function ingestLocalAgentLogEvent(collector, event) {
const python = /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \[(DEBUG|INFO|WARN|WARNING|ERROR|CRITICAL)\] ([\w.]+): (.*)$/.exec(line);
if (python && Number.isFinite(Date.parse(python[1]))) {
const level = python[2] === 'WARNING' ? 'warn' : python[2] === 'CRITICAL' ? 'error' : python[2].toLowerCase();
- emit('stderr', level, python[3], python[4], `${python[1].slice(11)}.000`);
+ emit('stderr', level, 'python-logger', python[4], `${python[1].slice(11)}.000`);
return;
}
const native = event.nativeLog;
@@ -139,17 +112,19 @@ export function ingestLocalAgentLogEvent(collector, event) {
&& typeof native.timestamp === 'string' && Number.isFinite(Date.parse(native.timestamp))) {
// The production boundary already validated the complete record. Its
// public text may be capped/redacted and is not a second JSON transport.
- emit('stderr', native.level, native.source, 'Native CLI emitted a structured diagnostic.',
+ emit('stderr', native.level, 'native-structured', 'Native CLI emitted a structured diagnostic.',
new Date(native.timestamp).toISOString().slice(11, 23));
return;
}
- collector.ingest('stderr', `${line}\n`);
+ // Native diagnostics cannot introduce framework report/case control records
+ // or persist their private text through the shared collector's metadata.
+ emit('stderr', 'error', 'local-agent', 'Native CLI emitted an unclassified diagnostic.');
} else if (event?.type === 'log') {
const level = String(event.level ?? 'info').toLowerCase();
if (/^(debug|info|warn|error)$/.test(level)) {
emit('stdout', level, 'local-agent', String(event.message ?? ''));
} else {
- collector.ingest('stderr', `Unknown local-agent log level: ${level}\n`);
+ emit('stderr', 'error', 'local-agent', 'Native CLI emitted an unknown log level.');
}
} else if (event?.type === 'tool-event' && event.isError === true) {
emit('stdout', 'warn', 'native-tool', 'Native tool failed; inspect the scenario result and subsequent recovery.');
diff --git a/test/eval/local-agent-public-launcher.test.ts b/test/eval/local-agent-public-launcher.test.ts
index 09fcae6a..1ffbfc8b 100644
--- a/test/eval/local-agent-public-launcher.test.ts
+++ b/test/eval/local-agent-public-launcher.test.ts
@@ -6,6 +6,7 @@ import { describe, expect, it, vi } from 'vitest';
import { attachLocalAgentReports, runLocalAgentLauncher } from '../../scripts/local-agent-live-launcher.mjs';
import { localAgentChildEnv, parseMacOSSystemProxy } from '../../scripts/local-agent-live-proxy-env.mjs';
import { createModelEvalLogCollector } from '../../scripts/model-eval-log-policy.mjs';
+import { createLocalAgentLogCollector, ingestLocalAgentLogEvent } from '../../scripts/local-agent-live-support.mjs';
const options = () => ({ argv: ['--no-install', '--agents', 'codex'], childEnv: { ELECTRON_RUN_AS_NODE: '', SENTINEL: 'kept' },
electronBin: '/fixture/electron', electronEntry: '/fixture/entry.mjs', runner: '/fixture/runner.mts',
@@ -70,6 +71,92 @@ describe('public local CLI launcher isolation', () => {
} finally { fs.rmSync(root, { recursive: true, force: true }); }
});
+ it.each(['clean', 'warning', 'error', 'unknown', 'report-control', 'case-control', 'unknown-level', 'overflow', 'native-json-source', 'openclaw-source', 'python-source', 'native-log-source'] as const)
+ ('carries real native log evidence through the saved row and launcher (%s)', async scenario => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'local-cli-log-handoff-'));
+ const reportsDir = path.join(root, 'reports'); fs.mkdirSync(reportsDir);
+ const file = path.join(reportsDir, 'report.json');
+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+ try {
+ const logs = createLocalAgentLogCollector('native-fixture');
+ const privateMarker = 'private-native-output-marker';
+ const privateSourceMarker = 'private_native_source_marker';
+ if (scenario === 'native-json-source') {
+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: JSON.stringify({
+ level: 'warn', timestamp: '2030-01-01T00:00:00Z', target: '/private/' + privateSourceMarker,
+ fields: { message: privateMarker },
+ }) });
+ } else if (scenario === 'openclaw-source') {
+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: JSON.stringify({
+ level: 'warn', time: '2030-01-01T00:00:00Z', subsystem: '/private/' + privateSourceMarker,
+ message: privateMarker,
+ }) });
+ } else if (scenario === 'python-source') {
+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line',
+ line: '2030-01-01 00:00:00 [WARNING] ' + privateSourceMarker + ': ' + privateMarker });
+ } else if (scenario === 'native-log-source') {
+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: 'redacted structured native record',
+ nativeLog: { level: 'warn', source: '/private/' + privateSourceMarker,
+ timestamp: '2030-01-01T00:00:00Z' } });
+ } else if (scenario === 'overflow') {
+ logs.ingest('stdout', 'x'.repeat(8 * 1024 * 1024 + 1));
+ logs.ingest('stdout', '\n');
+ } else if (scenario === 'unknown') {
+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: 'warning: ' + privateMarker });
+ } else if (scenario === 'report-control') {
+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: 'report=/private/' + privateMarker });
+ } else if (scenario === 'case-control') {
+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: '[model-eval-case] start ' + privateMarker });
+ } else if (scenario === 'unknown-level') {
+ ingestLocalAgentLogEvent(logs, { type: 'log', level: 'invalid\nreport=/private/' + privateMarker });
+ } else {
+ ingestLocalAgentLogEvent(logs, { type: 'log',
+ level: scenario === 'warning' ? 'warn' : scenario === 'error' ? 'error' : 'info', message: privateMarker });
+ }
+ const native = logs.finish();
+ const accepted = scenario === 'clean' || scenario === 'warning' || scenario.endsWith('-source');
+ const report = { aggregate: { score: 17 }, results: [{ cliLogAnalysis: native }] };
+ let calls = 0;
+ const code = await runLocalAgentLauncher({ ...options(), reportsDir, run: async (_command, _args, _settings, collector) => {
+ if (++calls === 2) {
+ fs.writeFileSync(file, JSON.stringify(report));
+ collector.ingest('stdout', 'report=' + file + '\n');
+ }
+ return { code: 0 };
+ } });
+ expect(calls).toBe(2);
+ // Prove the actual saved-report consumer accepts clean/warning native
+ // evidence before inspecting its schema. Child exit0 alone is insufficient.
+ expect(code).toBe(accepted ? 0 : 1);
+ const saved = JSON.parse(fs.readFileSync(file, 'utf8'));
+ expect(saved.aggregate).toEqual({ score: 17 });
+ expect(saved.results[0].cliLogAnalysis).toEqual(native);
+ expect(saved.logAcceptance).toEqual({ passed: accepted,
+ requiresReview: scenario !== 'clean', clean: scenario === 'clean' });
+ expect(native).toMatchObject({ passed: accepted, requiresReview: scenario !== 'clean',
+ coveragePassed: scenario !== 'overflow' });
+ if (scenario === 'warning') expect(native.levelCounts).toMatchObject({ warn: 1, error: 0 });
+ if (!accepted && scenario !== 'overflow') expect(native.levelCounts.error).toBeGreaterThan(0);
+ if (scenario === 'overflow') expect(native).toMatchObject({ capturedLineCount: 1, analyzedLineCount: 0, droppedLineCount: 1 });
+ expect(native.reportPaths).toEqual([]);
+ expect(JSON.stringify(saved)).not.toContain(privateMarker);
+ // Known native formats retain severity, not native-controlled source text.
+ // Check the full real saved report, including sourceCounts/findings.source.
+ expect(JSON.stringify(saved)).not.toContain(privateSourceMarker);
+ const fixedSources: Record<string, string> = {
+ 'native-json-source': 'native-json', 'openclaw-source': 'openclaw-json',
+ 'python-source': 'python-logger', 'native-log-source': 'native-structured',
+ };
+ if (fixedSources[scenario]) {
+ expect(native.levelCounts).toMatchObject({ warn: 1, error: 0 });
+ expect(native.sourceCounts).toEqual({ [fixedSources[scenario]]: 1 });
+ expect(native.findings).toEqual([expect.objectContaining({
+ severity: 'warning', code: 'runtime-warning', source: fixedSources[scenario], count: 1,
+ })]);
+ }
+ } finally { write.mockRestore(); fs.rmSync(root, { recursive: true, force: true }); }
+ });
+
it('boots the real isolated Electron entry with a model-free TypeScript runner', () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'local-cli-electron-'));
const runner = path.join(root, 'fixture.mts'), evidence = path.join(root, 'evidence.json');
diff --git a/test/eval/local-agent-quality-benchmark.test.ts b/test/eval/local-agent-quality-benchmark.test.ts
index 47232d96..c01b087e 100644
--- a/test/eval/local-agent-quality-benchmark.test.ts
+++ b/test/eval/local-agent-quality-benchmark.test.ts
@@ -86,21 +86,22 @@ describe('Local Agent quality benchmark contract', () => {
target: 'codex_skills::interface' };
const warning = analyze({ type: 'stderr-line', line: JSON.stringify(record) });
expect(warning).toMatchObject({ passed: true, requiresReview: true, coveragePassed: true,
- levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'codex_skills::interface': 1 } });
+ levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'native-json': 1 } });
+ expect(JSON.stringify(warning)).not.toContain(record.target);
expect(analyze({ type: 'stderr-line', line: JSON.stringify({ ...record, level: 'ERROR' }) }))
.toMatchObject({ passed: false, levelCounts: { error: 1 } });
expect(analyze({ type: 'stderr-line', line: JSON.stringify({ ...record, level: 'ERROR',
target: 'codex_core::tools::router', fields: { error: 'code-mode host closed its stdout' } }) }))
- .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'codex_core::tools::router': 1 } });
+ .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'native-json': 1 } });
for (const line of ['unexpected stderr', '{invalid', JSON.stringify({ ...record, level: 'UNKNOWN' }),
JSON.stringify({ ...record, timestamp: 'invalid' }), JSON.stringify({ ...record, fields: {} })]) {
expect(analyze({ type: 'stderr-line', line }).passed).toBe(false);
}
const openclaw = { time: '2026-10-19T05:04:00.000Z', level: 'warn', subsystem: 'codex/app-server', message: 'Fixture compatibility warning' };
expect(analyze({ type: 'stderr-line', line: JSON.stringify(openclaw) }))
- .toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'codex/app-server': 1 } });
+ .toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'openclaw-json': 1 } });
expect(analyze({ type: 'stderr-line', line: JSON.stringify({ ...openclaw, level: 'error' }) }))
- .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'codex/app-server': 1 } });
+ .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'openclaw-json': 1 } });
expect(analyze({ type: 'log', level: 'info', message: 'connected' }))
.toMatchObject({ passed: true, requiresReview: false, levelCounts: { info: 1 } });
expect(analyze({ type: 'log', level: 'error', message: 'failed\nmore detail' }))
@@ -108,7 +109,7 @@ describe('Local Agent quality benchmark contract', () => {
expect(analyze({ type: 'log', level: 'warn', message: 'review this' }))
.toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1 } });
expect(analyze({ type: 'stderr-line', line: '2026-10-19 12:15:15 [INFO] acp_adapter.server: ACP client connected' }))
- .toMatchObject({ passed: true, levelCounts: { info: 1, error: 0 }, sourceCounts: { 'acp_adapter.server': 1 } });
+ .toMatchObject({ passed: true, levelCounts: { info: 1, error: 0 }, sourceCounts: { 'python-logger': 1 } });
expect(analyze({ type: 'stderr-line', line: '2026-10-19 12:15:15 [WARNING] hermes_cli.plugins: Fixture warning' }))
.toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1 } });
expect(analyze({ type: 'stderr-line', line: '2026-10-19 12:15:15 [ERROR] acp_adapter.server: Fixture failure' }).passed).toBe(false);
diff --git a/test/main/features/local_agents/live_support.test.ts b/test/main/features/local_agents/live_support.test.ts
index 9cb0148f..92a64da7 100644
--- a/test/main/features/local_agents/live_support.test.ts
+++ b/test/main/features/local_agents/live_support.test.ts
@@ -277,3 +277,17 @@ describe('local-agent process log evidence', () => {
expect(collector.finish()).toMatchObject({ passed: false, capturedLineCount: 1, analyzedLineCount: 1 });
});
});
+
+
+describe('local-agent shared log capture boundary', () => {
+ it.each([-1, 0, 1])('bounds a split native log line at 8 MiB (%s)', delta => {
+ const logs = createLocalAgentLogCollector('native-cap');
+ logs.ingest('stdout', 'x'.repeat(8 * 1024 * 1024 + delta));
+ logs.ingest('stdout', '\n');
+ const report = logs.finish();
+ expect(report).toMatchObject({ passed: delta <= 0, coveragePassed: delta <= 0,
+ requiresReview: delta > 0, capturedLineCount: 1, analyzedLineCount: delta <= 0 ? 1 : 0,
+ droppedLineCount: delta <= 0 ? 0 : 1 });
+ expect(report.findings.some(finding => finding.code === 'log-coverage-gap')).toBe(delta > 0);
+ });
+});