Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
fix(testing): preserve native CLI log acceptance contracts
wytc41be8a725f6
diff --git a/scripts/local-agent-live-support.mjs b/scripts/local-agent-live-support.mjsindex 9153c612..9f4c982e 100644--- a/scripts/local-agent-live-support.mjs+++ b/scripts/local-agent-live-support.mjs@@ -7,6 +7,7 @@import * as path from 'node:path';import { createHash } from 'node:crypto';+import { createModelEvalLogCollector } from './model-eval-log-policy.mjs';export function localAgentLiveUsage() {return [@@ -47,35 +48,7 @@ export async function waitForLiveCliExit(pids, { timeoutMs = 12_000, probe = pid/** Collect the established local-agent log protocol without retaining private messages. */export function createLocalAgentLogCollector(phase) {- const buffers = { stdout: '', stderr: '' };- const result = { phase, passed: true, capturedLineCount: 0, analyzedLineCount: 0,- levelCounts: { debug: 0, info: 0, warn: 0, error: 0 }, unclassifiedStderr: 0 };- const analyze = (stream, line) => {- result.capturedLineCount += 1;- result.analyzedLineCount += 1;- if (!line.trim()) return;- const record = line.match(/^\[\d{2}:\d{2}:\d{2}\.\d{3}\] \[(debug|info|warn|error)\] \[\([^)]*\)\] /);- if (record) result.levelCounts[record[1]] += 1;- else if (stream === 'stderr') result.unclassifiedStderr += 1;- };- return {- ingest(stream, chunk) {- if (!(stream in buffers)) throw new Error('Unknown local-agent log stream');- buffers[stream] += String(chunk);- const lines = buffers[stream].split(/\r?\n/);- buffers[stream] = lines.pop();- for (const line of lines) analyze(stream, line);- },- finish() {- for (const stream of ['stdout', 'stderr']) {- if (buffers[stream]) analyze(stream, buffers[stream]);- buffers[stream] = '';- }- result.passed = result.levelCounts.error === 0 && result.unclassifiedStderr === 0- && result.capturedLineCount === result.analyzedLineCount;- return { ...result, levelCounts: { ...result.levelCounts } };- },- };+ return createModelEvalLogCollector(phase);}/** Adapt native CLI diagnostics to the shared evaluator log protocol without@@ -110,7 +83,7 @@ export function ingestLocalAgentLogEvent(collector, event) {&& typeof record.timestamp === 'string' && Number.isFinite(Date.parse(record.timestamp))&& typeof record.target === 'string' && /^[\w:./-]+$/.test(record.target)&& typeof message === 'string') {- emit('stderr', level, record.target, message, new Date(record.timestamp).toISOString().slice(11, 23));+ emit('stderr', level, 'native-json', message, new Date(record.timestamp).toISOString().slice(11, 23));return;}// OpenClaw's configured JSON console transport uses explicit time,@@ -121,7 +94,7 @@ export function ingestLocalAgentLogEvent(collector, event) {&& typeof record.subsystem === 'string' && /^[\w:./-]+$/.test(record.subsystem)&& typeof record.message === 'string') {emit('stderr', level === 'fatal' ? 'error' : level === 'trace' ? 'debug' : level,- record.subsystem, record.message, new Date(record.time).toISOString().slice(11, 23));+ 'openclaw-json', record.message, new Date(record.time).toISOString().slice(11, 23));return;}} catch { /* Unknown native stderr remains fail-closed below. */ }@@ -130,7 +103,7 @@ export function ingestLocalAgentLogEvent(collector, event) {const python = /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \[(DEBUG|INFO|WARN|WARNING|ERROR|CRITICAL)\] ([\w.]+): (.*)$/.exec(line);if (python && Number.isFinite(Date.parse(python[1]))) {const level = python[2] === 'WARNING' ? 'warn' : python[2] === 'CRITICAL' ? 'error' : python[2].toLowerCase();- emit('stderr', level, python[3], python[4], `${python[1].slice(11)}.000`);+ emit('stderr', level, 'python-logger', python[4], `${python[1].slice(11)}.000`);return;}const native = event.nativeLog;@@ -139,17 +112,19 @@ export function ingestLocalAgentLogEvent(collector, event) {&& typeof native.timestamp === 'string' && Number.isFinite(Date.parse(native.timestamp))) {// The production boundary already validated the complete record. Its// public text may be capped/redacted and is not a second JSON transport.- emit('stderr', native.level, native.source, 'Native CLI emitted a structured diagnostic.',+ emit('stderr', native.level, 'native-structured', 'Native CLI emitted a structured diagnostic.',new Date(native.timestamp).toISOString().slice(11, 23));return;}- collector.ingest('stderr', `${line}\n`);+ // Native diagnostics cannot introduce framework report/case control records+ // or persist their private text through the shared collector's metadata.+ emit('stderr', 'error', 'local-agent', 'Native CLI emitted an unclassified diagnostic.');} else if (event?.type === 'log') {const level = String(event.level ?? 'info').toLowerCase();if (/^(debug|info|warn|error)$/.test(level)) {emit('stdout', level, 'local-agent', String(event.message ?? ''));} else {- collector.ingest('stderr', `Unknown local-agent log level: ${level}\n`);+ emit('stderr', 'error', 'local-agent', 'Native CLI emitted an unknown log level.');}} else if (event?.type === 'tool-event' && event.isError === true) {emit('stdout', 'warn', 'native-tool', 'Native tool failed; inspect the scenario result and subsequent recovery.');diff --git a/test/eval/local-agent-public-launcher.test.ts b/test/eval/local-agent-public-launcher.test.tsindex 09fcae6a..1ffbfc8b 100644--- a/test/eval/local-agent-public-launcher.test.ts+++ b/test/eval/local-agent-public-launcher.test.ts@@ -6,6 +6,7 @@ import { describe, expect, it, vi } from 'vitest';import { attachLocalAgentReports, runLocalAgentLauncher } from '../../scripts/local-agent-live-launcher.mjs';import { localAgentChildEnv, parseMacOSSystemProxy } from '../../scripts/local-agent-live-proxy-env.mjs';import { createModelEvalLogCollector } from '../../scripts/model-eval-log-policy.mjs';+import { createLocalAgentLogCollector, ingestLocalAgentLogEvent } from '../../scripts/local-agent-live-support.mjs';const options = () => ({ argv: ['--no-install', '--agents', 'codex'], childEnv: { ELECTRON_RUN_AS_NODE: '', SENTINEL: 'kept' },electronBin: '/fixture/electron', electronEntry: '/fixture/entry.mjs', runner: '/fixture/runner.mts',@@ -70,6 +71,92 @@ describe('public local CLI launcher isolation', () => {} finally { fs.rmSync(root, { recursive: true, force: true }); }});+ it.each(['clean', 'warning', 'error', 'unknown', 'report-control', 'case-control', 'unknown-level', 'overflow', 'native-json-source', 'openclaw-source', 'python-source', 'native-log-source'] as const)+ ('carries real native log evidence through the saved row and launcher (%s)', async scenario => {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'local-cli-log-handoff-'));+ const reportsDir = path.join(root, 'reports'); fs.mkdirSync(reportsDir);+ const file = path.join(reportsDir, 'report.json');+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);+ try {+ const logs = createLocalAgentLogCollector('native-fixture');+ const privateMarker = 'private-native-output-marker';+ const privateSourceMarker = 'private_native_source_marker';+ if (scenario === 'native-json-source') {+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: JSON.stringify({+ level: 'warn', timestamp: '2030-01-01T00:00:00Z', target: '/private/' + privateSourceMarker,+ fields: { message: privateMarker },+ }) });+ } else if (scenario === 'openclaw-source') {+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: JSON.stringify({+ level: 'warn', time: '2030-01-01T00:00:00Z', subsystem: '/private/' + privateSourceMarker,+ message: privateMarker,+ }) });+ } else if (scenario === 'python-source') {+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line',+ line: '2030-01-01 00:00:00 [WARNING] ' + privateSourceMarker + ': ' + privateMarker });+ } else if (scenario === 'native-log-source') {+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: 'redacted structured native record',+ nativeLog: { level: 'warn', source: '/private/' + privateSourceMarker,+ timestamp: '2030-01-01T00:00:00Z' } });+ } else if (scenario === 'overflow') {+ logs.ingest('stdout', 'x'.repeat(8 * 1024 * 1024 + 1));+ logs.ingest('stdout', '\n');+ } else if (scenario === 'unknown') {+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: 'warning: ' + privateMarker });+ } else if (scenario === 'report-control') {+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: 'report=/private/' + privateMarker });+ } else if (scenario === 'case-control') {+ ingestLocalAgentLogEvent(logs, { type: 'stderr-line', line: '[model-eval-case] start ' + privateMarker });+ } else if (scenario === 'unknown-level') {+ ingestLocalAgentLogEvent(logs, { type: 'log', level: 'invalid\nreport=/private/' + privateMarker });+ } else {+ ingestLocalAgentLogEvent(logs, { type: 'log',+ level: scenario === 'warning' ? 'warn' : scenario === 'error' ? 'error' : 'info', message: privateMarker });+ }+ const native = logs.finish();+ const accepted = scenario === 'clean' || scenario === 'warning' || scenario.endsWith('-source');+ const report = { aggregate: { score: 17 }, results: [{ cliLogAnalysis: native }] };+ let calls = 0;+ const code = await runLocalAgentLauncher({ ...options(), reportsDir, run: async (_command, _args, _settings, collector) => {+ if (++calls === 2) {+ fs.writeFileSync(file, JSON.stringify(report));+ collector.ingest('stdout', 'report=' + file + '\n');+ }+ return { code: 0 };+ } });+ expect(calls).toBe(2);+ // Prove the actual saved-report consumer accepts clean/warning native+ // evidence before inspecting its schema. Child exit0 alone is insufficient.+ expect(code).toBe(accepted ? 0 : 1);+ const saved = JSON.parse(fs.readFileSync(file, 'utf8'));+ expect(saved.aggregate).toEqual({ score: 17 });+ expect(saved.results[0].cliLogAnalysis).toEqual(native);+ expect(saved.logAcceptance).toEqual({ passed: accepted,+ requiresReview: scenario !== 'clean', clean: scenario === 'clean' });+ expect(native).toMatchObject({ passed: accepted, requiresReview: scenario !== 'clean',+ coveragePassed: scenario !== 'overflow' });+ if (scenario === 'warning') expect(native.levelCounts).toMatchObject({ warn: 1, error: 0 });+ if (!accepted && scenario !== 'overflow') expect(native.levelCounts.error).toBeGreaterThan(0);+ if (scenario === 'overflow') expect(native).toMatchObject({ capturedLineCount: 1, analyzedLineCount: 0, droppedLineCount: 1 });+ expect(native.reportPaths).toEqual([]);+ expect(JSON.stringify(saved)).not.toContain(privateMarker);+ // Known native formats retain severity, not native-controlled source text.+ // Check the full real saved report, including sourceCounts/findings.source.+ expect(JSON.stringify(saved)).not.toContain(privateSourceMarker);+ const fixedSources: Record<string, string> = {+ 'native-json-source': 'native-json', 'openclaw-source': 'openclaw-json',+ 'python-source': 'python-logger', 'native-log-source': 'native-structured',+ };+ if (fixedSources[scenario]) {+ expect(native.levelCounts).toMatchObject({ warn: 1, error: 0 });+ expect(native.sourceCounts).toEqual({ [fixedSources[scenario]]: 1 });+ expect(native.findings).toEqual([expect.objectContaining({+ severity: 'warning', code: 'runtime-warning', source: fixedSources[scenario], count: 1,+ })]);+ }+ } finally { write.mockRestore(); fs.rmSync(root, { recursive: true, force: true }); }+ });+it('boots the real isolated Electron entry with a model-free TypeScript runner', () => {const root = fs.mkdtempSync(path.join(os.tmpdir(), 'local-cli-electron-'));const runner = path.join(root, 'fixture.mts'), evidence = path.join(root, 'evidence.json');diff --git a/test/eval/local-agent-quality-benchmark.test.ts b/test/eval/local-agent-quality-benchmark.test.tsindex 47232d96..c01b087e 100644--- a/test/eval/local-agent-quality-benchmark.test.ts+++ b/test/eval/local-agent-quality-benchmark.test.ts@@ -86,21 +86,22 @@ describe('Local Agent quality benchmark contract', () => {target: 'codex_skills::interface' };const warning = analyze({ type: 'stderr-line', line: JSON.stringify(record) });expect(warning).toMatchObject({ passed: true, requiresReview: true, coveragePassed: true,- levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'codex_skills::interface': 1 } });+ levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'native-json': 1 } });+ expect(JSON.stringify(warning)).not.toContain(record.target);expect(analyze({ type: 'stderr-line', line: JSON.stringify({ ...record, level: 'ERROR' }) })).toMatchObject({ passed: false, levelCounts: { error: 1 } });expect(analyze({ type: 'stderr-line', line: JSON.stringify({ ...record, level: 'ERROR',target: 'codex_core::tools::router', fields: { error: 'code-mode host closed its stdout' } }) }))- .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'codex_core::tools::router': 1 } });+ .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'native-json': 1 } });for (const line of ['unexpected stderr', '{invalid', JSON.stringify({ ...record, level: 'UNKNOWN' }),JSON.stringify({ ...record, timestamp: 'invalid' }), JSON.stringify({ ...record, fields: {} })]) {expect(analyze({ type: 'stderr-line', line }).passed).toBe(false);}const openclaw = { time: '2026-10-19T05:04:00.000Z', level: 'warn', subsystem: 'codex/app-server', message: 'Fixture compatibility warning' };expect(analyze({ type: 'stderr-line', line: JSON.stringify(openclaw) }))- .toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'codex/app-server': 1 } });+ .toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1, error: 0 }, sourceCounts: { 'openclaw-json': 1 } });expect(analyze({ type: 'stderr-line', line: JSON.stringify({ ...openclaw, level: 'error' }) }))- .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'codex/app-server': 1 } });+ .toMatchObject({ passed: false, levelCounts: { error: 1 }, sourceCounts: { 'openclaw-json': 1 } });expect(analyze({ type: 'log', level: 'info', message: 'connected' })).toMatchObject({ passed: true, requiresReview: false, levelCounts: { info: 1 } });expect(analyze({ type: 'log', level: 'error', message: 'failed\nmore detail' }))@@ -108,7 +109,7 @@ describe('Local Agent quality benchmark contract', () => {expect(analyze({ type: 'log', level: 'warn', message: 'review this' })).toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1 } });expect(analyze({ type: 'stderr-line', line: '2026-10-19 12:15:15 [INFO] acp_adapter.server: ACP client connected' }))- .toMatchObject({ passed: true, levelCounts: { info: 1, error: 0 }, sourceCounts: { 'acp_adapter.server': 1 } });+ .toMatchObject({ passed: true, levelCounts: { info: 1, error: 0 }, sourceCounts: { 'python-logger': 1 } });expect(analyze({ type: 'stderr-line', line: '2026-10-19 12:15:15 [WARNING] hermes_cli.plugins: Fixture warning' })).toMatchObject({ passed: true, requiresReview: true, levelCounts: { warn: 1 } });expect(analyze({ type: 'stderr-line', line: '2026-10-19 12:15:15 [ERROR] acp_adapter.server: Fixture failure' }).passed).toBe(false);diff --git a/test/main/features/local_agents/live_support.test.ts b/test/main/features/local_agents/live_support.test.tsindex 9cb0148f..92a64da7 100644--- a/test/main/features/local_agents/live_support.test.ts+++ b/test/main/features/local_agents/live_support.test.ts@@ -277,3 +277,17 @@ describe('local-agent process log evidence', () => {expect(collector.finish()).toMatchObject({ passed: false, capturedLineCount: 1, analyzedLineCount: 1 });});});+++describe('local-agent shared log capture boundary', () => {+ it.each([-1, 0, 1])('bounds a split native log line at 8 MiB (%s)', delta => {+ const logs = createLocalAgentLogCollector('native-cap');+ logs.ingest('stdout', 'x'.repeat(8 * 1024 * 1024 + delta));+ logs.ingest('stdout', '\n');+ const report = logs.finish();+ expect(report).toMatchObject({ passed: delta <= 0, coveragePassed: delta <= 0,+ requiresReview: delta > 0, capturedLineCount: 1, analyzedLineCount: delta <= 0 ? 1 : 0,+ droppedLineCount: delta <= 0 ? 0 : 1 });+ expect(report.findings.some(finding => finding.code === 'log-coverage-gap')).toBe(delta > 0);+ });+});