Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
import fs from "node:fs";
import path from "node:path";
const RUNTIME_LOG_RE = /^\[\d{2}:\d{2}:\d{2}\.\d{3}\]\s+\[(debug|info|warn|error)\]\s+\[\s*\(([^)]*)\)\s*\]\s*(.*)$/i;
const CASE_MARKER_RE = /^\[model-eval-case\]\s+(start|end)\s+(.+)$/;
const CASE_LOG_RE = /^\[model-eval-case-log ([A-Za-z0-9_./:-]{1,240})\] (.*)$/;
const REPORT_RE = /^report=(.+)$/;
export const ELECTRON_MACOS_TASK_POLICY_WARNING_RE = /^\[\d+:\d{4}\/\d{6}\.\d+:ERROR:base\/process\/process_mac\.cc:\d+\]\s+task_policy_set\s+TASK_(?:CATEGORY|SUPPRESSION)_POLICY:\s+\(os\/kern\)\s+invalid argument\s+\(4\)$/;
const ELECTRON_MACOS_SPELLCHECKER_WARNING_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ Electron\[\d+:\d+\] NSSpellServer dataFromCheckingString (?:timed out|succeeded), index is \d+$/;
/** The one runtime warning whose message can span several lines: AgentRunner
* splices a slice of the failed tool's result, newlines included. */
const TOOL_ERROR_PAYLOAD_RE = /\bTool\s+\S+\s+returned error:/i;
// Node-only toolFailureForLog records contain bounded correlation and parser
// facts. Admit their known shapes; arbitrary indented stderr still fails.
const NODE_TOOL_DIAGNOSTIC_SCALAR_RE = /^ (?:tool: '[A-Za-z0-9_-]+'|runner_ref: '[a-f0-9]{24}'|batch_sequence: \d+|execution_mode: '(?:single|parallel|programmatic)'|(?:session_hash|call_hash|command_hash|output_ref_hash): '[a-f0-9]{12}'|input_digest: 'sha256:[a-f0-9]{64}'|result: \{ text_hash: '[a-f0-9]{12}', text_chars: \d+ \}|output_storage: '(?:inline|persisted)'),?$/;
const NODE_TOOL_INPUT_DIAGNOSTIC_RE = /^ (?:stream_json: '(?:strict|non_strict|unobserved|limit)'|tolerant_parse: '(?:used|not_used|unknown)'|parsed_input_matches: (?:true|false)),?$/;
const STDERR_CONTEXT_CONTINUATION_RE = /^(?:\s+|\[\d+\]\s|[}\]]|\(Use\s|<\/?[a-z][\w:.-]*(?:\s[^>]*)?>?\s*$)/i;
/** A partial line held per stream until its newline arrives. A child printing
* one unbounded line must not grow the coordinator; the remainder is counted
* as a captured but never analyzed line so the coverage gate fails. */
const MAX_PARTIAL_LINE_CHARS = 8 * 1024 * 1024;
/** Exact progress records emitted by bin/ensure-runtime.cjs, not evidence that
* dependency preparation succeeded; the launcher still checks its exit status. */
function isRuntimePreparationProgress(line) {
const record = /^runtime (?:python|uv|node)(: downloading| self-check ok) (\{.*\})$/.exec(line);
if (!record) return false;
try {
const metadata = JSON.parse(record[2]);
const fields = record[1] === ": downloading" ? ["key", "asset"] : ["version"];
return Object.keys(metadata).length === fields.length
&& fields.every((field) => typeof metadata[field] === "string");
} catch {
return false;
}
}
function findingDetail(code) {
switch (code) {
case "tool-definition-soft-budget":
return "one or more tool definitions exceeded the configured soft description budget";
case "node-deprecation-warning":
return "the evaluation process used a deprecated Node.js API";
case "node-runtime-warning":
return "the evaluation process emitted a Node.js runtime warning";
case "runtime-warning":
return "the production runtime emitted a warning that requires review";
case "runtime-error":
return "the production runtime emitted an error-level log";
case "tool-execution-failed":
return "a tool failed; inspect its structured call outcome and recovery without erasing the failure";
case "plain-process-warning":
return "a non-structured process warning requires review";
case "electron-runner-failure":
return "the Electron model-evaluation runner failed before normal completion";
case "electron-runner-signaled":
return "the Electron model-evaluation runner was terminated by an operating-system signal";
case "electron-macos-task-policy-warning":
return "Chromium could not apply a non-critical macOS task policy to a child process";
case "electron-macos-spellchecker-warning":
return "the macOS spellchecking service reported a timeout or recovery requiring review";
case "unclassified-stderr":
return "unclassified stderr is treated as a process error until explicitly classified";
case "log-coverage-gap":
return "not every captured non-empty process-log line was analyzed";
case "preflight-failure":
return "the strict model-evaluation case audit failed";
case "environment-preflight-failure":
return "the normal product dependency preparation failed before evaluation dispatch";
case "launcher-failure":
return "the model-evaluation launcher could not start a required child process";
default:
return "the model-evaluation process emitted a classified diagnostic";
}
}
function findingKey(finding) {
return [
finding.severity,
finding.code,
finding.phase,
finding.source || "",
finding.caseKey || "",
finding.attribution || "",
].join("\0");
}
function aggregateFindings(findings) {
const aggregated = new Map();
for (const finding of findings) {
const key = findingKey(finding);
const current = aggregated.get(key);
if (current) {
current.count += finding.count;
} else {
aggregated.set(key, { ...finding });
}
}
return [...aggregated.values()].sort((left, right) => (
`${left.severity}/${left.phase}/${left.caseKey || ""}/${left.code}`
.localeCompare(`${right.severity}/${right.phase}/${right.caseKey || ""}/${right.code}`)
));
}
/** `onFinding(finding, line)` lets a caller keep its own bounded, redacted
* signature of the analyzed line; the collector itself never retains text. */
export function createModelEvalLogCollector(phase, { maxLineChars = MAX_PARTIAL_LINE_CHARS, onFinding } = {}) {
const state = {
phase,
buffers: { stdout: "", stderr: "" },
overflow: { stdout: false, stderr: false },
droppedBytes: 0,
droppedLineCount: 0,
capturedBytes: 0,
capturedLineCount: 0,
analyzedLineCount: 0,
blankLineCount: 0,
classificationCounts: {
runtime: 0,
processWarning: 0,
processError: 0,
control: 0,
other: 0,
},
levelCounts: { debug: 0, info: 0, warn: 0, error: 0 },
streamLineCounts: { stdout: 0, stderr: 0 },
sourceCounts: {},
activeCases: new Set(),
findings: new Map(),
reportPaths: [],
};
const contexts = new Map();
let context = state;
let attribution = {};
let currentLine;
const addFinding = (severity, code, source) => {
const finding = {
severity,
code,
count: 1,
phase,
...(source ? { source } : {}),
...attribution,
detail: findingDetail(code),
};
const key = findingKey(finding);
const existing = state.findings.get(key);
if (existing) existing.count += 1;
else state.findings.set(key, { ...finding });
onFinding?.(finding, currentLine);
};
const analyzeLine = (stream, rawLine) => {
state.capturedLineCount += 1;
state.analyzedLineCount += 1;
state.streamLineCounts[stream] += 1;
let line = rawLine.replace(/\r$/, "");
const framed = line.match(CASE_LOG_RE);
const legacyKey = state.activeCases.size === 1 ? [...state.activeCases][0] : undefined;
const key = framed?.[1] ?? legacyKey;
attribution = framed ? { caseKey: key, attribution: 'explicit' }
: key ? { caseKey: key, attribution: 'legacy-marker' }
: state.activeCases.size > 1 ? { attribution: 'ambiguous' } : {};
if (framed) line = framed[2];
currentLine = line;
// Continuation windows belong to one case and stream, not whichever case
// happened to emit the previous process-wide warning.
const contextKey = `${stream}:${key ?? '<unattributed>'}`;
if (!contexts.has(contextKey)) contexts.set(contextKey, {});
context = contexts.get(contextKey);
if (!line.trim()) {
state.blankLineCount += 1;
return;
}
const marker = line.match(CASE_MARKER_RE);
if (marker) {
context.nodeToolPayloadRemaining = 0;
context.nodeToolDiagnostic = false;
state.classificationCounts.control += 1;
if (marker[1] === "start") state.activeCases.add(marker[2].trim());
else state.activeCases.delete(marker[2].trim());
return;
}
const report = line.match(REPORT_RE);
if (report) {
state.classificationCounts.control += 1;
state.reportPaths.push(report[1].trim());
return;
}
const runtime = line.match(RUNTIME_LOG_RE);
if (runtime) {
context.nodeToolPayloadRemaining = 0;
context.nodeToolDiagnostic = false;
let level = runtime[1].toLowerCase();
const source = runtime[2].trim() || "unknown";
const message = runtime[3];
// Electron's console bridge labels Node's process-warning protocol as
// error. Preserve its warning classification without excusing ordinary
// application errors that merely mention deprecation.
const nodeDeprecation = level === "error" && source === "console"
&& /^\(node:\d+\) \[DEP\d{4}\] DeprecationWarning: /.test(message);
const nodeListenerWarning = level === "error" && source === "console"
&& /^\(node:\d+\) MaxListenersExceededWarning: /.test(message);
if (nodeDeprecation || nodeListenerWarning) level = "warn";
state.classificationCounts.runtime += 1;
state.levelCounts[level] += 1;
state.sourceCounts[source] = (state.sourceCounts[source] || 0) + 1;
if (level === "error") {
addFinding("error", "runtime-error", source);
} else if (level === "warn") {
addFinding(
"warning",
nodeDeprecation ? "node-deprecation-warning"
: nodeListenerWarning ? "node-runtime-warning"
: /tool definition description exceeds soft budget/i.test(message)
? "tool-definition-soft-budget"
: "runtime-warning",
source,
);
}
context.stderrContext = stream === "stderr" && (level === "warn" || level === "error")
? level
: undefined;
// AgentRunner logs a failed tool by splicing the first 150 characters of
// the tool result into one warn record, and a tool result carries its own
// newlines, so that single record arrives as several stderr lines. Only
// the first one keeps the log prefix; the rest used to fall through to
// the unclassified branch and fail the run for output the runtime was
// asked to print. The window opens only for this known emitter and closes
// at the next log record, so genuinely unknown stderr still fails.
context.stderrPayload = stream === "stderr" && level === "warn"
&& TOOL_ERROR_PAYLOAD_RE.test(message);
return;
}
if (stream === "stderr") {
if (line === '[agent-runner] Tool returned error {') {
state.classificationCounts.runtime += 1;
state.levelCounts.warn += 1;
context.stderrContext = undefined;
context.stderrPayload = false;
context.nodeToolDiagnostic = 'root';
addFinding('warning', 'tool-execution-failed', 'agent-runner');
return;
}
if (context.nodeToolDiagnostic) {
if (context.nodeToolDiagnostic === 'root' && line === '}') {
context.nodeToolDiagnostic = false;
state.classificationCounts.other += 1;
return;
}
if (context.nodeToolDiagnostic === 'root' && line === ' input_parse: {') {
context.nodeToolDiagnostic = 'input';
state.classificationCounts.other += 1;
return;
}
if (context.nodeToolDiagnostic === 'input' && /^ \},?$/.test(line)) {
context.nodeToolDiagnostic = 'root';
state.classificationCounts.other += 1;
return;
}
if ((context.nodeToolDiagnostic === 'root' ? NODE_TOOL_DIAGNOSTIC_SCALAR_RE : NODE_TOOL_INPUT_DIAGNOSTIC_RE).test(line)) {
state.classificationCounts.other += 1;
return;
}
context.nodeToolDiagnostic = false;
}
// Node-only core-agent uses console.warn rather than Electron's prefixed
// logger. Admit only its known tool-failure emitter, not arbitrary prose.
const toolError = /^\[agent-runner\] Tool [A-Za-z0-9_]+ returned error: ?(.*)$/.exec(line);
if (toolError) {
state.classificationCounts.runtime += 1;
state.levelCounts.warn += 1;
context.stderrContext = undefined;
context.stderrPayload = false;
context.nodeToolPayloadRemaining = Math.max(0, 150 - toolError[1].length);
addFinding("warning", "tool-execution-failed", "agent-runner");
return;
}
if (context.nodeToolPayloadRemaining > 0) {
const remaining = context.nodeToolPayloadRemaining;
context.nodeToolPayloadRemaining = 0;
if (line.length + 1 <= remaining && !/^\[[A-Za-z]/.test(line)) {
context.nodeToolPayloadRemaining = remaining - line.length - 1;
state.classificationCounts.other += 1;
return;
}
}
if (/^\[model-eval\]\s+network proxy source=/i.test(line)) {
state.classificationCounts.control += 1;
context.stderrContext = undefined;
context.stderrPayload = false;
return;
}
if (phase === "environment" && isRuntimePreparationProgress(line)) {
state.classificationCounts.control += 1;
context.stderrContext = undefined;
context.stderrPayload = false;
return;
}
if (/\bDeprecationWarning\b|\[DEP\d+\]/i.test(line)) {
state.classificationCounts.processWarning += 1;
state.levelCounts.warn += 1;
context.stderrContext = "warning";
context.stderrPayload = false;
addFinding("warning", "node-deprecation-warning", "node");
return;
}
// The explicit plain-process prefix also matches the generic Node
// Warning alternative; resolve its existing category before that branch.
if (/^warning:/i.test(line)) {
state.classificationCounts.processWarning += 1;
state.levelCounts.warn += 1;
context.stderrContext = "warning";
context.stderrPayload = false;
addFinding("warning", "plain-process-warning", "stderr");
return;
}
if (/\b(?:ExperimentalWarning|MaxListenersExceededWarning|Warning):/i.test(line)) {
state.classificationCounts.processWarning += 1;
state.levelCounts.warn += 1;
context.stderrContext = "warning";
context.stderrPayload = false;
addFinding("warning", "node-runtime-warning", "node");
return;
}
if (ELECTRON_MACOS_TASK_POLICY_WARNING_RE.test(line)) {
state.classificationCounts.processWarning += 1;
state.levelCounts.warn += 1;
context.stderrContext = "warning";
context.stderrPayload = false;
addFinding("warning", "electron-macos-task-policy-warning", "electron");
return;
}
if (ELECTRON_MACOS_SPELLCHECKER_WARNING_RE.test(line)) {
state.classificationCounts.processWarning += 1;
state.levelCounts.warn += 1;
context.stderrContext = undefined;
context.stderrPayload = false;
addFinding("warning", "electron-macos-spellchecker-warning", "electron");
return;
}
if (/model-eval Electron runner failed|failed to start .*runner/i.test(line)) {
state.classificationCounts.processError += 1;
state.levelCounts.error += 1;
context.stderrContext = "error";
context.stderrPayload = false;
addFinding("error", "electron-runner-failure", "launcher");
return;
}
if (
context.stderrContext
&& STDERR_CONTEXT_CONTINUATION_RE.test(line)
) {
state.classificationCounts.other += 1;
return;
}
if (context.stderrPayload) {
state.classificationCounts.other += 1;
return;
}
state.classificationCounts.processError += 1;
state.levelCounts.error += 1;
context.stderrContext = "error";
context.stderrPayload = false;
addFinding("error", "unclassified-stderr", "stderr");
return;
}
if (/^warning:/i.test(line)) {
state.classificationCounts.processWarning += 1;
state.levelCounts.warn += 1;
addFinding("warning", "plain-process-warning", "stdout");
return;
}
state.classificationCounts.other += 1;
};
// The end of a line that exceeded the cap: captured, never analyzed.
const countDroppedLine = (stream) => {
state.overflow[stream] = false;
state.capturedLineCount += 1;
state.streamLineCounts[stream] += 1;
state.droppedLineCount += 1;
};
const ingest = (stream, chunk) => {
const text = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
state.capturedBytes += Buffer.byteLength(text, "utf8");
state.buffers[stream] += text;
const lines = state.buffers[stream].split("\n");
state.buffers[stream] = lines.pop() || "";
for (const line of lines) {
if (state.overflow[stream]) {
state.droppedBytes += Buffer.byteLength(line, "utf8");
countDroppedLine(stream);
continue;
}
analyzeLine(stream, line);
}
if (state.overflow[stream] || state.buffers[stream].length > maxLineChars) {
state.droppedBytes += Buffer.byteLength(state.buffers[stream], "utf8");
state.buffers[stream] = "";
state.overflow[stream] = true;
}
};
const finish = () => {
for (const stream of ["stdout", "stderr"]) {
if (state.overflow[stream]) countDroppedLine(stream);
else if (state.buffers[stream].length) analyzeLine(stream, state.buffers[stream]);
state.buffers[stream] = "";
}
const coveragePassed = state.capturedLineCount === state.analyzedLineCount;
const findings = aggregateFindings([
...state.findings.values(),
...(coveragePassed ? [] : [{
severity: "error",
code: "log-coverage-gap",
count: 1,
phase,
detail: findingDetail("log-coverage-gap"),
}]),
]);
return {
policyVersion: 2,
phase,
passed: coveragePassed && !findings.some((finding) => finding.severity === "error"),
requiresReview: findings.length > 0,
coveragePassed,
capturedBytes: state.capturedBytes,
capturedLineCount: state.capturedLineCount,
analyzedLineCount: state.analyzedLineCount,
droppedLineCount: state.droppedLineCount,
droppedBytes: state.droppedBytes,
blankLineCount: state.blankLineCount,
classificationCounts: { ...state.classificationCounts },
levelCounts: { ...state.levelCounts },
streamLineCounts: { ...state.streamLineCounts },
sourceCounts: { ...state.sourceCounts },
findings,
reportPaths: [...new Set(state.reportPaths)],
};
};
return { ingest, finish, addFinding: (...args) => { attribution = {}; currentLine = undefined; addFinding(...args); } };
}
export function mergeModelEvalLogAnalyses(analyses) {
const findings = aggregateFindings(analyses.flatMap((analysis) => analysis.findings));
const sumObject = (key) => Object.fromEntries(
[...new Set(analyses.flatMap((analysis) => Object.keys(analysis[key] || {})))]
.map((name) => [
name,
analyses.reduce((sum, analysis) => sum + Number(analysis[key]?.[name] || 0), 0),
]),
);
const capturedLineCount = analyses.reduce((sum, item) => sum + item.capturedLineCount, 0);
const analyzedLineCount = analyses.reduce((sum, item) => sum + item.analyzedLineCount, 0);
const coveragePassed = analyses.every((item) => item.coveragePassed)
&& capturedLineCount === analyzedLineCount;
return {
policyVersion: 2,
passed: coveragePassed && !findings.some((finding) => finding.severity === "error"),
requiresReview: findings.length > 0,
coveragePassed,
capturedBytes: analyses.reduce((sum, item) => sum + item.capturedBytes, 0),
capturedLineCount,
analyzedLineCount,
droppedLineCount: analyses.reduce((sum, item) => sum + Number(item.droppedLineCount || 0), 0),
droppedBytes: analyses.reduce((sum, item) => sum + Number(item.droppedBytes || 0), 0),
blankLineCount: analyses.reduce((sum, item) => sum + item.blankLineCount, 0),
classificationCounts: sumObject("classificationCounts"),
levelCounts: sumObject("levelCounts"),
streamLineCounts: sumObject("streamLineCounts"),
sourceCounts: sumObject("sourceCounts"),
phases: analyses.map((analysis) => ({
phase: analysis.phase,
passed: analysis.passed,
requiresReview: analysis.requiresReview,
capturedLineCount: analysis.capturedLineCount,
analyzedLineCount: analysis.analyzedLineCount,
})),
findings,
reportPaths: [...new Set(analyses.flatMap((analysis) => analysis.reportPaths || []))],
};
}
export function attachProcessLogAnalysis(reportPath, analysis) {
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
report.processLogAnalysis = analysis;
const temporary = `${reportPath}.logs-${process.pid}.tmp`;
fs.writeFileSync(temporary, `${JSON.stringify(report, null, 2)}\n`, "utf8");
fs.renameSync(temporary, reportPath);
}
export function attachProcessTermination(reportPath, termination) {
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
const observedAt = new Date().toISOString();
const signal = String(termination?.signal || "unknown").slice(0, 32);
report.processTermination = {
kind: "signal",
signal,
observedAt,
};
if (report.runStatus === "running") {
report.runStatus = "failed";
report.completedAt = observedAt;
report.failure = {
code: "runner-signaled",
phase: "process-lifecycle",
...(report.progress?.activeCase ? { caseKey: report.progress.activeCase } : {}),
detail: `the Electron model-evaluation runner was terminated by ${signal}`,
};
}
const temporary = `${reportPath}.termination-${process.pid}.tmp`;
fs.writeFileSync(temporary, `${JSON.stringify(report, null, 2)}\n`, "utf8");
fs.renameSync(temporary, reportPath);
}
export function resolveReportedPath(cwd, reportedPath) {
return path.isAbsolute(reportedPath) ? reportedPath : path.resolve(cwd, reportedPath);
}
export function formatProcessLogSummary(analysis) {
const status = !analysis.passed ? "FAIL" : analysis.requiresReview ? "WARN" : "PASS";
const findingCounts = analysis.findings
.map((finding) => `${finding.severity}:${finding.code}=${finding.count}`)
.join(",");
return `[model-eval-log-analysis] ${status}`
+ ` coverage=${analysis.analyzedLineCount}/${analysis.capturedLineCount}`
+ ` warnings=${analysis.levelCounts.warn || 0}`
+ ` errors=${analysis.levelCounts.error || 0}`
+ (findingCounts ? ` findings=${findingCounts}` : "");
}