Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { describe, it, expect } from 'vitest';
import {
FACTS_TOKEN_CAP,
analyzeLines,
estimateTokensMirror,
classifyHeaderLine,
coldStartsFromTimestamps,
jsonlLineTimestamp,
percentiles,
summarizeSidecar,
} from '../../scripts/analyze-context-gates.mjs';
import {
HISTORY_EXACT_FACTS_MAX_TOKENS,
estimateTextTokens,
} from '../../src/core-agent/src/agent/session.js';
// Synthetic lines reproducing the electron-log pretty-print shape:
// header line, then indented object fields. The analyzer's evidence feeds
// retirement/tuning decisions, so a parser that over- or under-matches
// produces wrong decisions — both directions are pinned here.
const HEADER = (message: string, level = 'info') =>
`[2026-08-13 10:00:00.000] [${level}] [ (console) ] [agent-runner] ${message} {`;
describe('analyze-context-gates classification', () => {
it('classifies real gate header shapes', () => {
expect(classifyHeaderLine(HEADER('context compaction done'))?.gate).toBe('compaction_done');
expect(classifyHeaderLine(HEADER('context compaction skipped', 'error'))?.gate).toBe('ceiling_diagnostic');
expect(classifyHeaderLine(HEADER('emergency context reduction applied', 'error'))?.gate).toBe('l3_applied');
expect(classifyHeaderLine(HEADER('context overflow recovery applied', 'warn'))?.gate).toBe('g9_recovered');
expect(
classifyHeaderLine(
'[2026-08-13 10:00:01.000] [warn] [ (console) ] [agent-runner] loop_detection: identical tool call repeated 5x — stopping run',
)?.gate,
).toBe('loop_exact');
expect(
classifyHeaderLine(HEADER('run_progress: nudged model after consecutive unsuccessful tool rounds'))?.gate,
).toBe('no_progress_nudge');
});
it('rejects look-alikes that must not count as gate firings', () => {
// Message equality, not substring: a different message sharing the prefix.
expect(classifyHeaderLine(HEADER('context compaction startup checks'))?.gate).toBeNull();
// An indented FIELD line whose string value quotes a gate message.
expect(classifyHeaderLine(" note: 'context compaction start',")).toBeNull();
// Free text inside a multi-line value (no timestamp header shape).
expect(classifyHeaderLine('context compaction done')).toBeNull();
// A header from a random module still closes scope but is not a gate.
expect(classifyHeaderLine(HEADER('workspace observations reconciled'))?.gate).toBeNull();
});
});
describe('analyze-context-gates aggregation', () => {
it('refines compaction lifecycle by phase, attributes metrics, splits reasons', () => {
const lines = [
HEADER('context compaction done'),
" phase: 'history_summary',",
' tokensBefore: 30000,', // not allowed under l1_done → ignored
' readsSinceLastCompaction: 4,',
HEADER('context compaction done'),
" phase: 'active_checkpoint',",
' appliedCheckpointTokens: 1500,',
HEADER('context compaction skipped', 'error'),
" reason: 'layered_triggers_exceeded',",
' tokensBefore: 25000,',
HEADER('workspace observations reconciled'),
' appliedCheckpointTokens: 999999,', // out of scope → ignored
// A foreign-transport record whose header shape the strict regex does
// not recognize: the non-indented line must CLOSE scope so its fields
// cannot bleed into the last gate's counts (the bug the first live run
// exposed: unrelated `reason: 'documents'` rows under every gate).
HEADER('context compaction skipped', 'error'),
'2026-08-13T10:00:02.000Z some-other-transport reason line',
" reason: 'documents',",
' readsSinceLastCompaction: 999,',
];
const { gates, samples } = analyzeLines(lines);
expect(gates).toMatchObject({
l1_done: 1,
l2_done: 1,
ceiling_diagnostic: 2,
'ceiling_diagnostic:layered_triggers_exceeded': 1,
});
expect(gates).not.toHaveProperty('compaction_done');
expect(gates).not.toHaveProperty('ceiling_diagnostic:documents');
expect(samples.appliedCheckpointTokens).toEqual([1500]);
expect(samples.readsSinceLastCompaction).toEqual([4]);
expect(samples.tokensBefore).toEqual([25000]);
});
it('percentiles summarize without leaking raw values order', () => {
expect(percentiles([])).toBeNull();
expect(percentiles([5, 1, 9, 3, 7])).toEqual({ n: 5, min: 1, p50: 5, p90: 9, max: 9 });
});
});
describe('evidence modes', () => {
it('summarizes sidecar water levels as sizes only, tolerating absent fields', () => {
// getSerializedContextState omits empty fields — a minimal sidecar must
// read as zeros, and a populated one must report sizes, never content.
expect(summarizeSidecar({ version: 1, completedTurns: [{}], resources: [] })).toEqual({
historySummaryChars: 0,
factsCount: 0,
factsChars: 0,
factsEstTokens: 0,
checkpointChars: 0,
completedTurns: 1,
resources: 0,
});
const populated = summarizeSidecar({
historySummary: 'x'.repeat(1_500),
historyExactFacts: ['- fact=1', '- 长事实'.repeat(10)],
activeTurn: { checkpointSummary: 'c'.repeat(300) },
completedTurns: [{}, {}],
resources: [{}],
});
expect(populated.historySummaryChars).toBe(1_500);
expect(populated.factsCount).toBe(2);
expect(populated.factsChars).toBe('- fact=1'.length + '- 长事实'.repeat(10).length);
expect(populated.factsEstTokens).toBe(
estimateTokensMirror('- fact=1') + estimateTokensMirror('- 长事实'.repeat(10)),
);
expect(populated.checkpointChars).toBe(300);
expect(JSON.stringify(populated)).not.toContain('fact=1');
});
it('reads only the trailing top-level ts, never one embedded in content', () => {
// Accepted: the real writer shape — JSON.stringify({ ...record, ts }) puts
// the record timestamp as the LAST key before the closing brace.
expect(jsonlLineTimestamp('{"role":"user","content":[{"type":"text","text":"hi"}],"ts":1755021600000}'))
.toBe(1755021600000);
expect(jsonlLineTimestamp('{"role":"assistant","content":[],"turnId":3,"ts":1755021600001}'))
.toBe(1755021600001);
// Rejected look-alike: a tool_result whose CONTENT serializes an API
// payload with its own epoch "ts". Reading that value fabricated an idle
// gap between the payload's clock and the record's clock, inflating
// cold-start counts. The record here has no trailing top-level ts at all
// (legacy line), so it must be skipped, not mis-timed.
expect(jsonlLineTimestamp('{"role":"user","content":[{"type":"tool_result","content":"{\\"ts\\": 1600000000000, \\"ok\\":true}"}]}'))
.toBeNull();
// Rejected: embedded ts present AND a real trailing ts — the trailing one wins.
expect(jsonlLineTimestamp('{"role":"user","content":[{"type":"tool_result","content":"{\\"ts\\": 1600000000000}"}],"ts":1755021600002}'))
.toBe(1755021600002);
// Rejected: non-JSONL noise.
expect(jsonlLineTimestamp('')).toBeNull();
expect(jsonlLineTimestamp('"ts": 1600000000000')).toBeNull();
});
it('pins the sidecar cap mirror and estimator weights to the session module', () => {
// The plain-node analyzer cannot import the TS session module, so it
// mirrors the token cap and the estimator. If this fails, update
// FACTS_TOKEN_CAP / estimateTokensMirror in the script to match session.ts.
expect(FACTS_TOKEN_CAP).toBe(HISTORY_EXACT_FACTS_MAX_TOKENS);
for (const text of ['', 'plain ascii 1234', '中文事实字段', 'かなカナ', '한글', 'mixed 中英 with 😀 and 𠮷']) {
expect(estimateTokensMirror(text)).toBe(estimateTextTokens(text));
}
});
it('finds cold-start gaps beyond the TTL with their byte offsets', () => {
const hour = 3_600_000;
const entries = [
{ ts: 0, byteOffset: 0 },
{ ts: 10 * 60_000, byteOffset: 4_000 }, // 10min gap: warm
{ ts: 10 * 60_000 + 9 * hour, byteOffset: 9_000 }, // 9h gap: cold
{ ts: 10 * 60_000 + 9 * hour + 30_000, byteOffset: 12_000 }, // 30s: warm
];
const gaps = coldStartsFromTimestamps(entries, hour);
expect(gaps).toHaveLength(1);
expect(gaps[0].gapHours).toBeCloseTo(9, 5);
expect(gaps[0].bytesBefore).toBe(9_000);
// TTL boundary is exclusive: exactly-TTL gaps are not cold.
expect(coldStartsFromTimestamps([{ ts: 0, byteOffset: 0 }, { ts: hour, byteOffset: 1 }], hour)).toHaveLength(0);
});
it('reports incomplete sidecar evidence without exposing skipped paths or content', () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-context-evidence-'));
const sessionsDir = path.join(tmpDir, 'user-private', 'cloud', 'sessions');
const unreadableSessions = path.join(tmpDir, 'user-other', 'cloud', 'sessions');
fs.mkdirSync(sessionsDir, { recursive: true });
fs.mkdirSync(path.dirname(unreadableSessions), { recursive: true });
fs.writeFileSync(path.join(sessionsDir, 'valid.context.json'), JSON.stringify({ completedTurns: [{}] }));
fs.writeFileSync(path.join(sessionsDir, 'private-name.context.json'), '{private-content');
fs.writeFileSync(unreadableSessions, 'not-a-directory');
try {
const result = spawnSync(process.execPath, [
path.resolve('scripts/analyze-context-gates.mjs'),
'--sidecars', '--json', '--data', tmpDir,
], { encoding: 'utf8' });
const report = JSON.parse(result.stdout);
expect(result.status).toBe(0);
expect(report.evidence).toEqual({
complete: false,
dataDirectoryReadable: true,
unreadableSessionDirectories: 1,
discoveredFiles: 2,
processedFiles: 1,
skippedFiles: 1,
});
expect(result.stderr).toContain('sidecar evidence incomplete');
expect(result.stderr).not.toContain('user-private');
expect(result.stderr).not.toContain('private-name');
expect(result.stderr).not.toContain('private-content');
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it('fails closed when the evidence data directory is unavailable', () => {
const missingDir = path.join(os.tmpdir(), `orkas-missing-evidence-${process.pid}-${Date.now()}`);
const result = spawnSync(process.execPath, [
path.resolve('scripts/analyze-context-gates.mjs'),
'--cold-starts', '--json', '--data', missingDir,
], { encoding: 'utf8' });
const report = JSON.parse(result.stdout);
expect(result.status).toBe(2);
expect(report.evidence).toMatchObject({
complete: false,
dataDirectoryReadable: false,
discoveredFiles: 0,
processedFiles: 0,
skippedFiles: 0,
});
expect(result.stderr).toContain('cold-start evidence incomplete');
});
});