Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

test(eval): chat, execute, memory, provider and /trace scenarios

Salman Qureshid473bce4ddec
diff --git a/src/eval/scenarios/chat.test.ts b/src/eval/scenarios/chat.test.ts
new file mode 100644
index 00000000..4146ee76
--- /dev/null
+++ b/src/eval/scenarios/chat.test.ts
@@ -0,0 +1,90 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { runScenario, type TurnResult } from '../runner.js';
+import { call, textStep, toolStep } from '../fixture.js';
+
+const NO_GUARD_TEXT = [
+ 'getting started on the real build',
+ "couldn't get started",
+ 'Checking my work before I call it done',
+ 'Taking a completely different run',
+];
+
+function expectTimings(turn: TurnResult): void {
+ expect(turn.timings.steps.length).toBe(turn.providerCalls.length);
+ expect(turn.timings.promptBuildMs).toBeGreaterThan(0);
+ for (const step of turn.timings.steps) {
+ expect(step.startMs).toBeGreaterThan(0);
+ expect(step.durationMs).toBeGreaterThanOrEqual(0);
+ }
+ expect(turn.timings.deliveryMs).toBeGreaterThanOrEqual(0);
+ expect(turn.timings.totalMs).toBeGreaterThanOrEqual(turn.timings.promptBuildMs);
+}
+
+describe('eval · chat mode never forces guard rounds', () => {
+ afterAll(async () => {
+ const { removeEvalRoot } = await import('../env.js');
+ removeEvalRoot();
+ });
+
+ it('"make me a workout plan" → one delivered answer, zero forced rounds', async () => {
+ const plan = 'Here is a 3-day plan:\n- Day 1: squats, push-ups\n- Day 2: rest and a 30 minute walk\n- Day 3: deadlifts, rows, planks';
+ const result = await runScenario({
+ name: 'chat-workout-plan',
+ turns: [{
+ message: 'make me a workout plan',
+ steps: [textStep(plan, { inputTokens: 900, outputTokens: 60 })],
+ expect: {
+ deliveredCount: 1,
+ finalTextIncludes: ['3-day plan'],
+ providerCalls: 1,
+ forcedActionRounds: 0,
+ verificationRounds: 0,
+ pause: null,
+ noEventIncludes: NO_GUARD_TEXT,
+ },
+ }],
+ });
+ expect(result.failures).toEqual([]);
+ const [turn] = result.turns;
+ expect(turn.guard.forcedToolChoiceCalls).toBe(0);
+ expect(turn.toolCalls).toEqual([]);
+ expect(turn.tokens).toMatchObject({ input: 900, output: 60 });
+ expect(turn.usedProvider).toBe('scripted');
+ // user message + assistant answer, no tool trace for a tool-free turn
+ expect(turn.sessionEntries.map((m) => `${m.role}:${m.kind}`)).toEqual(['user:message', 'assistant:message']);
+ expectTimings(turn);
+ });
+
+ it('"write my notes to notes.md" → real create_file runs, answer delivered, no verification nudge', async () => {
+ const result = await runScenario({
+ name: 'chat-write-notes',
+ turns: [{
+ message: 'write my notes to notes.md',
+ steps: [
+ toolStep([call('create_file', { path: 'notes.md', content: '# Notes\n- buy milk\n' })]),
+ textStep('Saved your notes to notes.md.'),
+ ],
+ expect: {
+ deliveredCount: 1,
+ finalTextIncludes: ['notes.md'],
+ providerCalls: 2,
+ forcedActionRounds: 0,
+ verificationRounds: 0,
+ pause: null,
+ toolsUsed: ['create_file'],
+ noEventIncludes: NO_GUARD_TEXT,
+ },
+ }],
+ });
+ expect(result.failures).toEqual([]);
+ const [turn] = result.turns;
+ expect(turn.toolCalls[0].recorded).toBe(false);
+ expect(readFileSync(join(result.workDir, 'notes.md'), 'utf8')).toContain('buy milk');
+ expect(existsSync(join(result.workDir, 'notes.md'))).toBe(true);
+ // The tool result went back to the model on the second call.
+ expect(JSON.stringify(turn.providerCalls[1].prompt)).toContain('tool-result');
+ expectTimings(turn);
+ });
+});
diff --git a/src/eval/scenarios/execute.test.ts b/src/eval/scenarios/execute.test.ts
new file mode 100644
index 00000000..263385d9
--- /dev/null
+++ b/src/eval/scenarios/execute.test.ts
@@ -0,0 +1,138 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { runScenario } from '../runner.js';
+import { call, textStep, toolStep, type ScriptedStep } from '../fixture.js';
+import { STEPS_PAUSED_BANNER, VERIFICATION_FAILED_BANNER } from '../../core/completion-verdict.js';
+
+const TASK = 'fix the failing add() test in src/math.ts';
+const edit = call('edit_file', { path: 'src/math.ts', old_string: 'return a - b;', new_string: 'return a + b;' }, 'Successfully edited src/math.ts (1 replacement)');
+const failingTest = call('run_command', { command: 'npm test' }, 'Command exited with code 1\n\nFAIL src/math.test.ts > add > adds two numbers\nAssertionError: expected -1 to be 3');
+const passingTest = call('run_command', { command: 'npm test' }, '> vitest run\n\n ✓ src/math.test.ts (3 tests) 4ms\n\n Test Files 1 passed (1)\n Tests 3 passed (3)');
+
+/**
+ * KNOWN REGRESSION (found by this harness, fix belongs in core/agent.ts):
+ * on the STREAMING path handleMessage stores `result = { text, usage,
+ * reasoning }` without `finishReason`, so `classifyTurnEnd` reads
+ * `undefined` → 'interrupted' for every streamed turn. The verification gate
+ * (`turnEnd() === 'text-stop'`) and step-budget continuation/pause
+ * (`'steps-exhausted'`) are therefore dead on every streaming channel (CLI,
+ * web, Telegram/Discord/Slack with streaming on, Signal). The scenarios below
+ * pin the correct behaviour on the non-streaming path and mark the streaming
+ * variants `it.fails` — when agent.ts adds `finishReason` to the streamed
+ * result they start passing, `it.fails` turns red, and the marker must go.
+ */
+const PATHS = [
+ { label: 'non-streaming', streaming: false, gateWorks: true },
+ { label: 'streaming', streaming: true, gateWorks: false },
+] as const;
+
+afterAll(async () => (await import('../env.js')).removeEvalRoot());
+
+describe.each(PATHS)('eval · execute mode completion contract ($label delivery)', ({ label, streaming, gateWorks }) => {
+ const itGate = gateWorks ? it : it.fails;
+
+ const verifyFail = () => runScenario({
+ name: `execute-verify-fail-${label}`,
+ channel: { streaming },
+ programmingMode: 'execute',
+ turns: [{
+ message: TASK,
+ steps: [
+ toolStep([edit]),
+ toolStep([failingTest]),
+ textStep('I fixed the sign in add().'),
+ // The verification gate forces exactly one evidence round
+ // (toolChoice: required); it fails again.
+ toolStep([failingTest]),
+ textStep('The test still fails.'),
+ ],
+ expect: {
+ pause: 'verification-failed',
+ forcedActionRounds: 0,
+ toolsUsed: ['edit_file', 'run_command'],
+ noEventIncludes: ['Task complete'],
+ anyEventIncludes: [VERIFICATION_FAILED_BANNER, 'npm test ✗'],
+ },
+ }],
+ });
+
+ it('edit → FAILING npm test → text: pauses with the verification-failed banner, never "Task complete"', async () => {
+ const result = await verifyFail();
+ // The script carries a step for the forced evidence round; the
+ // streaming path never takes it (see KNOWN REGRESSION above).
+ const failures = gateWorks ? result.failures : result.failures.filter((f) => !f.includes('more model calls'));
+ expect(failures).toEqual([]);
+ const [turn] = result.turns;
+ expect(turn.finalText).toContain(VERIFICATION_FAILED_BANNER);
+ expect(turn.finalText).toMatch(/\/trace [a-z0-9]+/i);
+ expect(turn.events.some((e) => e.kind === 'completion')).toBe(false);
+ // A pause is not a completed answer: no assistant message is stored.
+ expect(turn.sessionEntries.some((m) => m.role === 'assistant' && m.kind === 'message')).toBe(false);
+ expect(turn.timings.steps.length).toBe(turn.providerCalls.length);
+ expect(turn.timings.promptBuildMs).toBeGreaterThan(0);
+ });
+
+ itGate('the verification gate forces exactly one evidence round (toolChoice: required)', async () => {
+ const [turn] = (await verifyFail()).turns;
+ expect(turn.guard.verificationRounds).toBe(1);
+ expect(turn.guard.forcedToolChoiceCalls).toBe(1);
+ expect(turn.providerCalls).toHaveLength(5);
+ expect(turn.notices.some((n) => n.includes('Checking my work'))).toBe(true);
+ });
+
+ it('edit → passing npm test → completes with a ✓ verification note and a /trace id', async () => {
+ const result = await runScenario({
+ name: `execute-verify-pass-${label}`,
+ channel: { streaming },
+ programmingMode: 'execute',
+ turns: [{
+ message: TASK,
+ steps: [toolStep([edit]), toolStep([passingTest]), textStep('Fixed add(): it now returns a + b. All 3 tests pass.')],
+ expect: {
+ deliveredCount: 1,
+ pause: null,
+ verificationRounds: 0,
+ forcedActionRounds: 0,
+ providerCalls: 3,
+ finalTextIncludes: ['Fixed add()', 'npm test ✓', '/trace '],
+ },
+ }],
+ });
+ expect(result.failures).toEqual([]);
+ const [turn] = result.turns;
+ expect(turn.traceId).toBeTruthy();
+ expect(turn.finalText).toContain(`/trace ${turn.traceId}`);
+ const answer = turn.sessionEntries.find((m) => m.role === 'assistant' && m.kind === 'message');
+ expect(answer?.metadata).toMatchObject({ verification: 'npm test ✓', steps: 3, provider: 'scripted' });
+ expect(turn.events.some((e) => e.kind === 'completion')).toBe(false); // short task: footer, no banner
+ expect(turn.timings.steps.map((s) => s.durationMs).every((d) => d >= 0)).toBe(true);
+ expect(turn.timings.deliveryMs).toBeGreaterThanOrEqual(0);
+ });
+
+ itGate('step budget exhausted → honest pause banner, delivered as a persistent message (not a status-card notice)', async () => {
+ let n = 0;
+ const endless = (): ScriptedStep => {
+ n++;
+ return toolStep([call('create_file', { path: `gen/part-${n}.ts`, content: `export const part${n} = ${n};\n` }, `Created gen/part-${n}.ts`)]);
+ };
+ const result = await runScenario({
+ name: `execute-steps-exhausted-${label}`,
+ channel: { streaming },
+ programmingMode: 'execute',
+ turns: [{ message: 'build the generator module', onExhausted: endless, expect: { pause: 'steps', forcedActionRounds: 0, noEventIncludes: ['Task complete'] } }],
+ });
+ expect(result.failures).toEqual([]);
+ const [turn] = result.turns;
+ // Bounded automatic continuations happened before the pause.
+ expect(turn.guard.stepBudgetContinuations).toBe(6); // MAX_AUTOMATIC_CONTINUATIONS
+ expect(turn.delivered.some((t) => t.includes('(no text response)'))).toBe(false);
+ const banner = turn.events.find((e) => e.text.includes(STEPS_PAUSED_BANNER));
+ expect(banner?.kind).toBe('delivered');
+ expect(turn.notices.some((t) => t.includes(STEPS_PAUSED_BANNER))).toBe(false);
+ // The task was ended BEFORE the banner was sent, so it is not swallowed by the status card.
+ const end = turn.events.findIndex((e) => e.kind === 'task-end');
+ expect(end).toBeGreaterThanOrEqual(0);
+ expect(turn.events.indexOf(banner!)).toBeGreaterThan(end);
+ expect(turn.finalText).toContain(STEPS_PAUSED_BANNER);
+ expect(turn.timings.steps.length).toBe(turn.providerCalls.length);
+ });
+});
diff --git a/src/eval/scenarios/memory.test.ts b/src/eval/scenarios/memory.test.ts
new file mode 100644
index 00000000..9da2b872
--- /dev/null
+++ b/src/eval/scenarios/memory.test.ts
@@ -0,0 +1,90 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { runScenario } from '../runner.js';
+import { call, textStep, toolStep } from '../fixture.js';
+import { promptText } from '../script-player.js';
+
+const TRACE_HEADER = '[Tool activity in my previous turn]';
+
+/**
+ * KNOWN REGRESSION (found by this harness, fix belongs in core/agent.ts):
+ * the per-turn tool trace (`turnToolTrace`, written at turn end as the
+ * `[Tool activity in my previous turn]` session entry) is only fed by
+ * `noteToolSteps` in the guard / step-budget / verification continuation
+ * rounds. The MAIN loop's `onStepFinish` (streamText and generateText
+ * paths) never calls it, so an ordinary tool-using turn leaves no trace and
+ * the next turn starts blind. The first scenario is `it.fails` until that
+ * call is added; the read side is pinned by the passing scenarios.
+ */
+describe('eval · cross-turn memory carries the tool trace', () => {
+ afterAll(async () => (await import('../env.js')).removeEvalRoot());
+
+ it.fails('main-loop tool calls are written to the trace the NEXT turn sends to the model', async () => {
+ const result = await runScenario({
+ name: 'memory-main-loop-trace',
+ turns: [
+ {
+ message: 'save a packing list for my trip',
+ steps: [
+ toolStep([call('create_file', { path: 'trip/packing.md', content: '- passport\n- charger\n' }, 'Created trip/packing.md')]),
+ textStep('Saved your packing list to trip/packing.md.'),
+ ],
+ expect: { deliveredCount: 1, toolsUsed: ['create_file'] },
+ },
+ { message: 'which file did you put it in?', steps: [textStep('It is in trip/packing.md.')], expect: { deliveredCount: 1, providerCalls: 1 } },
+ ],
+ });
+ expect(result.failures).toEqual([]);
+ const input = promptText(result.turns[1].providerCalls[0].prompt);
+ expect(input).toContain(TRACE_HEADER);
+ expect(input).toContain('create_file path=trip/packing.md');
+ });
+
+ it('a trace written during a turn (verification round) reaches the next turn\'s model input', async () => {
+ const result = await runScenario({
+ name: 'memory-continuation-trace',
+ programmingMode: 'execute',
+ channel: { streaming: false },
+ turns: [
+ {
+ message: 'fix the failing add() test in src/math.ts',
+ steps: [
+ toolStep([call('edit_file', { path: 'src/math.ts', old_string: 'a - b', new_string: 'a + b' }, 'Successfully edited src/math.ts')]),
+ textStep('Fixed the sign.'),
+ // forced verification round (its tool steps ARE traced)
+ toolStep([call('run_command', { command: 'npm test' }, 'Tests 3 passed (3)')]),
+ textStep('Verified: npm test passes.'),
+ ],
+ expect: { verificationRounds: 1, pause: null },
+ },
+ { message: 'what did you run to check it?', steps: [textStep('npm test.')], expect: { deliveredCount: 1, providerCalls: 1 } },
+ ],
+ });
+ expect(result.failures).toEqual([]);
+ const [first, second] = result.turns;
+ const trace = first.sessionEntries.find((m) => m.kind === 'tool-call');
+ expect(trace?.content.startsWith(TRACE_HEADER)).toBe(true);
+ expect(trace?.content).toContain('run_command command=npm test');
+
+ const input = promptText(second.providerCalls[0].prompt);
+ expect(input).toContain(TRACE_HEADER);
+ expect(input).toContain('run_command command=npm test');
+ // Trace, then the previous answer, then the new question.
+ expect(input.indexOf(TRACE_HEADER)).toBeLessThan(input.indexOf('Verified: npm test passes.'));
+ expect(input.indexOf('Verified: npm test passes.')).toBeLessThan(input.indexOf('what did you run to check it?'));
+ });
+
+ it('a seeded session with a prior tool trace reaches the model input', async () => {
+ const result = await runScenario({
+ name: 'memory-seeded-trace',
+ history: [
+ { role: 'user', content: 'check the build config' },
+ { role: 'assistant', kind: 'tool-call', content: `${TRACE_HEADER}\n- read_file path=tsup.config.ts → ok (24 lines)` },
+ { role: 'assistant', content: 'The build targets node20 with ESM output.' },
+ ],
+ turns: [{ message: 'and the entry point?', steps: [textStep('src/index.ts.')], expect: { deliveredCount: 1 } }],
+ });
+ expect(result.failures).toEqual([]);
+ const input = promptText(result.turns[0].providerCalls[0].prompt);
+ expect(input).toContain(`${TRACE_HEADER}\n- read_file path=tsup.config.ts`);
+ });
+});
diff --git a/src/eval/scenarios/provider.test.ts b/src/eval/scenarios/provider.test.ts
new file mode 100644
index 00000000..2183d41e
--- /dev/null
+++ b/src/eval/scenarios/provider.test.ts
@@ -0,0 +1,41 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { EvalHarness } from '../runner.js';
+import { textStep } from '../fixture.js';
+
+describe('eval · provider fallback', () => {
+ afterAll(async () => (await import('../env.js')).removeEvalRoot());
+
+ it('first provider throws, second answers; the next turn starts on the configured default again', async () => {
+ const harness = await EvalHarness.create({
+ name: 'provider-fallback',
+ providers: [{ name: 'primary', failFirst: 1, failWith: 'primary: 503 Service Unavailable' }, { name: 'backup' }],
+ turns: [],
+ });
+ try {
+ const first = await harness.runTurn({
+ message: 'what should I cook tonight',
+ steps: [textStep('Try a mushroom risotto.')],
+ expect: { deliveredCount: 1, usedProvider: 'backup', finalTextIncludes: ['risotto'] },
+ });
+ expect(first.failures).toEqual([]);
+ expect(first.providerCalls.map((c) => [c.provider, c.error ? 'error' : 'ok'])).toEqual([['primary', 'error'], ['backup', 'ok']]);
+ expect(first.guard.providerNotices.some((n) => n.includes('switching to `backup`'))).toBe(true);
+ expect(first.guard.providerNotices.some((n) => n.includes('served by `backup`'))).toBe(true);
+ expect(first.timings.steps[0].error).toContain('503');
+ expect(harness.providerRegistry.getLastSuccessful()).toBe('backup');
+
+ const second = await harness.runTurn({
+ message: 'and a dessert?',
+ steps: [textStep('Poached pears.')],
+ expect: { deliveredCount: 1, usedProvider: 'primary', providerCalls: 1 },
+ });
+ expect(second.failures).toEqual([]);
+ // Not sticky: the configured default leads the chain again.
+ expect(second.providerCalls[0].provider).toBe('primary');
+ expect(harness.providerRegistry.getDefault().name).toBe('primary');
+ expect(second.guard.providerNotices).toEqual([]);
+ } finally {
+ await harness.dispose();
+ }
+ });
+});
diff --git a/src/eval/scenarios/trace.test.ts b/src/eval/scenarios/trace.test.ts
new file mode 100644
index 00000000..94d55017
--- /dev/null
+++ b/src/eval/scenarios/trace.test.ts
@@ -0,0 +1,71 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { EvalHarness, runScenario } from '../runner.js';
+import { call, textStep, toolStep } from '../fixture.js';
+
+describe('eval · /trace', () => {
+ afterAll(async () => (await import('../env.js')).removeEvalRoot());
+
+ const listTurn = {
+ message: 'list what is in my downloads folder',
+ steps: [
+ toolStep([call('list_dir', { path: 'downloads' }, 'file: report.pdf\nfile: photo.jpg')], undefined, { inputTokens: 700, outputTokens: 20 }),
+ textStep('Two files: report.pdf and photo.jpg.', { inputTokens: 800, outputTokens: 30, cachedInputTokens: 600 }),
+ ],
+ expect: { deliveredCount: 1, toolsUsed: ['list_dir'] },
+ };
+
+ it('/trace after a turn reports provider, tokens and steps — answered by the agent, not the model', async () => {
+ const result = await runScenario({
+ name: 'trace-after-turn',
+ turns: [listTurn, { message: '/trace', expect: { deliveredCount: 1, providerCalls: 0 } }],
+ });
+ expect(result.failures).toEqual([]);
+ const [turn, trace] = result.turns;
+ expect(trace.finalText).toMatch(new RegExp(`^Trace ${turn.traceId}`));
+ expect(trace.finalText).toContain('Provider: scripted / scripted-model');
+ expect(trace.finalText).toMatch(/Tokens: \d+ in \(\d+ cached\) · \d+ out/);
+ expect(trace.finalText).toContain('Steps: 2');
+ expect(trace.finalText).toContain('Verification: none');
+ });
+
+ // KNOWN REGRESSION: main-loop tool calls never reach the turn trace (see
+ // memory.test.ts) — /trace prints "No tool calls in this turn." here.
+ it.fails('/trace lists the main loop\'s tool calls', async () => {
+ const result = await runScenario({ name: 'trace-tool-lines', turns: [listTurn, { message: '/trace' }] });
+ expect(result.turns[1].finalText).toContain('- list_dir path=downloads');
+ });
+
+ it('/trace <id> resolves the id printed on a terminal message, with verification and tool lines', async () => {
+ const harness = await EvalHarness.create({
+ name: 'trace-by-id',
+ programmingMode: 'execute',
+ channel: { streaming: false },
+ turns: [],
+ });
+ try {
+ const work = await harness.runTurn({
+ message: 'fix the failing add() test in src/math.ts',
+ steps: [
+ toolStep([call('edit_file', { path: 'src/math.ts', old_string: 'a - b', new_string: 'a + b' }, 'Successfully edited src/math.ts')]),
+ textStep('Fixed the sign.'),
+ toolStep([call('run_command', { command: 'npm test' }, 'Tests 3 passed (3)')]),
+ textStep('Verified: npm test passes.'),
+ ],
+ expect: { verificationRounds: 1, finalTextIncludes: ['npm test ✓', '/trace '] },
+ });
+ expect(work.failures).toEqual([]);
+ expect(work.finalText).toContain(`/trace ${work.traceId}`);
+ // An unrelated turn in between: /trace <id> must still find the first one.
+ await harness.runTurn({ message: 'thanks', steps: [textStep('Any time.')] });
+
+ const trace = await harness.runTurn({ message: `/trace ${work.traceId}`, expect: { deliveredCount: 1, providerCalls: 0 } });
+ expect(trace.failures).toEqual([]);
+ expect(trace.finalText).toMatch(new RegExp(`^Trace ${work.traceId}`));
+ expect(trace.finalText).toContain('Verification: npm test ✓');
+ expect(trace.finalText).toContain('- run_command command=npm test');
+ expect(trace.finalText).toMatch(/Steps: \d+/);
+ } finally {
+ await harness.dispose();
+ }
+ });
+});