Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
test(eval): chat, execute, memory, provider and /trace scenarios
Salman Qureshid473bce4ddec
diff --git a/src/eval/scenarios/chat.test.ts b/src/eval/scenarios/chat.test.tsnew file mode 100644index 00000000..4146ee76--- /dev/null+++ b/src/eval/scenarios/chat.test.ts@@ -0,0 +1,90 @@+import { afterAll, describe, expect, it } from 'vitest';+import { existsSync, readFileSync } from 'node:fs';+import { join } from 'node:path';+import { runScenario, type TurnResult } from '../runner.js';+import { call, textStep, toolStep } from '../fixture.js';++const NO_GUARD_TEXT = [+ 'getting started on the real build',+ "couldn't get started",+ 'Checking my work before I call it done',+ 'Taking a completely different run',+];++function expectTimings(turn: TurnResult): void {+ expect(turn.timings.steps.length).toBe(turn.providerCalls.length);+ expect(turn.timings.promptBuildMs).toBeGreaterThan(0);+ for (const step of turn.timings.steps) {+ expect(step.startMs).toBeGreaterThan(0);+ expect(step.durationMs).toBeGreaterThanOrEqual(0);+ }+ expect(turn.timings.deliveryMs).toBeGreaterThanOrEqual(0);+ expect(turn.timings.totalMs).toBeGreaterThanOrEqual(turn.timings.promptBuildMs);+}++describe('eval · chat mode never forces guard rounds', () => {+ afterAll(async () => {+ const { removeEvalRoot } = await import('../env.js');+ removeEvalRoot();+ });++ it('"make me a workout plan" → one delivered answer, zero forced rounds', async () => {+ const plan = 'Here is a 3-day plan:\n- Day 1: squats, push-ups\n- Day 2: rest and a 30 minute walk\n- Day 3: deadlifts, rows, planks';+ const result = await runScenario({+ name: 'chat-workout-plan',+ turns: [{+ message: 'make me a workout plan',+ steps: [textStep(plan, { inputTokens: 900, outputTokens: 60 })],+ expect: {+ deliveredCount: 1,+ finalTextIncludes: ['3-day plan'],+ providerCalls: 1,+ forcedActionRounds: 0,+ verificationRounds: 0,+ pause: null,+ noEventIncludes: NO_GUARD_TEXT,+ },+ }],+ });+ expect(result.failures).toEqual([]);+ const [turn] = result.turns;+ expect(turn.guard.forcedToolChoiceCalls).toBe(0);+ expect(turn.toolCalls).toEqual([]);+ expect(turn.tokens).toMatchObject({ input: 900, output: 60 });+ expect(turn.usedProvider).toBe('scripted');+ // user message + assistant answer, no tool trace for a tool-free turn+ expect(turn.sessionEntries.map((m) => `${m.role}:${m.kind}`)).toEqual(['user:message', 'assistant:message']);+ expectTimings(turn);+ });++ it('"write my notes to notes.md" → real create_file runs, answer delivered, no verification nudge', async () => {+ const result = await runScenario({+ name: 'chat-write-notes',+ turns: [{+ message: 'write my notes to notes.md',+ steps: [+ toolStep([call('create_file', { path: 'notes.md', content: '# Notes\n- buy milk\n' })]),+ textStep('Saved your notes to notes.md.'),+ ],+ expect: {+ deliveredCount: 1,+ finalTextIncludes: ['notes.md'],+ providerCalls: 2,+ forcedActionRounds: 0,+ verificationRounds: 0,+ pause: null,+ toolsUsed: ['create_file'],+ noEventIncludes: NO_GUARD_TEXT,+ },+ }],+ });+ expect(result.failures).toEqual([]);+ const [turn] = result.turns;+ expect(turn.toolCalls[0].recorded).toBe(false);+ expect(readFileSync(join(result.workDir, 'notes.md'), 'utf8')).toContain('buy milk');+ expect(existsSync(join(result.workDir, 'notes.md'))).toBe(true);+ // The tool result went back to the model on the second call.+ expect(JSON.stringify(turn.providerCalls[1].prompt)).toContain('tool-result');+ expectTimings(turn);+ });+});diff --git a/src/eval/scenarios/execute.test.ts b/src/eval/scenarios/execute.test.tsnew file mode 100644index 00000000..263385d9--- /dev/null+++ b/src/eval/scenarios/execute.test.ts@@ -0,0 +1,138 @@+import { afterAll, describe, expect, it } from 'vitest';+import { runScenario } from '../runner.js';+import { call, textStep, toolStep, type ScriptedStep } from '../fixture.js';+import { STEPS_PAUSED_BANNER, VERIFICATION_FAILED_BANNER } from '../../core/completion-verdict.js';++const TASK = 'fix the failing add() test in src/math.ts';+const edit = call('edit_file', { path: 'src/math.ts', old_string: 'return a - b;', new_string: 'return a + b;' }, 'Successfully edited src/math.ts (1 replacement)');+const failingTest = call('run_command', { command: 'npm test' }, 'Command exited with code 1\n\nFAIL src/math.test.ts > add > adds two numbers\nAssertionError: expected -1 to be 3');+const passingTest = call('run_command', { command: 'npm test' }, '> vitest run\n\n ✓ src/math.test.ts (3 tests) 4ms\n\n Test Files 1 passed (1)\n Tests 3 passed (3)');++/**+ * KNOWN REGRESSION (found by this harness, fix belongs in core/agent.ts):+ * on the STREAMING path handleMessage stores `result = { text, usage,+ * reasoning }` without `finishReason`, so `classifyTurnEnd` reads+ * `undefined` → 'interrupted' for every streamed turn. The verification gate+ * (`turnEnd() === 'text-stop'`) and step-budget continuation/pause+ * (`'steps-exhausted'`) are therefore dead on every streaming channel (CLI,+ * web, Telegram/Discord/Slack with streaming on, Signal). The scenarios below+ * pin the correct behaviour on the non-streaming path and mark the streaming+ * variants `it.fails` — when agent.ts adds `finishReason` to the streamed+ * result they start passing, `it.fails` turns red, and the marker must go.+ */+const PATHS = [+ { label: 'non-streaming', streaming: false, gateWorks: true },+ { label: 'streaming', streaming: true, gateWorks: false },+] as const;++afterAll(async () => (await import('../env.js')).removeEvalRoot());++describe.each(PATHS)('eval · execute mode completion contract ($label delivery)', ({ label, streaming, gateWorks }) => {+ const itGate = gateWorks ? it : it.fails;++ const verifyFail = () => runScenario({+ name: `execute-verify-fail-${label}`,+ channel: { streaming },+ programmingMode: 'execute',+ turns: [{+ message: TASK,+ steps: [+ toolStep([edit]),+ toolStep([failingTest]),+ textStep('I fixed the sign in add().'),+ // The verification gate forces exactly one evidence round+ // (toolChoice: required); it fails again.+ toolStep([failingTest]),+ textStep('The test still fails.'),+ ],+ expect: {+ pause: 'verification-failed',+ forcedActionRounds: 0,+ toolsUsed: ['edit_file', 'run_command'],+ noEventIncludes: ['Task complete'],+ anyEventIncludes: [VERIFICATION_FAILED_BANNER, 'npm test ✗'],+ },+ }],+ });++ it('edit → FAILING npm test → text: pauses with the verification-failed banner, never "Task complete"', async () => {+ const result = await verifyFail();+ // The script carries a step for the forced evidence round; the+ // streaming path never takes it (see KNOWN REGRESSION above).+ const failures = gateWorks ? result.failures : result.failures.filter((f) => !f.includes('more model calls'));+ expect(failures).toEqual([]);+ const [turn] = result.turns;+ expect(turn.finalText).toContain(VERIFICATION_FAILED_BANNER);+ expect(turn.finalText).toMatch(/\/trace [a-z0-9]+/i);+ expect(turn.events.some((e) => e.kind === 'completion')).toBe(false);+ // A pause is not a completed answer: no assistant message is stored.+ expect(turn.sessionEntries.some((m) => m.role === 'assistant' && m.kind === 'message')).toBe(false);+ expect(turn.timings.steps.length).toBe(turn.providerCalls.length);+ expect(turn.timings.promptBuildMs).toBeGreaterThan(0);+ });++ itGate('the verification gate forces exactly one evidence round (toolChoice: required)', async () => {+ const [turn] = (await verifyFail()).turns;+ expect(turn.guard.verificationRounds).toBe(1);+ expect(turn.guard.forcedToolChoiceCalls).toBe(1);+ expect(turn.providerCalls).toHaveLength(5);+ expect(turn.notices.some((n) => n.includes('Checking my work'))).toBe(true);+ });++ it('edit → passing npm test → completes with a ✓ verification note and a /trace id', async () => {+ const result = await runScenario({+ name: `execute-verify-pass-${label}`,+ channel: { streaming },+ programmingMode: 'execute',+ turns: [{+ message: TASK,+ steps: [toolStep([edit]), toolStep([passingTest]), textStep('Fixed add(): it now returns a + b. All 3 tests pass.')],+ expect: {+ deliveredCount: 1,+ pause: null,+ verificationRounds: 0,+ forcedActionRounds: 0,+ providerCalls: 3,+ finalTextIncludes: ['Fixed add()', 'npm test ✓', '/trace '],+ },+ }],+ });+ expect(result.failures).toEqual([]);+ const [turn] = result.turns;+ expect(turn.traceId).toBeTruthy();+ expect(turn.finalText).toContain(`/trace ${turn.traceId}`);+ const answer = turn.sessionEntries.find((m) => m.role === 'assistant' && m.kind === 'message');+ expect(answer?.metadata).toMatchObject({ verification: 'npm test ✓', steps: 3, provider: 'scripted' });+ expect(turn.events.some((e) => e.kind === 'completion')).toBe(false); // short task: footer, no banner+ expect(turn.timings.steps.map((s) => s.durationMs).every((d) => d >= 0)).toBe(true);+ expect(turn.timings.deliveryMs).toBeGreaterThanOrEqual(0);+ });++ itGate('step budget exhausted → honest pause banner, delivered as a persistent message (not a status-card notice)', async () => {+ let n = 0;+ const endless = (): ScriptedStep => {+ n++;+ return toolStep([call('create_file', { path: `gen/part-${n}.ts`, content: `export const part${n} = ${n};\n` }, `Created gen/part-${n}.ts`)]);+ };+ const result = await runScenario({+ name: `execute-steps-exhausted-${label}`,+ channel: { streaming },+ programmingMode: 'execute',+ turns: [{ message: 'build the generator module', onExhausted: endless, expect: { pause: 'steps', forcedActionRounds: 0, noEventIncludes: ['Task complete'] } }],+ });+ expect(result.failures).toEqual([]);+ const [turn] = result.turns;+ // Bounded automatic continuations happened before the pause.+ expect(turn.guard.stepBudgetContinuations).toBe(6); // MAX_AUTOMATIC_CONTINUATIONS+ expect(turn.delivered.some((t) => t.includes('(no text response)'))).toBe(false);+ const banner = turn.events.find((e) => e.text.includes(STEPS_PAUSED_BANNER));+ expect(banner?.kind).toBe('delivered');+ expect(turn.notices.some((t) => t.includes(STEPS_PAUSED_BANNER))).toBe(false);+ // The task was ended BEFORE the banner was sent, so it is not swallowed by the status card.+ const end = turn.events.findIndex((e) => e.kind === 'task-end');+ expect(end).toBeGreaterThanOrEqual(0);+ expect(turn.events.indexOf(banner!)).toBeGreaterThan(end);+ expect(turn.finalText).toContain(STEPS_PAUSED_BANNER);+ expect(turn.timings.steps.length).toBe(turn.providerCalls.length);+ });+});diff --git a/src/eval/scenarios/memory.test.ts b/src/eval/scenarios/memory.test.tsnew file mode 100644index 00000000..9da2b872--- /dev/null+++ b/src/eval/scenarios/memory.test.ts@@ -0,0 +1,90 @@+import { afterAll, describe, expect, it } from 'vitest';+import { runScenario } from '../runner.js';+import { call, textStep, toolStep } from '../fixture.js';+import { promptText } from '../script-player.js';++const TRACE_HEADER = '[Tool activity in my previous turn]';++/**+ * KNOWN REGRESSION (found by this harness, fix belongs in core/agent.ts):+ * the per-turn tool trace (`turnToolTrace`, written at turn end as the+ * `[Tool activity in my previous turn]` session entry) is only fed by+ * `noteToolSteps` in the guard / step-budget / verification continuation+ * rounds. The MAIN loop's `onStepFinish` (streamText and generateText+ * paths) never calls it, so an ordinary tool-using turn leaves no trace and+ * the next turn starts blind. The first scenario is `it.fails` until that+ * call is added; the read side is pinned by the passing scenarios.+ */+describe('eval · cross-turn memory carries the tool trace', () => {+ afterAll(async () => (await import('../env.js')).removeEvalRoot());++ it.fails('main-loop tool calls are written to the trace the NEXT turn sends to the model', async () => {+ const result = await runScenario({+ name: 'memory-main-loop-trace',+ turns: [+ {+ message: 'save a packing list for my trip',+ steps: [+ toolStep([call('create_file', { path: 'trip/packing.md', content: '- passport\n- charger\n' }, 'Created trip/packing.md')]),+ textStep('Saved your packing list to trip/packing.md.'),+ ],+ expect: { deliveredCount: 1, toolsUsed: ['create_file'] },+ },+ { message: 'which file did you put it in?', steps: [textStep('It is in trip/packing.md.')], expect: { deliveredCount: 1, providerCalls: 1 } },+ ],+ });+ expect(result.failures).toEqual([]);+ const input = promptText(result.turns[1].providerCalls[0].prompt);+ expect(input).toContain(TRACE_HEADER);+ expect(input).toContain('create_file path=trip/packing.md');+ });++ it('a trace written during a turn (verification round) reaches the next turn\'s model input', async () => {+ const result = await runScenario({+ name: 'memory-continuation-trace',+ programmingMode: 'execute',+ channel: { streaming: false },+ turns: [+ {+ message: 'fix the failing add() test in src/math.ts',+ steps: [+ toolStep([call('edit_file', { path: 'src/math.ts', old_string: 'a - b', new_string: 'a + b' }, 'Successfully edited src/math.ts')]),+ textStep('Fixed the sign.'),+ // forced verification round (its tool steps ARE traced)+ toolStep([call('run_command', { command: 'npm test' }, 'Tests 3 passed (3)')]),+ textStep('Verified: npm test passes.'),+ ],+ expect: { verificationRounds: 1, pause: null },+ },+ { message: 'what did you run to check it?', steps: [textStep('npm test.')], expect: { deliveredCount: 1, providerCalls: 1 } },+ ],+ });+ expect(result.failures).toEqual([]);+ const [first, second] = result.turns;+ const trace = first.sessionEntries.find((m) => m.kind === 'tool-call');+ expect(trace?.content.startsWith(TRACE_HEADER)).toBe(true);+ expect(trace?.content).toContain('run_command command=npm test');++ const input = promptText(second.providerCalls[0].prompt);+ expect(input).toContain(TRACE_HEADER);+ expect(input).toContain('run_command command=npm test');+ // Trace, then the previous answer, then the new question.+ expect(input.indexOf(TRACE_HEADER)).toBeLessThan(input.indexOf('Verified: npm test passes.'));+ expect(input.indexOf('Verified: npm test passes.')).toBeLessThan(input.indexOf('what did you run to check it?'));+ });++ it('a seeded session with a prior tool trace reaches the model input', async () => {+ const result = await runScenario({+ name: 'memory-seeded-trace',+ history: [+ { role: 'user', content: 'check the build config' },+ { role: 'assistant', kind: 'tool-call', content: `${TRACE_HEADER}\n- read_file path=tsup.config.ts → ok (24 lines)` },+ { role: 'assistant', content: 'The build targets node20 with ESM output.' },+ ],+ turns: [{ message: 'and the entry point?', steps: [textStep('src/index.ts.')], expect: { deliveredCount: 1 } }],+ });+ expect(result.failures).toEqual([]);+ const input = promptText(result.turns[0].providerCalls[0].prompt);+ expect(input).toContain(`${TRACE_HEADER}\n- read_file path=tsup.config.ts`);+ });+});diff --git a/src/eval/scenarios/provider.test.ts b/src/eval/scenarios/provider.test.tsnew file mode 100644index 00000000..2183d41e--- /dev/null+++ b/src/eval/scenarios/provider.test.ts@@ -0,0 +1,41 @@+import { afterAll, describe, expect, it } from 'vitest';+import { EvalHarness } from '../runner.js';+import { textStep } from '../fixture.js';++describe('eval · provider fallback', () => {+ afterAll(async () => (await import('../env.js')).removeEvalRoot());++ it('first provider throws, second answers; the next turn starts on the configured default again', async () => {+ const harness = await EvalHarness.create({+ name: 'provider-fallback',+ providers: [{ name: 'primary', failFirst: 1, failWith: 'primary: 503 Service Unavailable' }, { name: 'backup' }],+ turns: [],+ });+ try {+ const first = await harness.runTurn({+ message: 'what should I cook tonight',+ steps: [textStep('Try a mushroom risotto.')],+ expect: { deliveredCount: 1, usedProvider: 'backup', finalTextIncludes: ['risotto'] },+ });+ expect(first.failures).toEqual([]);+ expect(first.providerCalls.map((c) => [c.provider, c.error ? 'error' : 'ok'])).toEqual([['primary', 'error'], ['backup', 'ok']]);+ expect(first.guard.providerNotices.some((n) => n.includes('switching to `backup`'))).toBe(true);+ expect(first.guard.providerNotices.some((n) => n.includes('served by `backup`'))).toBe(true);+ expect(first.timings.steps[0].error).toContain('503');+ expect(harness.providerRegistry.getLastSuccessful()).toBe('backup');++ const second = await harness.runTurn({+ message: 'and a dessert?',+ steps: [textStep('Poached pears.')],+ expect: { deliveredCount: 1, usedProvider: 'primary', providerCalls: 1 },+ });+ expect(second.failures).toEqual([]);+ // Not sticky: the configured default leads the chain again.+ expect(second.providerCalls[0].provider).toBe('primary');+ expect(harness.providerRegistry.getDefault().name).toBe('primary');+ expect(second.guard.providerNotices).toEqual([]);+ } finally {+ await harness.dispose();+ }+ });+});diff --git a/src/eval/scenarios/trace.test.ts b/src/eval/scenarios/trace.test.tsnew file mode 100644index 00000000..94d55017--- /dev/null+++ b/src/eval/scenarios/trace.test.ts@@ -0,0 +1,71 @@+import { afterAll, describe, expect, it } from 'vitest';+import { EvalHarness, runScenario } from '../runner.js';+import { call, textStep, toolStep } from '../fixture.js';++describe('eval · /trace', () => {+ afterAll(async () => (await import('../env.js')).removeEvalRoot());++ const listTurn = {+ message: 'list what is in my downloads folder',+ steps: [+ toolStep([call('list_dir', { path: 'downloads' }, 'file: report.pdf\nfile: photo.jpg')], undefined, { inputTokens: 700, outputTokens: 20 }),+ textStep('Two files: report.pdf and photo.jpg.', { inputTokens: 800, outputTokens: 30, cachedInputTokens: 600 }),+ ],+ expect: { deliveredCount: 1, toolsUsed: ['list_dir'] },+ };++ it('/trace after a turn reports provider, tokens and steps — answered by the agent, not the model', async () => {+ const result = await runScenario({+ name: 'trace-after-turn',+ turns: [listTurn, { message: '/trace', expect: { deliveredCount: 1, providerCalls: 0 } }],+ });+ expect(result.failures).toEqual([]);+ const [turn, trace] = result.turns;+ expect(trace.finalText).toMatch(new RegExp(`^Trace ${turn.traceId}`));+ expect(trace.finalText).toContain('Provider: scripted / scripted-model');+ expect(trace.finalText).toMatch(/Tokens: \d+ in \(\d+ cached\) · \d+ out/);+ expect(trace.finalText).toContain('Steps: 2');+ expect(trace.finalText).toContain('Verification: none');+ });++ // KNOWN REGRESSION: main-loop tool calls never reach the turn trace (see+ // memory.test.ts) — /trace prints "No tool calls in this turn." here.+ it.fails('/trace lists the main loop\'s tool calls', async () => {+ const result = await runScenario({ name: 'trace-tool-lines', turns: [listTurn, { message: '/trace' }] });+ expect(result.turns[1].finalText).toContain('- list_dir path=downloads');+ });++ it('/trace <id> resolves the id printed on a terminal message, with verification and tool lines', async () => {+ const harness = await EvalHarness.create({+ name: 'trace-by-id',+ programmingMode: 'execute',+ channel: { streaming: false },+ turns: [],+ });+ try {+ const work = await harness.runTurn({+ message: 'fix the failing add() test in src/math.ts',+ steps: [+ toolStep([call('edit_file', { path: 'src/math.ts', old_string: 'a - b', new_string: 'a + b' }, 'Successfully edited src/math.ts')]),+ textStep('Fixed the sign.'),+ toolStep([call('run_command', { command: 'npm test' }, 'Tests 3 passed (3)')]),+ textStep('Verified: npm test passes.'),+ ],+ expect: { verificationRounds: 1, finalTextIncludes: ['npm test ✓', '/trace '] },+ });+ expect(work.failures).toEqual([]);+ expect(work.finalText).toContain(`/trace ${work.traceId}`);+ // An unrelated turn in between: /trace <id> must still find the first one.+ await harness.runTurn({ message: 'thanks', steps: [textStep('Any time.')] });++ const trace = await harness.runTurn({ message: `/trace ${work.traceId}`, expect: { deliveredCount: 1, providerCalls: 0 } });+ expect(trace.failures).toEqual([]);+ expect(trace.finalText).toMatch(new RegExp(`^Trace ${work.traceId}`));+ expect(trace.finalText).toContain('Verification: npm test ✓');+ expect(trace.finalText).toContain('- run_command command=npm test');+ expect(trace.finalText).toMatch(/Steps: \d+/);+ } finally {+ await harness.dispose();+ }+ });+});