Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
test(eval): cover denied writes and interrupted real work
Seth Wilson70e0898b25f9
diff --git a/src/eval/README.md b/src/eval/README.mdnew file mode 100644index 00000000..6244f2ae--- /dev/null+++ b/src/eval/README.md@@ -0,0 +1,38 @@+# Agent replay evaluations++Run `npm run eval -- --maxWorkers=2`. No provider credentials or network calls+are needed. Each fixture gets a scratch Mercury home and working directory.++The harness runs the production `Agent`, AI SDK tool loop, permissions, work+ledger, session repository, and memory stores. The provider replays scripted+steps and the channel records delivery. A tool call with a `result` uses a+recorded output; omit `result` to execute the actual capability in the scratch+directory. Check `toolCalls[].recorded` when a scenario relies on real effects.++`scenarios/reliability.test.ts` exercises both streaming and non-streaming+delivery with real filesystem tools:++| Scenario | Evidence checked |+| --- | --- |+| Write denied twice, then explicitly approved | Original bytes survive both denials; real approval prompts recur; denial reaches the next model call and stored tool trace; a fresh approval permits the write. |+| Completed inbound message redelivered | No new provider/tool call or session entry; later owner edits survive; ledger still has one attempt. |+| Provider fails after a write | Exactly one write; backup provider is never called; progress is reported as pending, with a durable continuation rather than a completed answer. |+| Duplicate while running, then owner stops | Duplicate is not queued; the real abort signal releases the pending provider; completed file survives; reloaded work remains cancelled and is not recovered or reexecuted on redelivery. |++Fixtures can opt out of auto-approval with `autoApprove: false`. Tests install+an explicit `permissions.onAsk` response so the actual permission manager+decides whether a write runs. `ScenarioTurn.messageId` can be reused to model+transport redelivery. Provider `failOnCalls` and `waitForAbortOnCalls` use+one-based call numbers across the provider's lifetime, including failed calls;+neither consumes a scripted output step. Abort waits must be released through+the real agent stop path, including test cleanup.++These are deterministic integration regressions, not measurements of a live+model's honesty or tool-selection quality. The denial case checks the evidence+and delivery pipeline; its final model wording is scripted. Fake timers keep+heartbeats and retry timers from firing during a turn. In particular, the+post-write provider failure test verifies suppression of immediate fallback+and preservation of the existing *ordinary chat* continuation policy; it does+not establish exactly-once execution across later retries, external services,+or process crashes. Scheduled-task recovery belongs in scheduler integration+tests, which can exercise its durable state without a live model.diff --git a/src/eval/fixture.ts b/src/eval/fixture.tsindex 0b29986f..789d4edc 100644--- a/src/eval/fixture.ts+++ b/src/eval/fixture.ts@@ -23,7 +23,7 @@ export interface ScriptedToolCall {/*** Recorded tool output. When present the real tool is NOT executed and* this is returned verbatim; when absent the real tool runs in the- * scenario's scratch cwd (auto-approved).+ * scenario's scratch cwd (auto-approved by default).*/result?: unknown;}@@ -38,6 +38,10 @@ export interface ScriptedProviderSpec {/** The first N model calls to this provider throw `failWith`. */failFirst?: number;failWith?: string;+ /** One-based model calls that fail, e.g. [2] interrupts after a real tool. */+ failOnCalls?: number[];+ /** One-based calls held until the real agent's abort signal fires. */+ waitForAbortOnCalls?: number[];}export interface FixtureHistoryEntry {@@ -70,6 +74,8 @@ export interface TurnExpectations {export interface ScenarioTurn {message: string;+ /** Reuse an inbound transport ID to exercise the production work ledger. */+ messageId?: string;/** Scripted model output, consumed one step per model call. */steps?: ScriptedStep[];/** Supplies a step when the script runs out (e.g. endless tool loops). */@@ -82,6 +88,8 @@ export interface ScenarioTurn {export interface ScenarioFixture {name: string;+ /** Default true for existing fixtures; false exercises real approval gates. */+ autoApprove?: boolean;channel?: {/** Default 'telegram' — exercises the messaging TaskSurface. */type?: ChannelType;diff --git a/src/eval/runner.ts b/src/eval/runner.tsindex 2ae41e16..e5305b92 100644--- a/src/eval/runner.ts+++ b/src/eval/runner.ts@@ -267,7 +267,7 @@ export class EvalHarness {capabilities.registerAll();capabilities.setCwd(workDir);capabilities.permissions.addTempScope(workDir, true, true);- capabilities.permissions.setAutoApproveAll(true);+ capabilities.permissions.setAutoApproveAll(fixture.autoApprove ?? true);capabilities.setChatCommandContext(chatCommandContext(config, tokenBudget, capabilities));const type = fixture.channel?.type ?? 'telegram';@@ -322,7 +322,7 @@ export class EvalHarness {const eventStart = this.channel.events.length;const sessionStart = this.sessionMessages().length;const msg: ChannelMessage = {- id: `eval-${this.fixture.name}-${++this.seq}`,+ id: turn.messageId ?? `eval-${this.fixture.name}-${++this.seq}`,channelId: this.channelId,channelType: this.channelType,senderId: 'eval-user',diff --git a/src/eval/scenarios/reliability.test.ts b/src/eval/scenarios/reliability.test.tsnew file mode 100644index 00000000..1c7ef358--- /dev/null+++ b/src/eval/scenarios/reliability.test.ts@@ -0,0 +1,170 @@+import { afterAll, describe, expect, it } from 'vitest';+// Bootstrap the isolated Mercury home before importing production modules.+import { EvalHarness } from '../runner.js';+import { existsSync, readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { performance } from 'node:perf_hooks';+import { WorkLedger } from '../../core/work-ledger.js';+import { call, textStep, toolStep } from '../fixture.js';++afterAll(async () => (await import('../env.js')).removeEvalRoot());++/** Wait for an observable model boundary, never a guessed delay or fake time. */+async function waitForModelCalls(harness: EvalHarness, count: number): Promise<void> {+ const deadline = performance.now() + 5_000;+ while (harness.player.calls.length < count) {+ if (performance.now() > deadline) throw new Error(`Expected ${count} model calls, saw ${harness.player.calls.length}`);+ await new Promise<void>((resolve) => setImmediate(resolve));+ }+}++describe.each([false, true])('eval · real-work reliability (streaming=%s)', (streaming) => {+ it('preserves a denied file, sends the denial back to the model, and requires a fresh approval', async () => {+ const harness = await EvalHarness.create({+ name: `denied-write-${streaming}`, channel: { streaming }, autoApprove: false, turns: [],+ });+ const target = join(harness.workDir, 'note.txt');+ writeFileSync(target, 'original');+ const approvals: string[] = [];+ harness.capabilities.permissions.onAsk(async (prompt) => { approvals.push(prompt); return 'no'; });+ try {+ const denied = await harness.runTurn({+ message: 'Replace the contents of note.txt with revised.',+ steps: [+ toolStep([call('write_file', { path: 'note.txt', content: 'revised' })]),+ // A model repeating the same action must not turn a denial into approval.+ toolStep([call('write_file', { path: 'note.txt', content: 'revised' })]),+ textStep('The write was denied. note.txt is unchanged.'),+ ],+ expect: { providerCalls: 3, finalTextIncludes: ['denied', 'unchanged'], noEventIncludes: ['Task complete'] },+ });+ expect(denied.failures).toEqual([]);+ expect(readFileSync(target, 'utf8')).toBe('original');+ expect(approvals).toHaveLength(2);+ expect(approvals.every((prompt) => prompt.includes(target))).toBe(true);+ expect(denied.toolCalls).toHaveLength(2);+ for (const invocation of denied.toolCalls) {+ expect(invocation.recorded).toBe(false);+ expect(invocation.output).toContain('Permission denied');+ }+ expect(harness.promptOf(1)).toContain('Permission denied');+ expect(harness.promptOf(2)).toContain('Permission denied');+ expect(denied.sessionEntries.some((entry) => entry.kind === 'tool-call' && entry.content.includes('Permission denied'))).toBe(true);++ harness.capabilities.permissions.onAsk(async (prompt) => { approvals.push(prompt); return 'yes'; });+ const approved = await harness.runTurn({+ message: 'Try writing revised to note.txt again; I will approve this time.',+ steps: [toolStep([call('write_file', { path: 'note.txt', content: 'revised' })]), textStep('Updated note.txt.')],+ expect: { providerCalls: 2, deliveredCount: 1 },+ });+ expect(approved.failures).toEqual([]);+ expect(approvals).toHaveLength(3);+ expect(readFileSync(target, 'utf8')).toBe('revised');+ expect(approved.toolCalls[0]).toMatchObject({ recorded: false, tool: 'write_file' });+ } finally {+ await harness.dispose();+ }+ });++ it('does not rerun tools or overwrite later edits when a completed transport message is redelivered', async () => {+ const harness = await EvalHarness.create({ name: `duplicate-completed-${streaming}`, channel: { streaming }, turns: [] });+ const target = join(harness.workDir, 'receipt.txt');+ const messageId = 'same-transport-message';+ try {+ const first = await harness.runTurn({+ messageId, message: 'Write receipt.txt with the text accepted.',+ steps: [toolStep([call('create_file', { path: 'receipt.txt', content: 'accepted' })]), textStep('Created receipt.txt.')],+ expect: { providerCalls: 2, deliveredCount: 1 },+ });+ expect(first.failures).toEqual([]);+ expect(first.toolCalls).toHaveLength(1);+ expect(first.toolCalls[0].recorded).toBe(false);+ expect(readFileSync(target, 'utf8')).toBe('accepted');+ writeFileSync(target, 'subsequent owner edit');++ const duplicate = await harness.runTurn({+ messageId, message: 'Write receipt.txt with the text accepted.',+ expect: { providerCalls: 0, toolsUsed: [], finalTextIncludes: ['already completed'] },+ });+ expect(duplicate.failures).toEqual([]);+ expect(duplicate.sessionEntries).toEqual([]);+ expect(readFileSync(target, 'utf8')).toBe('subsequent owner edit');+ expect(new WorkLedger().get(`channel:telegram:${messageId}`)).toMatchObject({ status: 'completed', attempts: 1, delivered: true });+ } finally {+ await harness.dispose();+ }+ });++ it('suppresses provider fallback after a real side effect and preserves the pending continuation honestly', async () => {+ const harness = await EvalHarness.create({+ name: `interrupted-write-${streaming}`, channel: { streaming },+ providers: [{ name: 'primary', failOnCalls: [2], failWith: 'connection lost after write' }, { name: 'backup' }], turns: [],+ });+ const messageId = 'interrupted-after-write';+ try {+ const result = await harness.runTurn({+ messageId, message: 'Create receipt.txt with accepted and report the result.',+ steps: [toolStep([call('create_file', { path: 'receipt.txt', content: 'accepted' })])],+ expect: { providerCalls: 2, noEventIncludes: ['Task complete', 'switching to `backup`'] },+ });+ expect(result.failures).toEqual([]);+ expect(result.toolCalls).toHaveLength(1);+ expect(result.toolCalls[0].recorded).toBe(false);+ expect(readFileSync(join(harness.workDir, 'receipt.txt'), 'utf8')).toBe('accepted');+ expect(harness.providers[1].callCount).toBe(0);+ expect(result.providerCalls[1].error).toBe('connection lost after write');+ expect(result.events.some((event) => event.text.includes('checkpointed'))).toBe(true);+ expect(result.sessionEntries.some((entry) => entry.role === 'assistant' && entry.kind === 'message')).toBe(false);+ // Ordinary chat has a durable inspected-continuation policy. This test+ // does not advance its timer or claim generic work is never retried.+ const saved = new WorkLedger().get(`channel:telegram:${messageId}`);+ expect(saved).toMatchObject({ status: 'queued', attempts: 1, message: { metadata: { workContinuation: true } } });+ expect(saved?.nextAttemptAt).toBeGreaterThan(Date.now());+ } finally {+ await harness.dispose();+ }+ });++ it('deduplicates running work and keeps an owner-stopped turn cancelled after reload', async () => {+ const harness = await EvalHarness.create({+ name: `stop-in-flight-${streaming}`, channel: { streaming },+ providers: [{ name: 'primary', waitForAbortOnCalls: [2] }, { name: 'backup' }], turns: [],+ });+ const messageId = 'stop-after-write';+ const message = 'Create receipt.txt with accepted, then report the result.';+ let pending: ReturnType<EvalHarness['runTurn']> | undefined;+ try {+ pending = harness.runTurn({+ messageId, message,+ steps: [toolStep([call('create_file', { path: 'receipt.txt', content: 'accepted' })])],+ expect: { providerCalls: 2, finalTextIncludes: ['Task stopped'], noEventIncludes: ['Task complete'] },+ });+ await waitForModelCalls(harness, 2);+ expect(existsSync(join(harness.workDir, 'receipt.txt'))).toBe(true);+ harness.channel.inject({+ id: messageId, content: message, channelId: harness.channelId, channelType: harness.channelType,+ senderId: 'eval-user', timestamp: Date.now(),+ });+ expect(harness.agent.queuedCount).toBe(0);+ expect(harness.agent.stopWorkFor(harness.channelId, 'Owner stopped this task.')).toBe(true);+ const result = await pending;+ pending = undefined;+ expect(result.failures).toEqual([]);+ expect(result.toolCalls).toHaveLength(1);+ expect(result.toolCalls[0].recorded).toBe(false);+ expect(result.events.some((event) => event.text.includes('already running'))).toBe(true);+ expect(harness.providers[1].callCount).toBe(0);+ expect(readFileSync(join(harness.workDir, 'receipt.txt'), 'utf8')).toBe('accepted');+ const reloaded = new WorkLedger();+ expect(reloaded.get(`channel:telegram:${messageId}`)).toMatchObject({ status: 'cancelled', attempts: 1 });+ expect(reloaded.recoverInterrupted()).toEqual([]);++ const redelivery = await harness.runTurn({ messageId, message, expect: { providerCalls: 0, toolsUsed: [], finalTextIncludes: ['cancelled earlier'] } });+ expect(redelivery.failures).toEqual([]);+ } finally {+ harness.agent.stopWorkFor(harness.channelId);+ if (pending) await pending;+ await harness.dispose();+ }+ });+});diff --git a/src/eval/scripted-provider.ts b/src/eval/scripted-provider.tsindex da3a7120..25a5aac4 100644--- a/src/eval/scripted-provider.ts+++ b/src/eval/scripted-provider.ts@@ -50,6 +50,8 @@ export class ScriptedProvider extends BaseProvider {callCount = 0;private readonly failFirst: number;private readonly failWith: string;+ private readonly failOnCalls: Set<number>;+ private readonly waitForAbortOnCalls: Set<number>;private readonly instance: MockLanguageModelV3;constructor(spec: ScriptedProviderSpec, private readonly player: ScriptPlayer) {@@ -59,6 +61,8 @@ export class ScriptedProvider extends BaseProvider {this.model = model;this.failFirst = spec.failFirst ?? 0;this.failWith = spec.failWith ?? `${spec.name}: scripted outage (503 Service Unavailable)`;+ this.failOnCalls = new Set(spec.failOnCalls);+ this.waitForAbortOnCalls = new Set(spec.waitForAbortOnCalls);this.instance = new MockLanguageModelV3({provider: `eval.${spec.name}`,modelId: model,@@ -67,19 +71,34 @@ export class ScriptedProvider extends BaseProvider {});}- private maybeFail(options: LanguageModelV3CallOptions, mode: 'stream' | 'generate'): ReturnType<ScriptPlayer['begin']> {+ private async beginCall(options: LanguageModelV3CallOptions, mode: 'stream' | 'generate'): Promise<ReturnType<ScriptPlayer['begin']>> {this.callCount++;const record = this.player.begin(this.name, this.model, mode, options);- if (this.callCount <= this.failFirst) {+ if (this.callCount <= this.failFirst || this.failOnCalls.has(this.callCount)) {record.error = this.failWith;this.player.end(record);throw new Error(this.failWith);}+ if (this.waitForAbortOnCalls.has(this.callCount)) {+ try {+ const signal = options.abortSignal;+ if (!signal) throw new Error('Scripted abort wait requires an abort signal');+ await new Promise<never>((_resolve, reject) => {+ const abort = () => reject(signal.reason ?? new Error('Aborted'));+ if (signal.aborted) abort();+ else signal.addEventListener('abort', abort, { once: true });+ });+ } catch (error) {+ record.error = error instanceof Error ? error.message : String(error);+ this.player.end(record);+ throw error;+ }+ }return record;}private async doStream(options: LanguageModelV3CallOptions) {- const record = this.maybeFail(options, 'stream');+ const record = await this.beginCall(options, 'stream');const step = this.player.nextStep(record);const parts: LanguageModelV3StreamPart[] = [{ type: 'stream-start', warnings: [] }];if (step.text) {@@ -103,7 +122,7 @@ export class ScriptedProvider extends BaseProvider {}private async doGenerate(options: LanguageModelV3CallOptions) {- const record = this.maybeFail(options, 'generate');+ const record = await this.beginCall(options, 'generate');const step = this.player.nextStep(record);this.player.end(record);return {