Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

test(eval): cover denied writes and interrupted real work

Seth Wilson70e0898b25f9
diff --git a/src/eval/README.md b/src/eval/README.md
new file mode 100644
index 00000000..6244f2ae
--- /dev/null
+++ b/src/eval/README.md
@@ -0,0 +1,38 @@
+# Agent replay evaluations
+
+Run `npm run eval -- --maxWorkers=2`. No provider credentials or network calls
+are needed. Each fixture gets a scratch Mercury home and working directory.
+
+The harness runs the production `Agent`, AI SDK tool loop, permissions, work
+ledger, session repository, and memory stores. The provider replays scripted
+steps and the channel records delivery. A tool call with a `result` uses a
+recorded output; omit `result` to execute the actual capability in the scratch
+directory. Check `toolCalls[].recorded` when a scenario relies on real effects.
+
+`scenarios/reliability.test.ts` exercises both streaming and non-streaming
+delivery with real filesystem tools:
+
+| Scenario | Evidence checked |
+| --- | --- |
+| Write denied twice, then explicitly approved | Original bytes survive both denials; real approval prompts recur; denial reaches the next model call and stored tool trace; a fresh approval permits the write. |
+| Completed inbound message redelivered | No new provider/tool call or session entry; later owner edits survive; ledger still has one attempt. |
+| Provider fails after a write | Exactly one write; backup provider is never called; progress is reported as pending, with a durable continuation rather than a completed answer. |
+| Duplicate while running, then owner stops | Duplicate is not queued; the real abort signal releases the pending provider; completed file survives; reloaded work remains cancelled and is not recovered or reexecuted on redelivery. |
+
+Fixtures can opt out of auto-approval with `autoApprove: false`. Tests install
+an explicit `permissions.onAsk` response so the actual permission manager
+decides whether a write runs. `ScenarioTurn.messageId` can be reused to model
+transport redelivery. Provider `failOnCalls` and `waitForAbortOnCalls` use
+one-based call numbers across the provider's lifetime, including failed calls;
+neither consumes a scripted output step. Abort waits must be released through
+the real agent stop path, including test cleanup.
+
+These are deterministic integration regressions, not measurements of a live
+model's honesty or tool-selection quality. The denial case checks the evidence
+and delivery pipeline; its final model wording is scripted. Fake timers keep
+heartbeats and retry timers from firing during a turn. In particular, the
+post-write provider failure test verifies suppression of immediate fallback
+and preservation of the existing *ordinary chat* continuation policy; it does
+not establish exactly-once execution across later retries, external services,
+or process crashes. Scheduled-task recovery belongs in scheduler integration
+tests, which can exercise its durable state without a live model.
diff --git a/src/eval/fixture.ts b/src/eval/fixture.ts
index 0b29986f..789d4edc 100644
--- a/src/eval/fixture.ts
+++ b/src/eval/fixture.ts
@@ -23,7 +23,7 @@ export interface ScriptedToolCall {
/**
* Recorded tool output. When present the real tool is NOT executed and
* this is returned verbatim; when absent the real tool runs in the
- * scenario's scratch cwd (auto-approved).
+ * scenario's scratch cwd (auto-approved by default).
*/
result?: unknown;
}
@@ -38,6 +38,10 @@ export interface ScriptedProviderSpec {
/** The first N model calls to this provider throw `failWith`. */
failFirst?: number;
failWith?: string;
+ /** One-based model calls that fail, e.g. [2] interrupts after a real tool. */
+ failOnCalls?: number[];
+ /** One-based calls held until the real agent's abort signal fires. */
+ waitForAbortOnCalls?: number[];
}
export interface FixtureHistoryEntry {
@@ -70,6 +74,8 @@ export interface TurnExpectations {
export interface ScenarioTurn {
message: string;
+ /** Reuse an inbound transport ID to exercise the production work ledger. */
+ messageId?: string;
/** Scripted model output, consumed one step per model call. */
steps?: ScriptedStep[];
/** Supplies a step when the script runs out (e.g. endless tool loops). */
@@ -82,6 +88,8 @@ export interface ScenarioTurn {
export interface ScenarioFixture {
name: string;
+ /** Default true for existing fixtures; false exercises real approval gates. */
+ autoApprove?: boolean;
channel?: {
/** Default 'telegram' — exercises the messaging TaskSurface. */
type?: ChannelType;
diff --git a/src/eval/runner.ts b/src/eval/runner.ts
index 2ae41e16..e5305b92 100644
--- a/src/eval/runner.ts
+++ b/src/eval/runner.ts
@@ -267,7 +267,7 @@ export class EvalHarness {
capabilities.registerAll();
capabilities.setCwd(workDir);
capabilities.permissions.addTempScope(workDir, true, true);
- capabilities.permissions.setAutoApproveAll(true);
+ capabilities.permissions.setAutoApproveAll(fixture.autoApprove ?? true);
capabilities.setChatCommandContext(chatCommandContext(config, tokenBudget, capabilities));
const type = fixture.channel?.type ?? 'telegram';
@@ -322,7 +322,7 @@ export class EvalHarness {
const eventStart = this.channel.events.length;
const sessionStart = this.sessionMessages().length;
const msg: ChannelMessage = {
- id: `eval-${this.fixture.name}-${++this.seq}`,
+ id: turn.messageId ?? `eval-${this.fixture.name}-${++this.seq}`,
channelId: this.channelId,
channelType: this.channelType,
senderId: 'eval-user',
diff --git a/src/eval/scenarios/reliability.test.ts b/src/eval/scenarios/reliability.test.ts
new file mode 100644
index 00000000..1c7ef358
--- /dev/null
+++ b/src/eval/scenarios/reliability.test.ts
@@ -0,0 +1,170 @@
+import { afterAll, describe, expect, it } from 'vitest';
+// Bootstrap the isolated Mercury home before importing production modules.
+import { EvalHarness } from '../runner.js';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { WorkLedger } from '../../core/work-ledger.js';
+import { call, textStep, toolStep } from '../fixture.js';
+
+afterAll(async () => (await import('../env.js')).removeEvalRoot());
+
+/** Wait for an observable model boundary, never a guessed delay or fake time. */
+async function waitForModelCalls(harness: EvalHarness, count: number): Promise<void> {
+ const deadline = performance.now() + 5_000;
+ while (harness.player.calls.length < count) {
+ if (performance.now() > deadline) throw new Error(`Expected ${count} model calls, saw ${harness.player.calls.length}`);
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ }
+}
+
+describe.each([false, true])('eval · real-work reliability (streaming=%s)', (streaming) => {
+ it('preserves a denied file, sends the denial back to the model, and requires a fresh approval', async () => {
+ const harness = await EvalHarness.create({
+ name: `denied-write-${streaming}`, channel: { streaming }, autoApprove: false, turns: [],
+ });
+ const target = join(harness.workDir, 'note.txt');
+ writeFileSync(target, 'original');
+ const approvals: string[] = [];
+ harness.capabilities.permissions.onAsk(async (prompt) => { approvals.push(prompt); return 'no'; });
+ try {
+ const denied = await harness.runTurn({
+ message: 'Replace the contents of note.txt with revised.',
+ steps: [
+ toolStep([call('write_file', { path: 'note.txt', content: 'revised' })]),
+ // A model repeating the same action must not turn a denial into approval.
+ toolStep([call('write_file', { path: 'note.txt', content: 'revised' })]),
+ textStep('The write was denied. note.txt is unchanged.'),
+ ],
+ expect: { providerCalls: 3, finalTextIncludes: ['denied', 'unchanged'], noEventIncludes: ['Task complete'] },
+ });
+ expect(denied.failures).toEqual([]);
+ expect(readFileSync(target, 'utf8')).toBe('original');
+ expect(approvals).toHaveLength(2);
+ expect(approvals.every((prompt) => prompt.includes(target))).toBe(true);
+ expect(denied.toolCalls).toHaveLength(2);
+ for (const invocation of denied.toolCalls) {
+ expect(invocation.recorded).toBe(false);
+ expect(invocation.output).toContain('Permission denied');
+ }
+ expect(harness.promptOf(1)).toContain('Permission denied');
+ expect(harness.promptOf(2)).toContain('Permission denied');
+ expect(denied.sessionEntries.some((entry) => entry.kind === 'tool-call' && entry.content.includes('Permission denied'))).toBe(true);
+
+ harness.capabilities.permissions.onAsk(async (prompt) => { approvals.push(prompt); return 'yes'; });
+ const approved = await harness.runTurn({
+ message: 'Try writing revised to note.txt again; I will approve this time.',
+ steps: [toolStep([call('write_file', { path: 'note.txt', content: 'revised' })]), textStep('Updated note.txt.')],
+ expect: { providerCalls: 2, deliveredCount: 1 },
+ });
+ expect(approved.failures).toEqual([]);
+ expect(approvals).toHaveLength(3);
+ expect(readFileSync(target, 'utf8')).toBe('revised');
+ expect(approved.toolCalls[0]).toMatchObject({ recorded: false, tool: 'write_file' });
+ } finally {
+ await harness.dispose();
+ }
+ });
+
+ it('does not rerun tools or overwrite later edits when a completed transport message is redelivered', async () => {
+ const harness = await EvalHarness.create({ name: `duplicate-completed-${streaming}`, channel: { streaming }, turns: [] });
+ const target = join(harness.workDir, 'receipt.txt');
+ const messageId = 'same-transport-message';
+ try {
+ const first = await harness.runTurn({
+ messageId, message: 'Write receipt.txt with the text accepted.',
+ steps: [toolStep([call('create_file', { path: 'receipt.txt', content: 'accepted' })]), textStep('Created receipt.txt.')],
+ expect: { providerCalls: 2, deliveredCount: 1 },
+ });
+ expect(first.failures).toEqual([]);
+ expect(first.toolCalls).toHaveLength(1);
+ expect(first.toolCalls[0].recorded).toBe(false);
+ expect(readFileSync(target, 'utf8')).toBe('accepted');
+ writeFileSync(target, 'subsequent owner edit');
+
+ const duplicate = await harness.runTurn({
+ messageId, message: 'Write receipt.txt with the text accepted.',
+ expect: { providerCalls: 0, toolsUsed: [], finalTextIncludes: ['already completed'] },
+ });
+ expect(duplicate.failures).toEqual([]);
+ expect(duplicate.sessionEntries).toEqual([]);
+ expect(readFileSync(target, 'utf8')).toBe('subsequent owner edit');
+ expect(new WorkLedger().get(`channel:telegram:${messageId}`)).toMatchObject({ status: 'completed', attempts: 1, delivered: true });
+ } finally {
+ await harness.dispose();
+ }
+ });
+
+ it('suppresses provider fallback after a real side effect and preserves the pending continuation honestly', async () => {
+ const harness = await EvalHarness.create({
+ name: `interrupted-write-${streaming}`, channel: { streaming },
+ providers: [{ name: 'primary', failOnCalls: [2], failWith: 'connection lost after write' }, { name: 'backup' }], turns: [],
+ });
+ const messageId = 'interrupted-after-write';
+ try {
+ const result = await harness.runTurn({
+ messageId, message: 'Create receipt.txt with accepted and report the result.',
+ steps: [toolStep([call('create_file', { path: 'receipt.txt', content: 'accepted' })])],
+ expect: { providerCalls: 2, noEventIncludes: ['Task complete', 'switching to `backup`'] },
+ });
+ expect(result.failures).toEqual([]);
+ expect(result.toolCalls).toHaveLength(1);
+ expect(result.toolCalls[0].recorded).toBe(false);
+ expect(readFileSync(join(harness.workDir, 'receipt.txt'), 'utf8')).toBe('accepted');
+ expect(harness.providers[1].callCount).toBe(0);
+ expect(result.providerCalls[1].error).toBe('connection lost after write');
+ expect(result.events.some((event) => event.text.includes('checkpointed'))).toBe(true);
+ expect(result.sessionEntries.some((entry) => entry.role === 'assistant' && entry.kind === 'message')).toBe(false);
+ // Ordinary chat has a durable inspected-continuation policy. This test
+ // does not advance its timer or claim generic work is never retried.
+ const saved = new WorkLedger().get(`channel:telegram:${messageId}`);
+ expect(saved).toMatchObject({ status: 'queued', attempts: 1, message: { metadata: { workContinuation: true } } });
+ expect(saved?.nextAttemptAt).toBeGreaterThan(Date.now());
+ } finally {
+ await harness.dispose();
+ }
+ });
+
+ it('deduplicates running work and keeps an owner-stopped turn cancelled after reload', async () => {
+ const harness = await EvalHarness.create({
+ name: `stop-in-flight-${streaming}`, channel: { streaming },
+ providers: [{ name: 'primary', waitForAbortOnCalls: [2] }, { name: 'backup' }], turns: [],
+ });
+ const messageId = 'stop-after-write';
+ const message = 'Create receipt.txt with accepted, then report the result.';
+ let pending: ReturnType<EvalHarness['runTurn']> | undefined;
+ try {
+ pending = harness.runTurn({
+ messageId, message,
+ steps: [toolStep([call('create_file', { path: 'receipt.txt', content: 'accepted' })])],
+ expect: { providerCalls: 2, finalTextIncludes: ['Task stopped'], noEventIncludes: ['Task complete'] },
+ });
+ await waitForModelCalls(harness, 2);
+ expect(existsSync(join(harness.workDir, 'receipt.txt'))).toBe(true);
+ harness.channel.inject({
+ id: messageId, content: message, channelId: harness.channelId, channelType: harness.channelType,
+ senderId: 'eval-user', timestamp: Date.now(),
+ });
+ expect(harness.agent.queuedCount).toBe(0);
+ expect(harness.agent.stopWorkFor(harness.channelId, 'Owner stopped this task.')).toBe(true);
+ const result = await pending;
+ pending = undefined;
+ expect(result.failures).toEqual([]);
+ expect(result.toolCalls).toHaveLength(1);
+ expect(result.toolCalls[0].recorded).toBe(false);
+ expect(result.events.some((event) => event.text.includes('already running'))).toBe(true);
+ expect(harness.providers[1].callCount).toBe(0);
+ expect(readFileSync(join(harness.workDir, 'receipt.txt'), 'utf8')).toBe('accepted');
+ const reloaded = new WorkLedger();
+ expect(reloaded.get(`channel:telegram:${messageId}`)).toMatchObject({ status: 'cancelled', attempts: 1 });
+ expect(reloaded.recoverInterrupted()).toEqual([]);
+
+ const redelivery = await harness.runTurn({ messageId, message, expect: { providerCalls: 0, toolsUsed: [], finalTextIncludes: ['cancelled earlier'] } });
+ expect(redelivery.failures).toEqual([]);
+ } finally {
+ harness.agent.stopWorkFor(harness.channelId);
+ if (pending) await pending;
+ await harness.dispose();
+ }
+ });
+});
diff --git a/src/eval/scripted-provider.ts b/src/eval/scripted-provider.ts
index da3a7120..25a5aac4 100644
--- a/src/eval/scripted-provider.ts
+++ b/src/eval/scripted-provider.ts
@@ -50,6 +50,8 @@ export class ScriptedProvider extends BaseProvider {
callCount = 0;
private readonly failFirst: number;
private readonly failWith: string;
+ private readonly failOnCalls: Set<number>;
+ private readonly waitForAbortOnCalls: Set<number>;
private readonly instance: MockLanguageModelV3;
constructor(spec: ScriptedProviderSpec, private readonly player: ScriptPlayer) {
@@ -59,6 +61,8 @@ export class ScriptedProvider extends BaseProvider {
this.model = model;
this.failFirst = spec.failFirst ?? 0;
this.failWith = spec.failWith ?? `${spec.name}: scripted outage (503 Service Unavailable)`;
+ this.failOnCalls = new Set(spec.failOnCalls);
+ this.waitForAbortOnCalls = new Set(spec.waitForAbortOnCalls);
this.instance = new MockLanguageModelV3({
provider: `eval.${spec.name}`,
modelId: model,
@@ -67,19 +71,34 @@ export class ScriptedProvider extends BaseProvider {
});
}
- private maybeFail(options: LanguageModelV3CallOptions, mode: 'stream' | 'generate'): ReturnType<ScriptPlayer['begin']> {
+ private async beginCall(options: LanguageModelV3CallOptions, mode: 'stream' | 'generate'): Promise<ReturnType<ScriptPlayer['begin']>> {
this.callCount++;
const record = this.player.begin(this.name, this.model, mode, options);
- if (this.callCount <= this.failFirst) {
+ if (this.callCount <= this.failFirst || this.failOnCalls.has(this.callCount)) {
record.error = this.failWith;
this.player.end(record);
throw new Error(this.failWith);
}
+ if (this.waitForAbortOnCalls.has(this.callCount)) {
+ try {
+ const signal = options.abortSignal;
+ if (!signal) throw new Error('Scripted abort wait requires an abort signal');
+ await new Promise<never>((_resolve, reject) => {
+ const abort = () => reject(signal.reason ?? new Error('Aborted'));
+ if (signal.aborted) abort();
+ else signal.addEventListener('abort', abort, { once: true });
+ });
+ } catch (error) {
+ record.error = error instanceof Error ? error.message : String(error);
+ this.player.end(record);
+ throw error;
+ }
+ }
return record;
}
private async doStream(options: LanguageModelV3CallOptions) {
- const record = this.maybeFail(options, 'stream');
+ const record = await this.beginCall(options, 'stream');
const step = this.player.nextStep(record);
const parts: LanguageModelV3StreamPart[] = [{ type: 'stream-start', warnings: [] }];
if (step.text) {
@@ -103,7 +122,7 @@ export class ScriptedProvider extends BaseProvider {
}
private async doGenerate(options: LanguageModelV3CallOptions) {
- const record = this.maybeFail(options, 'generate');
+ const record = await this.beginCall(options, 'generate');
const step = this.player.nextStep(record);
this.player.end(record);
return {