Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

13.0 KB
Raw Blame
import { describe, expect, it } from 'vitest';
import {
  MAX_EXECUTE_CONTINUATIONS,
  MAX_VERIFICATION_CONTINUATIONS,
  executeContinuationPrompt,
  isFailedToolResult,
  isTextDeliverableRequest,
  isVerificationOutputOk,
  responseAsksUser,
  shouldForceExecuteContinuation,
  shouldRequireVerification,
  verificationPrompt,
  wakeUpPrompt,
} from './execute-guard.js';

const ok = (names: string[]): Map<string, boolean> => new Map(names.map((n) => [n, true]));

describe('chat delivery contract', () => {
  it('text deliverables are exempt — the reply itself is the work', () => {
    for (const text of [
      'write a poem about the sea',
      'draft an email to the landlord',
      'give me a catchy slogan for the app',
      'write a short story about a fox',
      'write the lyrics for a chorus',
      'write a haiku about autumn',
      'brainstorm names for the project',
      'write a blog post intro',
    ]) {
      expect(isTextDeliverableRequest(text), text).toBe(true);
    }
  });

  it('repo/code-shaped requests are NOT text deliverables — the guard applies in chat', () => {
    for (const text of [
      'fix the login bug',
      'build a dashboard for the API',
      'add tests to the auth module',
      'refactor the session store',
      'create a component for the settings page',
      'implement the endpoint',
      'set up the database schema',
    ]) {
      expect(isTextDeliverableRequest(text), text).toBe(false);
    }
  });

  it('the narration guard still fires for chat implementation requests with no tools', () => {
    expect(shouldForceExecuteContinuation({
      taskText: 'fix the login bug',
      hasApprovedPlan: false,
      toolsUsed: [],
      toolsSucceeded: new Map(),
    })).toBe(true);
  });
});

describe('execute-mode completion guard', () => {
  it('forces continuation when an implementation request ended with no tools', () => {
    expect(shouldForceExecuteContinuation({
      taskText: 'please develop it now',
      hasApprovedPlan: false,
      toolsUsed: [],
      toolsSucceeded: new Map(),
    })).toBe(true);
    expect(shouldForceExecuteContinuation({
      taskText: 'build the Omega Project per its spec',
      hasApprovedPlan: false,
      toolsUsed: ['read_file', 'list_dir', 'git_status'],
      toolsSucceeded: ok(['read_file', 'list_dir', 'git_status']),
    })).toBe(true);
    expect(shouldForceExecuteContinuation({
      taskText: 'create a landing page for my app',
      hasApprovedPlan: false,
      toolsUsed: [],
      toolsSucceeded: new Map(),
    })).toBe(true);
  });

  it('allows finishing once a mutating tool actually ran (legacy semantics when success map is absent)', () => {
    for (const tool of ['write_file', 'edit_file', 'create_file', 'run_command', 'git_commit', 'delegate_task']) {
      expect(shouldForceExecuteContinuation({
        taskText: 'build the Omega Project per its spec',
        hasApprovedPlan: false,
        toolsUsed: ['read_file', tool, 'list_dir'],
      })).toBe(false);
    }
  });

  it('allows finishing when a mutating tool produced a successful result', () => {
    for (const tool of ['write_file', 'edit_file', 'create_file', 'run_command', 'git_commit']) {
      expect(shouldForceExecuteContinuation({
        taskText: 'build the Omega Project per its spec',
        hasApprovedPlan: false,
        toolsUsed: ['read_file', tool, 'list_dir'],
        toolsSucceeded: ok(['read_file', tool, 'list_dir']),
      })).toBe(false);
    }
  });

  it('does NOT accept a mutating tool that only ever failed', () => {
    expect(shouldForceExecuteContinuation({
      taskText: 'build the Omega Project per its spec',
      hasApprovedPlan: false,
      toolsUsed: ['create_file', 'write_file'],
      toolsSucceeded: new Map([['create_file', false], ['write_file', false]]),
    })).toBe(true);
    // Mixed: one succeeded → satisfied.
    expect(shouldForceExecuteContinuation({
      taskText: 'build the Omega Project per its spec',
      hasApprovedPlan: false,
      toolsUsed: ['create_file', 'write_file'],
      toolsSucceeded: new Map([['create_file', false], ['write_file', true]]),
    })).toBe(false);
  });

  it('allows finishing when the model paused with ask_user', () => {
    expect(shouldForceExecuteContinuation({
      taskText: 'build the Omega Project',
      hasApprovedPlan: false,
      toolsUsed: ['read_file', 'ask_user'],
      toolsSucceeded: ok(['read_file']),
    })).toBe(false);
  });

  it('forces continuation whenever an approved plan is pending', () => {
    expect(shouldForceExecuteContinuation({
      taskText: 'go ahead',
      hasApprovedPlan: true,
      toolsUsed: ['git_log'],
      toolsSucceeded: ok(['git_log']),
    })).toBe(true);
    expect(shouldForceExecuteContinuation({
      taskText: 'go ahead',
      hasApprovedPlan: true,
      toolsUsed: [],
      toolsSucceeded: new Map(),
    })).toBe(true);
  });

  it('never fires on conversation-only requests', () => {
    const noTools: string[] = [];
    for (const text of [
      'thanks',
      'Thank you!',
      'ok',
      'what is 2+2?',
      'why does this fail?',
      'how do I install mercury?',
      'explain this file',
      'tell me about the codebase',
      'hi',
    ]) {
      expect(shouldForceExecuteContinuation({
        taskText: text,
        hasApprovedPlan: false,
        toolsUsed: noTools,
        toolsSucceeded: new Map(),
      })).toBe(false);
    }
  });

  it('never fires for read-only research phrasing', () => {
    expect(shouldForceExecuteContinuation({
      taskText: 'list the files in src',
      hasApprovedPlan: false,
      toolsUsed: ['list_dir'],
      toolsSucceeded: ok(['list_dir']),
    })).toBe(false);
  });

  it('allows an empty task text to finish', () => {
    expect(shouldForceExecuteContinuation({
      taskText: '',
      hasApprovedPlan: false,
      toolsUsed: [],
      toolsSucceeded: new Map(),
    })).toBe(false);
  });

  it('bounds the continuation rounds', () => {
    // Generous by design — automatic continuation is the norm; the pause is
    // a runaway backstop, not a checkpoint.
    expect(MAX_EXECUTE_CONTINUATIONS).toBeGreaterThanOrEqual(3);
    expect(MAX_EXECUTE_CONTINUATIONS).toBeLessThanOrEqual(8);
  });

  it('builds a bounded continuation nudge', () => {
    const prompt = executeContinuationPrompt('build the omega project');
    expect(prompt).toContain('EXECUTE-MODE GUARD');
    expect(prompt).toContain('build the omega project');
    expect(prompt).toContain('tools');
    const long = 'x'.repeat(500);
    const bounded = executeContinuationPrompt(long);
    expect(bounded).not.toContain('x'.repeat(250));
  });

  it('detects failed tool results from executor output markers', () => {
    expect(isFailedToolResult('Error: Permission denied for write access to /repo')).toBe(true);
    expect(isFailedToolResult('Command exited with code 1')).toBe(true);
    expect(isFailedToolResult('⏱ Command timed out after 120s.')).toBe(true);
    expect(isFailedToolResult('Command failed: something broke')).toBe(true);
    expect(isFailedToolResult('Successfully created /repo/src/app.ts (120 bytes)')).toBe(false);
    expect(isFailedToolResult('Successfully wrote 512 bytes to /repo/src/app.ts')).toBe(false);
    expect(isFailedToolResult('src/app.ts +8 -2')).toBe(false);
    // Error markers beyond the first 300 chars don't flag an otherwise
    // successful result (error summary belongs at the head).
    const long = 'ok '.repeat(200) + 'Error: at the very end';
    expect(isFailedToolResult(long)).toBe(false);
  });
});
describe('evidence-based verification gate', () => {
  it('requires verification when changes landed but nothing verified them', () => {
    expect(shouldRequireVerification({
      taskText: 'add the export endpoint',
      hasApprovedPlan: false,
      commandsRun: ['ls src'],
      toolsSucceeded: ok(['edit_file', 'write_file']),
    })).toBe(true);
  });

  it('accepts a build/test/typecheck run as evidence', () => {
    for (const command of [
      'npm test',
      'npm run build',
      'pnpm typecheck',
      'npx vitest run src/app.test.ts',
      'cargo test',
      'go test ./...',
      'make check',
      'pytest -q',
      'tsc --noEmit',
    ]) {
      expect(shouldRequireVerification({
        taskText: 'add the export endpoint',
        hasApprovedPlan: false,
        commandsRun: [command],
        toolsSucceeded: ok(['edit_file']),
      }), command).toBe(false);
    }
  });

  it('never requires verification without a successful mutation', () => {
    expect(shouldRequireVerification({
      taskText: 'add the export endpoint',
      hasApprovedPlan: false,
      commandsRun: [],
      toolsSucceeded: ok(['read_file']),
    })).toBe(false);
    expect(shouldRequireVerification({
      taskText: 'add the export endpoint',
      hasApprovedPlan: false,
      commandsRun: [],
      toolsSucceeded: new Map([['edit_file', false]]),
    })).toBe(false);
  });

  it('skips conversational and question-style tasks', () => {
    expect(shouldRequireVerification({
      taskText: 'thanks!',
      hasApprovedPlan: false,
      commandsRun: [],
      toolsSucceeded: ok(['edit_file']),
    })).toBe(false);
    expect(shouldRequireVerification({
      taskText: 'why is the build failing?',
      hasApprovedPlan: false,
      commandsRun: [],
      toolsSucceeded: ok(['edit_file']),
    })).toBe(false);
  });

  it('treats non-verification commands as insufficient even for approved plans', () => {
    expect(shouldRequireVerification({
      taskText: 'implement the plan',
      hasApprovedPlan: true,
      commandsRun: ['git status', 'ls'],
      toolsSucceeded: ok(['create_file']),
    })).toBe(true);
  });

  it('bounds verification rounds to one', () => {
    expect(MAX_VERIFICATION_CONTINUATIONS).toBe(1);
  });

  // Regression: a verification command that FAILED or TIMED OUT used to
  // satisfy the gate because only the command string was inspected. With
  // result tracking, only a clean run after the last edit counts.
  it('a failed or timed-out verification is not evidence', () => {
    const base = {
      taskText: 'add the export endpoint',
      hasApprovedPlan: false,
      commandsRun: ['npm test'],
      toolsSucceeded: ok(['edit_file']),
    };
    expect(shouldRequireVerification({ ...base, verifiedOk: false })).toBe(true);
    expect(shouldRequireVerification({ ...base, verifiedOk: true })).toBe(false);
    // Without result tracking the legacy string heuristic still applies.
    expect(shouldRequireVerification(base)).toBe(false);
  });

  it('classifies verification output by outcome, not by command name', () => {
    expect(isVerificationOutputOk('Tests  25 passed (25)\nDuration 421ms')).toBe(true);
    expect(isVerificationOutputOk('Command exited with code 1\nFAIL src/app.test.ts')).toBe(false);
    expect(isVerificationOutputOk('⏱ Command timed out after 120s.\nPartial output:\n...')).toBe(false);
    expect(isVerificationOutputOk('Error: Permission denied for shell command')).toBe(false);
    expect(isVerificationOutputOk('src/app.ts(12,5): error: TS2322: Type ...')).toBe(false);
    expect(isVerificationOutputOk('')).toBe(false);
  });

  it('builds a verification nudge', () => {
    const prompt = verificationPrompt('add the export endpoint');
    expect(prompt).toContain('EXECUTE-MODE VERIFICATION');
    expect(prompt).toContain('add the export endpoint');
    expect(prompt).toContain('build, test, or typecheck');
  });
});

describe('responseAsksUser — prose questions are legitimate pauses', () => {
  it('detects a turn that ends by asking the user something', () => {
    expect(responseAsksUser('Could you remind me what the AI bot was supposed to do?')).toBe(true);
    expect(responseAsksUser('Which option do you want?\n2. A Telegram bot that creates notes?')).toBe(true);
    expect(responseAsksUser('Working on it.\n\nShall I proceed?')).toBe(true);
  });

  it('does not treat statements or mid-text questions as user questions', () => {
    expect(responseAsksUser('Built the endpoint. What changed: the router now handles POST /notes.')).toBe(false);
    expect(responseAsksUser('Why did this fail? The answer: missing env var. Fixed now.')).toBe(false);
    expect(responseAsksUser('')).toBe(false);
  });
});

describe('wakeUpPrompt — the final automatic attempt', () => {
  it('demands a mutating tool call first, zero prose', () => {
    const prompt = wakeUpPrompt('build the 3D world');
    expect(prompt).toContain('WAKE-UP CALL');
    expect(prompt).toContain('MUST begin with a mutating tool call');
    expect(prompt).toContain('create_file');
    expect(prompt).toContain('ZERO prose');
    expect(prompt).toContain('3D world');
  });
});

describe('chat: check before done', () => {
  it('only when the turn made something', async () => {
    const { chatNeedsCheck, chatCheckPrompt } = await import('./execute-guard.js');
    expect(chatNeedsCheck(new Map([['run_command', true]]))).toBe(true);
    expect(chatNeedsCheck(new Map([['run_command', false], ['web_search', true]]))).toBe(false);
    expect(chatNeedsCheck(new Map([['read_file', true]]))).toBe(false);
    expect(chatCheckPrompt('make an intro video')).toMatch(/probe a video/);
    expect(chatCheckPrompt()).toMatch(/do not present a workaround/);
  });
});