Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.0 KB
Raw Blame
/**
 * #6523: since #6424 forwarded a call's `maxTokens` as
 * CLAUDE_CODE_MAX_OUTPUT_TOKENS, Claude Code reports a response that hits
 * that cap as an `is_error: true` result envelope ("API Error: Claude's
 * response exceeded the N output token maximum. ..."), not as a success with
 * `stop_reason: 'max_tokens'`. `runClaude` rejected on `is_error`, so no
 * caller ever saw `finishReason: 'length'` and the escalation ladders
 * (loops_extract 2048 → 8192, facts/extract 1× → 2×) never fired: the minion
 * queue retried the same capped call three times and the job died.
 *
 * The provider now maps that envelope (anchored prefix, `is_error: true`, no
 * `api_error_status`, any cap, any exit code) to `length` with empty text and
 * the usage as reported. Every other error envelope stays a typed failure.
 *
 * A POSIX shell stub at GBRAIN_CLAUDE_CLI_BIN plays the CLI; it records the
 * cap it was handed so the ladder cases can assert the sequence of calls.
 * "Real gateway" below means the gateway code path with the stub transport;
 * no live provider account is used.
 *
 * Supersedes PR #6525 (@javieraldape), whose full-sentence match and exit-code
 * gate missed the discriminator cases below (R16).
 */
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test';
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { LanguageModelV2CallOptions } from '@ai-sdk/provider';
import { ClaudeCliLanguageModel, ClaudeCliProcessError } from '../src/core/ai/providers/claude-cli-language-model.ts';
import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { LoopsExtractRetryableError, runLoopsExtract } from '../src/core/google/loops-extract.ts';
import { extractFactsFromTurnWithOutcome } from '../src/core/facts/extract.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';

const capMessage = (cap: number) =>
  `API Error: Claude's response exceeded the ${cap} output token maximum. To configure this behavior, set the CLAUDE_CODE_MAX_OUTPUT_TOKENS environment variable.`;
const USAGE = { input_tokens: 10, output_tokens: 2048, cache_read_input_tokens: 3, cache_creation_input_tokens: 2 };

function envelope(result: string, extra: Record<string, unknown> = {}): Record<string, unknown> {
  return {
    type: 'result', subtype: 'error_during_execution', is_error: true,
    result, stop_reason: null, session_id: 'synthetic-session', num_turns: 1, usage: USAGE,
    ...extra,
  };
}

interface Stub { dir: string; caps: () => string[] }

/**
 * Stage a stub `claude`. `responses` maps the forwarded cap (the env value,
 * '' when none was forwarded) to the stdout it prints and its exit code; `'*'`
 * is the fallback. Raw strings are printed verbatim (malformed stdout).
 */
async function withCli<T>(
  responses: Record<string, { body: unknown; exit: number }>,
  run: (stub: Stub) => Promise<T>,
): Promise<T> {
  const dir = mkdtempSync(join(tmpdir(), 'claude-output-cap-'));
  const capsPath = join(dir, 'caps');
  const lines = ['#!/bin/sh', 'if [ "$1" = "--version" ]; then echo "2.1.316 (Claude Code)"; exit 0; fi', 'cat > /dev/null', `printf '%s\\n' "$CLAUDE_CODE_MAX_OUTPUT_TOKENS" >> "${capsPath}"`];
  for (const [cap, { body, exit }] of Object.entries(responses)) {
    const file = join(dir, `response-${cap === '*' ? 'default' : cap}.txt`);
    writeFileSync(file, typeof body === 'string' ? body : JSON.stringify(body));
    if (cap !== '*') lines.push(`if [ "$CLAUDE_CODE_MAX_OUTPUT_TOKENS" = "${cap}" ]; then cat "${file}"; exit ${exit}; fi`);
    else lines.push(`cat "${file}"; exit ${exit}`);
  }
  if (!responses['*']) lines.push('echo "unexpected cap $CLAUDE_CODE_MAX_OUTPUT_TOKENS" >&2; exit 99');
  const bin = join(dir, 'claude');
  writeFileSync(bin, lines.join('\n') + '\n');
  chmodSync(bin, 0o755);
  const stub: Stub = { dir, caps: () => (existsSync(capsPath) ? readFileSync(capsPath, 'utf8').trimEnd().split('\n') : []) };
  try {
    return await withEnv({ GBRAIN_CLAUDE_CLI_BIN: bin, CLAUDE_CODE_MAX_OUTPUT_TOKENS: undefined }, () => run(stub));
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
}

function options(maxOutputTokens?: number): LanguageModelV2CallOptions {
  return {
    prompt: [{ role: 'user', content: [{ type: 'text', text: 'Synthetic thread' }] }],
    ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
  } as LanguageModelV2CallOptions;
}

const generate = (opts = options(2048)) => new ClaudeCliLanguageModel('claude-sonnet-4-6').doGenerate(opts);

async function rejection(run: () => Promise<unknown>): Promise<unknown> {
  try {
    await run();
  } catch (error) {
    return error;
  }
  throw new Error('expected the call to reject');
}

describe('claude-cli output-cap envelope finishes as length (#6523)', () => {
  const positives: Array<[string, unknown, number, number | undefined]> = [
    ['exit 0, error_during_execution', envelope(capMessage(2048)), 0, 2048],
    ['exit 1', envelope(capMessage(2048)), 1, 2048],
    ['exit 2', envelope(capMessage(2048)), 2, 2048],
    ['verbose event-array form', [{ type: 'system', subtype: 'init' }, { type: 'assistant' }, envelope(capMessage(2048))], 1, 2048],
    ['subtype success as field-reported', envelope(capMessage(2048), { subtype: 'success' }), 1, 2048],
    ['no forwarded cap (the default cap was hit)', envelope(capMessage(32000)), 1, undefined],
    ['reported cap differs from the forwarded cap', envelope(capMessage(8192)), 1, 2048],
    ['changed trailing wording after the anchored prefix', envelope("API Error: Claude's response exceeded the 2048 output token maximum. Raise the limit in settings."), 1, 2048],
  ];
  for (const [label, body, exit, cap] of positives) {
    test(`${label} → length, empty text, usage as reported`, async () => {
      await withCli({ '*': { body, exit } }, async (stub) => {
        const out = await generate(options(cap));
        expect(out.finishReason).toBe('length');
        expect(out.content).toEqual([{ type: 'text', text: '' }]);
        expect(out.usage).toEqual({ inputTokens: 15, outputTokens: 2048, totalTokens: 2063, cachedInputTokens: 3 });
        expect(stub.caps()).toEqual([cap === undefined ? '' : String(cap)]);
      });
    });
  }

  test('usage absent → length with undefined usage, nothing invented', async () => {
    await withCli({ '*': { body: envelope(capMessage(2048), { usage: undefined }), exit: 1 } }, async () => {
      const out = await generate();
      expect(out.finishReason).toBe('length');
      expect(out.content).toEqual([{ type: 'text', text: '' }]);
      expect(out.usage).toEqual({ inputTokens: undefined, outputTokens: undefined, totalTokens: undefined, cachedInputTokens: undefined });
    });
  });

  test('error envelopes that are not the cap stay ClaudeCliProcessError with their status and exit code', async () => {
    const negatives: Array<[string, Record<string, unknown>]> = [
      ['api_error_status on the cap text', envelope(capMessage(2048), { api_error_status: 429 })],
      ['cap text after another sentence', envelope(`Authentication failed. ${capMessage(2048)}`, { api_error_status: 401 })],
      ['cap text quoted mid-answer', envelope(`Quoted provider documentation: ${capMessage(2048)}`)],
      ['insufficient credits', envelope('API Error: insufficient credits', { api_error_status: 402 })],
      ['server error carrying stop_reason max_tokens', envelope('API Error: transient server error', { stop_reason: 'max_tokens', api_error_status: 500 })],
      ['a non-string result', envelope(capMessage(2048), { result: { message: capMessage(2048) } })],
      ['is_error as a string', envelope(capMessage(2048), { is_error: 'true' })],
    ];
    for (const [label, body] of negatives) {
      for (const exit of [0, 1, 2]) {
        await withCli({ '*': { body, exit } }, async () => {
          const error = await rejection(generate);
          expect(error, `${label} (exit ${exit})`).toBeInstanceOf(ClaudeCliProcessError);
          expect((error as ClaudeCliProcessError).apiErrorStatus, label).toBe(body.api_error_status as number | undefined);
          expect((error as ClaudeCliProcessError).exitCode, label).toBe(exit);
        });
      }
    }
  });

  // #7273: Claude Code 2.1.305 writes the key as JSON null instead of leaving it out.
  test('a null api_error_status counts as no status: the cap is length, anything else stays a typed failure', async () => {
    const nullStatus = { api_error_status: null };
    for (const exit of [0, 1]) {
      for (const body of [envelope(capMessage(2048), { ...nullStatus, subtype: 'success', stop_reason: 'stop_sequence' }),
        [{ type: 'system', subtype: 'init' }, envelope(capMessage(2048), nullStatus)]]) {
        await withCli({ '*': { body, exit } }, async () => {
          const out = await generate();
          expect(out.finishReason, `exit ${exit}`).toBe('length');
          expect(out.content).toEqual([{ type: 'text', text: '' }]);
        });
      }
      for (const result of ['API Error: overloaded', `See the docs: ${capMessage(2048)}`]) {
        await withCli({ '*': { body: envelope(result, nullStatus), exit } }, async () => {
          const error = await rejection(generate);
          expect(error, `${result} (exit ${exit})`).toBeInstanceOf(ClaudeCliProcessError);
          expect((error as ClaudeCliProcessError).apiErrorStatus).toBeUndefined();
          expect((error as ClaudeCliProcessError).exitCode).toBe(exit);
        });
      }
    }
  });

  test('a success envelope quoting the prefix is ordinary model text', async () => {
    const quote = `The CLI said: ${capMessage(2048)}`;
    await withCli({ '*': { body: envelope(quote, { is_error: false, subtype: 'success', stop_reason: 'end_turn' }), exit: 0 } }, async () => {
      const out = await generate();
      expect(out.finishReason).toBe('stop');
      expect(out.content).toEqual([{ type: 'text', text: quote }]);
    });
    await withCli({ '*': { body: envelope(capMessage(2048), { is_error: false, subtype: 'success', stop_reason: 'end_turn' }), exit: 0 } }, async () => {
      const out = await generate();
      expect(out.finishReason).toBe('stop');
      expect(out.content).toEqual([{ type: 'text', text: capMessage(2048) }]);
    });
  });

  test('success + non-zero exit, exit 137 without an envelope, malformed stdout and a resultless event array still fail', async () => {
    await withCli({ '*': { body: envelope(capMessage(2048), { is_error: false, subtype: 'success' }), exit: 1 } }, async () => {
      const error = await rejection(generate);
      expect(error).toBeInstanceOf(ClaudeCliProcessError);
      expect((error as ClaudeCliProcessError).exitCode).toBe(1);
    });
    await withCli({ '*': { body: capMessage(2048), exit: 137 } }, async () => {
      const error = await rejection(generate);
      expect(error).toBeInstanceOf(ClaudeCliProcessError);
      expect((error as ClaudeCliProcessError).exitCode).toBe(137);
    });
    for (const body of [capMessage(2048), '"' + capMessage(2048) + '"', 'null', '2048']) {
      await withCli({ '*': { body, exit: 0 } }, async () => {
        const error = await rejection(generate);
        expect(error).toBeInstanceOf(Error);
        expect((error as Error).message).toContain('claude-cli output not JSON');
      });
    }
    await withCli({ '*': { body: [{ type: 'system', subtype: 'init' }, { type: 'assistant', message: capMessage(2048) }], exit: 0 } }, async () => {
      const error = await rejection(generate);
      expect((error as Error).message).toContain('had no "result" event');
    });
  });
});

const success = (text: string, usage: Record<string, number> = { input_tokens: 10, output_tokens: 30 }) =>
  envelope(text, { is_error: false, subtype: 'success', stop_reason: 'end_turn', usage });

describe('escalation ladders through the gateway with the stub transport', () => {
  const engines: Array<{ backend: string; engine: BrainEngine }> = [];
  const dbDir = mkdtempSync(join(tmpdir(), 'claude-output-cap-db-'));
  let closePostgres: (() => Promise<void>) | undefined;

  beforeAll(async () => {
    for (const backend of testBackends()) {
      if (backend === 'pglite') {
        const engine = new PGLiteEngine();
        await engine.connect({ database_path: dbDir });
        await engine.initSchema();
        engines.push({ backend, engine });
      } else {
        const isolated = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
        engines.push({ backend, engine: isolated.engine });
        closePostgres = isolated.close;
      }
    }
  }, 120_000);
  afterEach(resetGateway);
  afterAll(async () => {
    resetGateway();
    for (const { engine } of engines) await engine.disconnect();
    await closePostgres?.();
    rmSync(dbDir, { recursive: true, force: true });
  });

  async function thread(engine: BrainEngine, slug: string) {
    await engine.executeRaw('DELETE FROM open_loops');
    await engine.putPage(slug, {
      type: 'email', title: 'Synthetic decision',
      compiled_truth: 'Which week works for the kickoff?',
      frontmatter: { thread_id: `thread-${slug}` },
    }, { sourceId: 'default' });
    configureGateway({ chat_model: 'claude-cli:claude-sonnet-4-6', env: {} });
  }

  test('loops_extract: a 2048 cap escalates once to 8192 and publishes the decision', async () => {
    const complete = success(JSON.stringify({
      commitments: [], decisions_pending: [{ text: 'Pick a kickoff week', quote: 'Which week works for the kickoff?' }],
    }));
    for (const { backend, engine } of engines) {
      await withCli({ '2048': { body: envelope(capMessage(2048)), exit: 1 }, '8192': { body: complete, exit: 0 } }, async (stub) => {
        await thread(engine, 'emails/output-cap-escalates');
        const result = await runLoopsExtract(engine, { slug: 'emails/output-cap-escalates', sourceId: 'default' });
        expect(result.status, backend).toBe('extracted');
        expect(result.decisions, backend).toBe(1);
        expect(stub.caps(), backend).toEqual(['2048', '8192']);
        const rows = await engine.executeRaw<{ summary: string }>('SELECT summary FROM open_loops');
        expect(rows.map(row => row.summary), backend).toEqual(['Pick a kickoff week']);
      });
    }
  }, 120_000);

  test('loops_extract: two cap failures stop at the ceiling as a retryable truncation with no rows and no third call', async () => {
    for (const { backend, engine } of engines) {
      await withCli({ '2048': { body: envelope(capMessage(2048)), exit: 1 }, '8192': { body: envelope(capMessage(8192)), exit: 1 } }, async (stub) => {
        await thread(engine, 'emails/output-cap-ceiling');
        const error = await rejection(() => runLoopsExtract(engine, { slug: 'emails/output-cap-ceiling', sourceId: 'default' }));
        expect(error, backend).toBeInstanceOf(LoopsExtractRetryableError);
        expect((error as LoopsExtractRetryableError).reason, backend).toBe('truncated');
        expect((error as Error).message, backend).toContain('8192-token');
        expect(stub.caps(), backend).toEqual(['2048', '8192']);
        expect(await engine.executeRaw('SELECT id FROM open_loops'), backend).toHaveLength(0);
      });
    }
  }, 120_000);

  test('facts/extract: a cap at the configured maximum retries once at twice the cap', async () => {
    const facts = success(JSON.stringify({
      facts: [{ fact: 'Alice-example will confirm the kickoff week in November.', entity: 'people/alice-example', kind: 'fact', confidence: 0.9, notability: 'medium' }],
    }));
    await withCli({ '4000': { body: envelope(capMessage(4000)), exit: 1 }, '8000': { body: facts, exit: 0 } }, async (stub) => {
      configureGateway({ chat_model: 'claude-cli:claude-sonnet-4-6', env: {} });
      const outcome = await extractFactsFromTurnWithOutcome({
        turnText: 'Alice-example said she will confirm the kickoff week in November.', source: 'test:output-cap',
        model: 'claude-cli:claude-sonnet-4-6',
      });
      expect(outcome.ok).toBe(true);
      if (outcome.ok) expect(outcome.facts.map(f => f.fact)).toEqual(['Alice-example will confirm the kickoff week in November.']);
      expect(stub.caps()).toEqual(['4000', '8000']);
    });
  }, 60_000);

  test('facts/extract: caps at both rungs end as truncated_output after exactly two calls', async () => {
    await withCli({ '4000': { body: envelope(capMessage(4000)), exit: 1 }, '8000': { body: envelope(capMessage(8000)), exit: 2 } }, async (stub) => {
      configureGateway({ chat_model: 'claude-cli:claude-sonnet-4-6', env: {} });
      const outcome = await extractFactsFromTurnWithOutcome({
        turnText: 'Alice-example said she will confirm the kickoff week in November.', source: 'test:output-cap',
        model: 'claude-cli:claude-sonnet-4-6',
      });
      expect(outcome.ok).toBe(false);
      if (!outcome.ok) expect(outcome.reason).toBe('truncated_output');
      expect(stub.caps()).toEqual(['4000', '8000']);
    });
  }, 60_000);
});