Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.5 KB
Raw Blame
/**
 * GBRAIN_EMBED_MAX_BATCH_TOKENS — operator-declared batch cap for
 * no_batch_cap recipes (ollama, llama-server, litellm). Reimplemented from
 * PR #3622 (drdeebtech).
 *
 * Why: local inference servers (ollama with `-np 1`) process one request
 * at a time and keep computing client-aborted requests. Without a batch
 * cap, embed() sends a page's ENTIRE chunk set as one request; a request
 * that outlives the embed timeout is retried forever by the stale-page
 * loop while the server grinds abandoned work — a self-sustaining
 * congestion collapse (observed 2026-07-29/30: 6,900+ timeouts, ~4 cores
 * pinned). The recipes deliberately declare `no_batch_cap: true` because
 * real capacity depends on the operator's hardware — so the cap is an
 * operator env knob, mirroring GBRAIN_EMBED_CONCURRENCY.
 *
 * The knob is read from the gateway's configure-time env snapshot
 * (`cfg.env`, Codex C3 — the gateway never reads process.env at call
 * time); production folds the whole process env into that snapshot via
 * buildGatewayConfig/mergedProviderEnv, so `export
 * GBRAIN_EMBED_MAX_BATCH_TOKENS=2048` reaches it. Tests therefore pass the
 * knob through configureGateway({env}) — hermetic, no process.env writes.
 *
 * Coverage:
 *  - env cap set + no_batch_cap recipe → pre-split into bounded batches
 *  - env cap absent → existing single-call fast path (regression guard)
 *  - invalid env values (0, negative, garbage, empty) → fast path unchanged
 *  - recipe-declared max_batch_tokens still wins over the env knob
 */

import { afterAll, afterEach, beforeEach, describe, expect, mock, test } from 'bun:test';
import {
  configureGateway,
  resetGateway,
  embed,
  __setEmbedTransportForTests,
} from '../../src/core/ai/gateway.ts';

// Same rationale as test/ai/adaptive-embed-batch.test.ts: without a final
// reset, this file's last gateway config leaks into whichever test file the
// shard runs next.
afterAll(() => resetGateway());

function fakeEmbeddings(values: string[], dims: number): { embeddings: number[][] } {
  return {
    embeddings: values.map((_, i) =>
      Array.from({ length: dims }, (_, j) => (j === 0 ? i : 0.1)),
    ),
  };
}

const ENV_KEY = 'GBRAIN_EMBED_MAX_BATCH_TOKENS';

function configureOllama(env: Record<string, string | undefined> = {}): void {
  configureGateway({
    embedding_model: 'ollama:nomic-embed-text',
    embedding_dimensions: 768,
    env,
  });
}

describe('GBRAIN_EMBED_MAX_BATCH_TOKENS env cap for no_batch_cap recipes', () => {
  beforeEach(() => resetGateway());
  afterEach(() => __setEmbedTransportForTests(null));

  test('env cap set → ollama batches are pre-split and each stays within the token budget', async () => {
    configureOllama({ [ENV_KEY]: '1000' });

    const stub = mock(async ({ values }: { values: string[] }) => fakeEmbeddings(values, 768));
    __setEmbedTransportForTests(stub as any);

    // 10 texts × 2000 chars = 500 tokens each at the default 4 chars/token.
    // Budget 1000 tokens → at most 2 texts per batch (fewer once the default
    // safety factor tightens it) → several transport calls, never 1.
    const texts = Array.from({ length: 10 }, (_, i) => `${i}`.repeat(2000));
    const result = await embed(texts);

    expect(result).toHaveLength(10);
    expect(stub.mock.calls.length).toBeGreaterThan(1);
    for (const [arg] of stub.mock.calls) {
      const batch = (arg as { values: string[] }).values;
      const batchChars = batch.reduce((s, t) => s + t.length, 0);
      // 1000 tokens × 4 chars/token = 4000 chars absolute ceiling per batch
      // (the safety factor only ever tightens below this).
      expect(batchChars).toBeLessThanOrEqual(4000);
    }
  });

  test('env cap absent → single transport call (existing fast-path preserved)', async () => {
    configureOllama({});

    const stub = mock(async ({ values }: { values: string[] }) => fakeEmbeddings(values, 768));
    __setEmbedTransportForTests(stub as any);

    const texts = Array.from({ length: 10 }, (_, i) => `${i}`.repeat(2000));
    const result = await embed(texts);

    expect(stub).toHaveBeenCalledTimes(1);
    expect(result).toHaveLength(10);
  });

  test.each(['0', '-5', 'abc', ''])('invalid env value %j → fast path unchanged', async (bad) => {
    configureOllama({ [ENV_KEY]: bad });

    const stub = mock(async ({ values }: { values: string[] }) => fakeEmbeddings(values, 768));
    __setEmbedTransportForTests(stub as any);

    const result = await embed(['a'.repeat(8000), 'b'.repeat(8000)]);
    expect(stub).toHaveBeenCalledTimes(1);
    expect(result).toHaveLength(2);
  });

  test('recipe-declared max_batch_tokens wins over a larger env value', async () => {
    // Voyage declares max_batch_tokens=120000 with safety_factor 0.5 →
    // 60K-token effective budget. A huge env value must NOT loosen it.
    configureGateway({
      embedding_model: 'voyage:voyage-3-large',
      embedding_dimensions: 1024,
      env: { VOYAGE_API_KEY: 'sk-fake', [ENV_KEY]: '99000000' },
    });

    const stub = mock(async ({ values }: { values: string[] }) => fakeEmbeddings(values, 1024));
    __setEmbedTransportForTests(stub as any);

    // 20 texts × 8000 chars (= MAX_CHARS, so no truncation) = 8000 tokens
    // each at voyage's 1 char/token density → 160K tokens total against a
    // 60K effective budget → must split into >1 call. If the env value
    // (99M) leaked past the recipe cap, everything would fit in 1 call.
    const texts = Array.from({ length: 20 }, (_, i) => `${i % 10}`.repeat(8000));
    await embed(texts);
    expect(stub.mock.calls.length).toBeGreaterThan(1);
  });
});