Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.0 KB
Raw Blame
/**
 * S1: `search.reranker.model typesafe:jev-1.13.0` through the decide core.
 * Ports the #5178 reranker contract (credit: dsandrade): native System One
 * score questions, bearer auth, normalized four-level scores, stable ties,
 * batch-local indices merged globally, topN after all batches, no truncation,
 * bounded concurrency, status-only errors, budget accounting and fail-open
 * through applyReranker. Adds the resolved-model check (mixed ids fail open)
 * and the JEV_TYPESAFE_API_KEY alias.
 */
import { afterEach, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
  configureGateway, resetGateway, rerank, RerankError, __setRerankTransportForTests, withBudgetTracker, type RerankMeta,
} from '../../src/core/ai/gateway.ts';
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
import { rerankerReadiness } from '../../src/core/ai/reranker-readiness.ts';
import { DEFAULT_RERANKER_MODEL } from '../../src/core/ai/defaults.ts';
import { BudgetTracker } from '../../src/core/budget/budget-tracker.ts';
import { applyReranker } from '../../src/core/search/rerank.ts';
import type { SearchResult } from '../../src/core/types.ts';
import { estimateContextTokens, planBatches } from '../../src/core/ai/decide/pack.ts';
import { rerankQuestions } from '../../src/core/ai/decide/rerank-adapter.ts';
import { parseTypeSafeResponse, toWireQuestion } from '../../src/core/ai/decide/providers/typesafe.ts';

const MODEL = 'typesafe:jev-1.13.0';
const longDocuments = (count: number) => Array.from({ length: count }, (_, i) => `candidate ${i}: ${'evidence '.repeat(500)}`);
const queuedDocuments = () => Array.from({ length: 280 }, () => 'evidence '.repeat(2000));
function configure(env: Record<string, string> = { TYPESAFE_API_KEY: 'sk-test-typesafe' }): void {
  configureGateway({ reranker_model: MODEL, env });
}
function response(scores: number[], tokens = 100, model = 'jev-1.13.0'): Response {
  return new Response(JSON.stringify({
    model,
    answers: Object.fromEntries(scores.map((score, i) => [`document_${i}`, { type: 'score', score, confidence: 0.9 }])),
    usage: { input_tokens: tokens, output_tokens: 12 },
  }), { headers: { 'content-type': 'application/json' } });
}
function docsFromBody(body: any): string[] { return Object.values(body.questions).map((q: any) => q.instructions.candidate); }
function docsFor(init: RequestInit): string[] { return docsFromBody(JSON.parse(init.body as string)); }
/** Answer every question id the request carries (ids are stable across batches: document_<global index>). */
function answerBody(init: RequestInit, score: (doc: string) => number, tokens = 100, model = 'jev-1.13.0'): Response {
  const body = JSON.parse(init.body as string);
  return new Response(JSON.stringify({
    model,
    answers: Object.fromEntries(Object.entries(body.questions).map(([id, q]: [string, any]) => [id, { type: 'score', score: score(q.instructions.candidate), confidence: 0.9 }])),
    usage: { input_tokens: tokens, output_tokens: 12 },
  }), { headers: { 'content-type': 'application/json' } });
}
const batchCount = (documents: string[]) => planBatches(estimateContextTokens({ query: 'q' }), rerankQuestions(documents).map((q) => estimateContextTokens(toWireQuestion(q)))).length;

afterEach(() => {
  __setRerankTransportForTests(null);
  resetGateway();
});

describe('System One reranker (S1)', () => {
  test('registers decide + reranker touchpoints only and leaves the default reranker unchanged', () => {
    const recipe = getRecipe('typesafe')!;
    expect(recipe.auth_env?.required).toEqual(['TYPESAFE_API_KEY']);
    expect(recipe.touchpoints.reranker?.models).toContain('jev-1.13.0');
    expect(recipe.touchpoints.decide?.default_model).toBe('jev-1.13.0');
    expect(recipe.touchpoints.embedding).toBeUndefined();
    expect(recipe.touchpoints.chat).toBeUndefined();
    expect(DEFAULT_RERANKER_MODEL).toBe('voyage:rerank-3-lite');
    expect(rerankerReadiness(MODEL, {}).ready).toBe(false);
    expect(rerankerReadiness(MODEL, { TYPESAFE_API_KEY: 'sk-test' }).ready).toBe(true);
    expect(rerankerReadiness(MODEL, { JEV_TYPESAFE_API_KEY: 'sk-test' }).ready).toBe(true);
  });

  test('sends native System One score questions with bearer auth and normalizes fractional scores', async () => {
    configure();
    __setRerankTransportForTests(async (url, init) => {
      expect(url).toBe('https://api.typesafe.ai/v1/systemone');
      expect(new Headers(init.headers).get('authorization')).toBe('Bearer sk-test-typesafe');
      const body = JSON.parse(init.body as string);
      expect(body.model).toBe('jev-1.13.0');
      expect(body.state).toEqual({ query: 'Which launch decision was made?' });
      expect(docsFromBody(body)).toEqual(['routine', 'decision', 'background']);
      expect(body.questions.document_1.instructions.task).toContain('`candidate`');
      expect(body.questions.document_1.type).toBe('score');
      expect(body.questions.document_1.criteria).toHaveLength(4);
      expect(body.top_n).toBeUndefined();
      return response([0, 2.7, 1.2]);
    });
    expect(await rerank({ query: 'Which launch decision was made?', documents: ['routine', 'decision', 'background'], topN: 2 }))
      .toEqual([{ index: 1, relevanceScore: 0.9 }, { index: 2, relevanceScore: 1.2 / 3 }]);
  });

  test('accepts JEV_TYPESAFE_API_KEY as the credential', async () => {
    configure({ JEV_TYPESAFE_API_KEY: 'sk-test-alias' });
    __setRerankTransportForTests(async (_url, init) => {
      expect(new Headers(init.headers).get('authorization')).toBe('Bearer sk-test-alias');
      return response([3]);
    });
    expect(await rerank({ query: 'q', documents: ['d'] })).toEqual([{ index: 0, relevanceScore: 1 }]);
  });

  test('merges batch-local indices globally and applies topN after all batches', async () => {
    configure();
    const documents = longDocuments(66);
    let calls = 0;
    __setRerankTransportForTests(async (_url, init) => {
      calls++;
      return answerBody(init, (doc) => (doc.startsWith('candidate 65:') ? 3 : 1));
    });
    const result = await rerank({ query: 'q', documents, topN: 3 });
    expect(calls).toBe(batchCount(documents));
    expect(calls).toBeGreaterThan(1);
    expect(result.map((r) => r.index)).toEqual([65, 0, 1]);
  });

  test('packs fifty short candidates into one request', () => {
    const documents = Array.from({ length: 50 }, () => 'Useful background evidence. '.repeat(35));
    expect(batchCount(documents)).toBe(1);
  });

  test('reports the resolved model and rubric semantics through onMeta', async () => {
    configure();
    __setRerankTransportForTests(async () => response([2], 77, 'jev-1.13.0'));
    let meta: RerankMeta | undefined;
    await rerank({ query: 'q', documents: ['d'], onMeta: (m) => { meta = m; } });
    expect(meta).toMatchObject({ model_resolved: 'jev-1.13.0', score_semantics: 'rubric', input_tokens: 77, batches: 1 });
  });

  test('mixed resolved models across the batches of one call fail it open (the #5178 review gap)', async () => {
    configure();
    let calls = 0;
    __setRerankTransportForTests(async (_url, init) => {
      calls++;
      return answerBody(init, () => 1, 100, calls === 1 ? 'jev-1.13.0' : 'jev-1.14.0');
    });
    await expect(rerank({ model: 'typesafe:jev-latest', query: 'q', documents: longDocuments(66) }))
      .rejects.toMatchObject({ reason: 'unknown', message: expect.stringContaining('mixed_model') });
  });

  test('a response without a resolved model is malformed', async () => {
    configure();
    __setRerankTransportForTests(async () => new Response(JSON.stringify({ answers: { document_0: { type: 'score', score: 3 } } })));
    await expect(rerank({ query: 'q', documents: ['d'] })).rejects.toBeInstanceOf(RerankError);
  });

  for (const answer of [undefined, { type: 'choice', choice: 'yes' }, { type: 'score', score: '3' },
    { type: 'score', score: -1 }, { type: 'score', score: 3.1 }, { type: 'score', score: Infinity }]) {
    test(`rejects missing or invalid scores (${JSON.stringify(answer)})`, () => {
      expect(() => parseTypeSafeResponse({ model: 'jev-1.13.0', answers: { document_0: answer } }, rerankQuestions(['d']))).toThrow();
    });
  }

  test('digit-dense and Unicode input splits without dropping or truncating evidence', async () => {
    configure();
    const documents = Array.from({ length: 10 }, (_, i) => `${i}: ${'1234567890漢字🧠'.repeat(900)}`);
    const seen: string[] = [];
    __setRerankTransportForTests(async (_url, init) => { seen.push(...docsFor(init)); return answerBody(init, () => 1); });
    await rerank({ query: 'q', documents });
    expect(batchCount(documents)).toBeGreaterThan(1);
    expect([...seen].sort()).toEqual([...documents].sort());
  });

  test('oversized single pairs fail before any HTTP request', async () => {
    configure();
    let calls = 0;
    __setRerankTransportForTests(async () => { calls++; return response([3]); });
    await expect(rerank({ query: 'word '.repeat(20_000), documents: ['d'] })).rejects.toMatchObject({ reason: 'payload_too_large' });
    expect(calls).toBe(0);
  });

  test('missing key skips transport rather than changing providers', async () => {
    configure({});
    let calls = 0;
    __setRerankTransportForTests(async () => { calls++; return response([3]); });
    await expect(rerank({ query: 'q', documents: ['d'] })).rejects.toMatchObject({ reason: 'no_key' });
    expect(calls).toBe(0);
  });

  for (const [status, reason] of [[401, 'auth'], [429, 'rate_limit'], [529, 'network']] as const) {
    test(`classifies HTTP ${status} without leaking echoed evidence`, async () => {
      configure();
      __setRerankTransportForTests(async () => new Response('private-source-marker', { status }));
      try { await rerank({ query: 'q', documents: ['d'] }); throw new Error('expected failure'); }
      catch (err) {
        expect(err).toBeInstanceOf(RerankError);
        expect((err as RerankError).reason).toBe(reason);
        expect((err as Error).message).not.toContain('private-source-marker');
      }
    });
  }

  test('429 retry-after is honored only when it fits the deadline', async () => {
    configure();
    let calls = 0;
    __setRerankTransportForTests(async () => {
      calls++;
      return calls === 1 ? new Response('', { status: 429, headers: { 'retry-after': '0' } }) : response([3]);
    });
    expect(await rerank({ query: 'q', documents: ['d'] })).toEqual([{ index: 0, relevanceScore: 1 }]);
    expect(calls).toBe(2);
    calls = 0;
    __setRerankTransportForTests(async () => { calls++; return new Response('', { status: 429, headers: { 'retry-after': '60' } }); });
    await expect(rerank({ query: 'q', documents: ['d'], timeoutMs: 1000 })).rejects.toMatchObject({ reason: 'rate_limit' });
    expect(calls).toBe(1);
  });

  test('malformed JSON does not echo private response contents', async () => {
    configure();
    __setRerankTransportForTests(async () => new Response('private-source-marker'));
    await expect(rerank({ query: 'q', documents: ['d'] })).rejects.toMatchObject({ reason: 'unknown', message: 'TypeSafe rerank: malformed JSON' });
  });

  test('overall timeout aborts started calls and does not start later batches', async () => {
    configure();
    let calls = 0;
    __setRerankTransportForTests(async (_url, init) => {
      calls++;
      return new Promise((_resolve, reject) => {
        init.signal!.addEventListener('abort', () => reject(init.signal!.reason), { once: true });
      });
    });
    await expect(rerank({ query: 'q', documents: queuedDocuments(), timeoutMs: 15 })).rejects.toMatchObject({ reason: 'timeout' });
    expect(calls).toBe(16);
  });

  test('an already-aborted caller sends no request', async () => {
    configure();
    let calls = 0;
    __setRerankTransportForTests(async () => { calls++; return response([3]); });
    const ctrl = new AbortController(); ctrl.abort();
    await expect(rerank({ query: 'q', documents: ['d'], signal: ctrl.signal })).rejects.toBeInstanceOf(RerankError);
    expect(calls).toBe(0);
  });

  test('records reported input usage from every batch as one rerank call', async () => {
    configure();
    const dir = mkdtempSync(join(tmpdir(), 'gbrain-typesafe-budget-'));
    try {
      const tracker = new BudgetTracker({ label: 'test', maxCostUsd: 1, auditPath: join(dir, 'audit.jsonl') });
      __setRerankTransportForTests(async (_url, init) => answerBody(init, () => 1, 321));
      await withBudgetTracker(tracker, () => rerank({ query: 'q', documents: longDocuments(66) }));
      expect(tracker.snapshot().cumulativeCostUsd).toBeCloseTo((321 * batchCount(longDocuments(66))) * 0.042 / 1_000_000, 12);
      expect(tracker.snapshot().callsRecorded).toBe(1);
    } finally { rmSync(dir, { recursive: true, force: true }); }
  });

  test('a failed batch settles usage from all started calls and stops queued batches', async () => {
    configure();
    const dir = mkdtempSync(join(tmpdir(), 'gbrain-typesafe-partial-'));
    try {
      const tracker = new BudgetTracker({ label: 'test', maxCostUsd: 1, auditPath: join(dir, 'audit.jsonl') });
      let calls = 0;
      __setRerankTransportForTests(async (_url, init) => {
        calls++;
        return calls === 1 ? response([], 100) : answerBody(init, () => 2);
      });
      await expect(withBudgetTracker(tracker, () => rerank({ query: 'q', documents: queuedDocuments() }))).rejects.toMatchObject({ reason: 'unknown' });
      expect(calls).toBe(16);
      expect(tracker.snapshot().cumulativeCostUsd).toBeCloseTo(1600 * 0.042 / 1_000_000, 12);
      expect(tracker.snapshot().callsRecorded).toBe(1);
    } finally { rmSync(dir, { recursive: true, force: true }); }
  });

  test('budget denial happens before transport', async () => {
    configure();
    const dir = mkdtempSync(join(tmpdir(), 'gbrain-typesafe-denied-'));
    try {
      let calls = 0;
      const tracker = new BudgetTracker({ label: 'test', maxCostUsd: 0, auditPath: join(dir, 'audit.jsonl') });
      __setRerankTransportForTests(async () => { calls++; return response([3]); });
      await expect(withBudgetTracker(tracker, () => rerank({ query: 'q', documents: ['d'] }))).rejects.toMatchObject({ tag: 'BUDGET_EXHAUSTED' });
      expect(calls).toBe(0);
    } finally { rmSync(dir, { recursive: true, force: true }); }
  });

  test('one failed batch preserves all search results and original scores (fail open)', async () => {
    configure();
    const results = Array.from({ length: 40 }, (_, i) => ({ slug: `notes/candidate-${i}`, chunk_text: `d${i}`, title: 'Candidate', score: 40 - i })) as SearchResult[];
    const before = structuredClone(results);
    __setRerankTransportForTests(async (_url, init) => docsFor(init).includes('d32')
      ? new Response('{"answers":{}}', { headers: { 'content-type': 'application/json' } })
      : answerBody(init, () => 3));
    expect(await applyReranker('q', results, { enabled: true, topNIn: 34, topNOut: null, model: MODEL })).toEqual(before);
  });

  test('stamps rubric semantics so autocut and CRAG do not consume the level score', async () => {
    configure();
    const results = [0, 1].map((i) => ({ slug: `notes/c${i}`, chunk_text: `d${i}`, title: 'C', score: 2 - i })) as SearchResult[];
    __setRerankTransportForTests(async () => response([1, 3]));
    const out = await applyReranker('q', results, { enabled: true, topNIn: 2, topNOut: null, model: MODEL });
    expect(out.map((r) => r.slug)).toEqual(['notes/c1', 'notes/c0']);
    expect(out.every((r) => r.rerank_score_kind === 'rubric')).toBe(true);
  });

  test('keeps the ordinary Voyage wire unchanged when a TypeSafe key also exists', async () => {
    configureGateway({ reranker_model: 'voyage:rerank-2.5', env: { VOYAGE_API_KEY: 'sk-test-voyage', TYPESAFE_API_KEY: 'sk-test-typesafe' } });
    __setRerankTransportForTests(async (url, init) => {
      expect(url).toBe('https://api.voyageai.com/v1/rerank');
      expect(JSON.parse(init.body as string)).toEqual({ model: 'rerank-2.5', query: 'q', documents: ['d'], top_k: 1 });
      return new Response('{"data":[{"index":0,"relevance_score":0.8}]}');
    });
    const out = await rerank({ query: 'q', documents: ['d'], topN: 1 });
    expect(out).toEqual([{ index: 0, relevanceScore: 0.8 }]);
  });
});