Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* The embedding request plan shared by `embed()` and every caller that must
* predict it before spending (#5680: the migration worst-case authorization).
* One home for per-input truncation, the token-budget pre-split, the hard item
* cap, and the per-request input ceiling a guard reserves before dispatch.
*/
import type { Recipe } from './types.ts';
import { resolveRecipe } from './model-resolver.ts';
import { truncateUtf8 } from '../text-safe.ts';
import { sendableEmbeddingInputs } from './embedding-guard.ts';
/** Per-input character cap applied before any embedding request. */
export const EMBED_MAX_CHARS = 8000;
/** Default chars-per-token when a recipe omits it. Matches OpenAI tiktoken on English. */
export const DEFAULT_CHARS_PER_TOKEN = 4;
/** Default safety factor when a recipe omits it. */
export const DEFAULT_SAFETY_FACTOR = 0.8;
/**
* #3875: default per-call item cap for `no_batch_cap` recipes (Ollama,
* LiteLLM proxy). These recipes declare no static token/item cap because the
* backend's capacity is user-launched — but the per-SDK-call
* AI_EMBED_TIMEOUT_MS (60s default) then bounded a whole FILE's chunks in one
* request. A slow local model (CPU Ollama) embedding a large file timed out
* deterministically and every retry re-sent the same oversized batch. Capping
* items per sub-batch makes the 60s timeout a per-BATCH budget: 16 chunks per
* call finishes comfortably even on CPU-bound local models, and a genuinely
* wedged provider still surfaces the timeout loudly on the first sub-batch.
* An explicit `max_batch_items` on the recipe always wins over this default.
*
* @internal exported for tests; not part of the public gateway API.
*/
export const NO_BATCH_CAP_SUB_BATCH_ITEMS = 16;
export function truncateEmbedInputs(texts: ReadonlyArray<string | null | undefined>): string[] {
return texts.map(t => truncateUtf8(t ?? '', EMBED_MAX_CHARS));
}
/**
* Split texts into sub-batches that stay under the provided budget. Pure;
* no module state. Exported for the adaptive-embed-batch test suite.
*
* @param texts - The texts to partition. Each text counts as
* `Math.ceil(text.length / charsPerToken)` tokens for budget purposes.
* @param budgetTokens - The token ceiling for each sub-batch. Caller is
* responsible for applying any safety-factor shrink before passing in.
* @param charsPerToken - Provider-specific character density. Defaults to
* `DEFAULT_CHARS_PER_TOKEN` (4) when omitted, matching OpenAI tiktoken.
*
* @internal exported for tests; not part of the public gateway API.
*/
export function splitByTokenBudget(
texts: string[],
budgetTokens: number,
charsPerToken: number = DEFAULT_CHARS_PER_TOKEN,
): string[][] {
const ratio = charsPerToken > 0 ? charsPerToken : DEFAULT_CHARS_PER_TOKEN;
const batches: string[][] = [];
let current: string[] = [];
let currentTokens = 0;
for (const text of texts) {
const estTokens = Math.ceil(text.length / ratio);
if (current.length > 0 && currentTokens + estTokens > budgetTokens) {
batches.push(current);
current = [];
currentTokens = 0;
}
current.push(text);
currentTokens += estTokens;
}
if (current.length > 0) batches.push(current);
return batches;
}
/**
* Split a batch into sub-batches of at most `maxItems` inputs. Enforces a
* hard COUNT cap that the token-budget split can't (many tiny inputs fit
* under any token budget). Used for endpoints like llama.cpp's llama-server
* that reject requests exceeding their launch batch size.
*
* @internal exported for tests; not part of the public gateway API.
*/
export function capBatchItems(texts: string[], maxItems: number): string[][] {
if (maxItems <= 0 || texts.length <= maxItems) return [texts];
const batches: string[][] = [];
for (let i = 0; i < texts.length; i += maxItems) {
batches.push(texts.slice(i, i + maxItems));
}
return batches;
}
/**
* The provider requests `embed()` dispatches for already-truncated texts.
* Pre-split is gated on `max_batch_tokens` (or the operator's
* GBRAIN_EMBED_MAX_BATCH_TOKENS cap, #3622, for recipes that ship without
* one); recipes without either ride one request. The hard COUNT cap
* (e.g. llama-server's 32) then re-splits any oversized batch, and
* `no_batch_cap` recipes default to NO_BATCH_CAP_SUB_BATCH_ITEMS (#3875).
*/
export function planEmbedRequests(texts: string[], recipe: Recipe, safetyFactor: number, envMaxBatchTokens?: number): string[][] {
const embedding = recipe.touchpoints?.embedding;
const maxBatchTokens = embedding?.max_batch_tokens ?? envMaxBatchTokens;
const tokenBatches = maxBatchTokens
? splitByTokenBudget(texts, Math.floor(maxBatchTokens * safetyFactor), embedding?.chars_per_token ?? DEFAULT_CHARS_PER_TOKEN)
: [texts];
const maxBatchItems = embedding?.max_batch_items ?? (embedding?.no_batch_cap === true ? NO_BATCH_CAP_SUB_BATCH_ITEMS : undefined);
return maxBatchItems ? tokenBatches.flatMap(b => capBatchItems(b, maxBatchItems)) : tokenBatches;
}
/**
* #5680: the most input tokens one request can bill. Every token covers at
* least one UTF-8 byte of its input, and a declared per-input limit caps each
* text, so the ceiling follows the request's own texts instead of the
* recipe's whole batch cap. Additive, so it bounds any later split or retry
* of the same texts.
*/
export function embedRequestMaxInputTokens(texts: string[], recipe: Recipe, modelId: string): number {
const perInput = recipe.touchpoints?.embedding?.max_input_tokens?.[modelId] ?? Infinity;
return texts.reduce((sum, text) => sum + Math.min(Math.max(1, Buffer.byteLength(text, 'utf8')), perInput), 0);
}
/** Per-request input ceilings `embed()` would reserve for these texts on `modelStr`, at the declared safety factor (empty inputs are never sent, #4616). */
export function embedRequestCeilings(texts: ReadonlyArray<string>, modelStr: string, envMaxBatchTokens?: number): number[] {
const { parsed, recipe } = resolveRecipe(modelStr);
const safety = recipe.touchpoints?.embedding?.safety_factor ?? DEFAULT_SAFETY_FACTOR;
const truncated = truncateEmbedInputs(texts);
const sent = sendableEmbeddingInputs(truncated);
if (!sent.length) return [];
return planEmbedRequests(sent.map(i => truncated[i]!), recipe, safety, envMaxBatchTokens)
.map(batch => embedRequestMaxInputTokens(batch, recipe, parsed.modelId));
}
/** The most input tokens one rerank request can bill: the query is sent beside every document. */
export function rerankRequestMaxInputTokens(query: string, documents: ReadonlyArray<string>): number {
return documents.reduce((sum, document) => sum + Buffer.byteLength(query, 'utf8') + Buffer.byteLength(document, 'utf8'), 0);
}