Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.0 KB
Raw Blame
/**
 * Embed retry/backoff primitives + the title-tier restamp helper, peeled from
 * `src/commands/embed.ts` (mw2 layering fix).
 *
 * Why a core module: `src/core/import-file.ts` and `src/core/embed-stale.ts`
 * consume `embedBatchWithBackoff` / `restampIfDemotedToTitleTier`. When these
 * lived in the commands layer, every import-file consumer (sync, ops/pages,
 * operations.ts, MCP) eagerly loaded the embed COMMAND module, and any future
 * value import of operations.ts from embed.ts's closure would complete a real
 * ESM cycle — the exact class that produced a prior release's module-cycle
 * break. `src/commands/embed.ts` re-exports everything here (façade rule), so
 * historical import sites and tests never chase the peel.
 */

import { embedBatch } from './embedding.ts';
import { isSingleInputOverLengthError } from './embed-overlength-classify.ts';
import { isEmbeddingZeroNormError, type EmbeddingZeroNormError } from './ai/embedding-guard.ts';
import { serr } from './console-prefix.ts';
import { noteEmbedApiResponse } from './embed-stall.ts';
import { titleTierCorpusGeneration } from './contextual-retrieval-service.ts';
import type { BrainEngine } from './engine.ts';
import type { Page } from './types.ts';

/**
 * #3507 — after a plain re-embed fully re-embedded a `per_chunk_synopsis`
 * page at the title-only tier (see wrapChunkTextsForStoredMode), restamp the
 * page's CR state to 'title' so `contextual_retrieval_mode` keeps describing
 * the vectors actually in the column. The reindex sweep restores the synopsis
 * tier later. No-op for every other mode.
 */
export async function restampIfDemotedToTitleTier(
  engine: BrainEngine,
  page: Pick<Page, 'contextual_retrieval_mode' | 'knowledge_revision'> | null | undefined,
  slug: string,
  sourceId: string,
): Promise<void> {
  if (page?.contextual_retrieval_mode !== 'per_chunk_synopsis' || !page.knowledge_revision) return;
  await engine.transaction(async tx => {
    await tx.lockPageKeys([{ sourceId, slug }]);
    const current = await tx.readPageSnapshot(slug, { sourceId });
    if (current?.revision !== page.knowledge_revision) return;
    await tx.updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration());
  });
}

/**
 * v0.33.3: rate-limit-aware embedBatch wrapper.
 * #3966: also retries transient gateway errors (502/503/504) that NIM and
 * similar providers emit under sustained bulk load.
 *
 * The OpenAI SDK has built-in retry with exponential backoff, but its
 * backoff window (max ~4s) is too short for TPM (tokens-per-minute)
 * rate limits on large pages (~90K tokens).  This wrapper catches
 * 429-shaped and gateway-overload errors, parses the retry delay from
 * the error message (e.g. "Please try again in 248ms"), and sleeps before retrying.
 *
 * v0.33.4 hardening (codex + re-review findings):
 *   - D4: detect 429 via the wrapped error's `cause.status` (the gateway's
 *     normalizeAIError stores the original error there). Bare `e.status`
 *     never fires against an `AITransientError` wrap. Message-match stays
 *     as a fallback.
 *   - D4a: pass `maxRetries: 0` through `embedBatch` so the AI SDK's
 *     default 2-retry stack doesn't multiply this wrapper's 5 attempts.
 *   - D2: jitter the parsed delay ±30% so 20 concurrent workers don't
 *     resynchronize on the next 429 wave.
 *   - D3a/D8: when an external AbortSignal fires (wall-clock budget), the
 *     sleep wakes up early AND the abortSignal is threaded into the gateway
 *     embed call so an in-flight HTTP request cancels too.
 *
 * Up to MAX_RATE_LIMIT_RETRIES attempts with the parsed (jittered) delay
 * (or a 60s fallback when the message can't be parsed).
 *
 * @internal Exported for unit tests; not part of the public surface.
 */
export const MAX_RATE_LIMIT_RETRIES = 5;
export const RATE_LIMIT_FALLBACK_MS = 60_000;
export const RATE_LIMIT_PAD_MS = 500;
export const RATE_LIMIT_JITTER = 0.3;

export interface EmbedBatchWithBackoffOpts {
  abortSignal?: AbortSignal;
}

/**
 * Walk the cause chain looking for a 429 status. The current
 * `normalizeAIError` wraps once into `AITransientError` with `cause = original`,
 * so one level is sufficient — but iterate to handle future wrap layers
 * defensively (max 5 levels to bound a malformed cyclic chain).
 *
 * @internal exported for unit tests.
 */
export function detect429FromCause(e: unknown): boolean {
  let cur: unknown = e;
  for (let depth = 0; depth < 5 && cur !== undefined && cur !== null; depth++) {
    const obj = cur as { status?: unknown; statusCode?: unknown; cause?: unknown };
    if (obj.status === 429 || obj.statusCode === 429) return true;
    cur = obj.cause;
  }
  return false;
}

/** Gateway overload statuses retried with the same backoff as 429 (#3966). */
const RETRIABLE_GATEWAY_STATUSES = new Set([502, 503, 504]);

/**
 * Walk the cause chain looking for 502/503/504. Same depth bound as
 * detect429FromCause — one normalizeAIError wrap is typical.
 *
 * @internal exported for unit tests.
 */
export function detectGatewayErrorFromCause(e: unknown): boolean {
  let cur: unknown = e;
  for (let depth = 0; depth < 5 && cur !== undefined && cur !== null; depth++) {
    const obj = cur as { status?: unknown; statusCode?: unknown; cause?: unknown };
    const status = obj.status ?? obj.statusCode;
    if (typeof status === 'number' && RETRIABLE_GATEWAY_STATUSES.has(status)) return true;
    cur = obj.cause;
  }
  return false;
}

/**
 * Parse a Retry-After hint out of an OpenAI-style 429 message. Falls back
 * to `RATE_LIMIT_FALLBACK_MS` when the message can't be parsed. Adds
 * `RATE_LIMIT_PAD_MS` padding and `RATE_LIMIT_JITTER` randomization so
 * concurrent workers don't resynchronize.
 *
 * @internal exported for unit tests.
 */
export function parseRetryDelayMs(msg: string, rng: () => number = Math.random): number {
  let delayMs = RATE_LIMIT_FALLBACK_MS;
  const msMatch = msg.match(/try again in (\d+)ms/i);
  const secMatch = msg.match(/try again in ([\d.]+)s/i);
  if (msMatch) delayMs = parseInt(msMatch[1], 10) + RATE_LIMIT_PAD_MS;
  else if (secMatch) delayMs = Math.ceil(parseFloat(secMatch[1]) * 1000) + RATE_LIMIT_PAD_MS;
  // D2: ±30% jitter to decorrelate the herd of 20 workers.
  const jitterFactor = 1 + (rng() * 2 - 1) * RATE_LIMIT_JITTER;
  return Math.max(1, Math.floor(delayMs * jitterFactor));
}

/**
 * #3796 — attempt-scaled floor for 429 waits. Providers with ROLLING
 * per-minute token budgets (TPM) return tiny Retry-After hints ("try again
 * in 132ms") that are honest about the next REQUEST slot but not about the
 * token budget: five hint-sized waits burn every retry inside the same TPM
 * minute and the batch fails despite the limit being about to clear.
 * Escalating floors guarantee the retry ladder spans the rolling window
 * (sum ≈ 120s ≫ 60s even at -30% jitter) while a LARGER provider hint still
 * wins (a 4-minute Retry-After must be honored, not floored down).
 */
export const RATE_LIMIT_ATTEMPT_FLOOR_MS = [1_000, 4_000, 10_000, 30_000, 75_000] as const;

let _rateLimitFloorsMs: readonly number[] = RATE_LIMIT_ATTEMPT_FLOOR_MS;

/**
 * Test seam: shrink the #3796 floors so sustained-429 exhaustion tests don't
 * spend ~2min of real wall clock (the production ladder's whole point).
 * Pass null to restore. @internal
 */
export function _setRateLimitFloorsForTests(floors: readonly number[] | null): void {
  _rateLimitFloorsMs = floors ?? RATE_LIMIT_ATTEMPT_FLOOR_MS;
}

/**
 * #3796 — the effective 429 wait: max(jittered provider hint, jittered
 * attempt-scaled floor). Keeps parseRetryDelayMs's contract (hint + pad +
 * jitter) intact for its other consumers/tests; the floor gets its own
 * jitter so a floored herd doesn't resynchronize.
 *
 * @internal exported for unit tests.
 */
export function rateLimitDelayMs(
  msg: string,
  attempt: number,
  rng: () => number = Math.random,
): number {
  const hinted = parseRetryDelayMs(msg, rng);
  const floorIdx = Math.min(Math.max(attempt, 0), _rateLimitFloorsMs.length - 1);
  const jitterFactor = 1 + (rng() * 2 - 1) * RATE_LIMIT_JITTER;
  const floored = Math.max(1, Math.floor(_rateLimitFloorsMs[floorIdx] * jitterFactor));
  return Math.max(hinted, floored);
}

/**
 * Sleep for `ms` milliseconds. Resolves early (not rejects) when `signal`
 * fires, so the retry loop's caller can re-check `signal.aborted` and
 * exit cleanly without an unhandled rejection.
 *
 * NOTE: distinct contract from `src/core/retry.ts:abortableSleep`, which
 * REJECTS with RetryAbortError on abort. This one resolves.
 *
 * @internal exported for unit tests.
 */
export function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
  return new Promise((resolve) => {
    if (signal?.aborted) {
      resolve();
      return;
    }
    const timer = setTimeout(() => {
      signal?.removeEventListener('abort', onAbort);
      resolve();
    }, ms);
    const onAbort = () => {
      clearTimeout(timer);
      signal?.removeEventListener('abort', onAbort);
      resolve();
    };
    signal?.addEventListener('abort', onAbort, { once: true });
  });
}

export async function embedBatchWithBackoff(
  texts: string[],
  opts: EmbedBatchWithBackoffOpts = {},
): Promise<Float32Array[]> {
  const signal = opts.abortSignal;
  for (let attempt = 0; attempt <= MAX_RATE_LIMIT_RETRIES; attempt++) {
    if (signal?.aborted) throw new Error('embed budget aborted');
    try {
      // D4a + D8: maxRetries:0 disables the SDK's stacked retries (so this
      // wrapper is the single source of truth) and abortSignal threads
      // through to the gateway so an in-flight HTTP request cancels mid-fetch.
      const out = await embedBatch(texts, { maxRetries: 0, ...(signal && { abortSignal: signal }) });
      // #4599: every SETTLED embed attempt ticks the stall watchdog's
      // liveness clock (T6 — API-response grain, successes here, errors in
      // the catch). Liveness is diagnostic; the stall TRIGGER stays keyed on
      // successful chunk progress in the caller.
      noteEmbedApiResponse();
      return out;
    } catch (e: unknown) {
      noteEmbedApiResponse();
      // If the budget fired we may have been aborted mid-fetch; bubble out.
      // This check is what keeps caller-initiated aborts out of BOTH retry
      // branches below (#3374) — an abort is never reclassified as transient.
      if (signal?.aborted) throw e;
      const msg = e instanceof Error ? e.message : String(e);
      const rateLimitish = isEmbedRetriableError(e);
      // #3374 — transient NETWORK blips (socket timeout / conn reset) get a
      // plain bounded backoff beside the 429/gateway retry-after path.
      const netTransient = !rateLimitish && isTransientNetworkEmbedError(e);
      if ((!rateLimitish && !netTransient) || attempt === MAX_RATE_LIMIT_RETRIES) throw e;

      // #3796: 429s take the attempt-floored wait (rolling-TPM-aware);
      // network blips keep their own bounded exponential ladder.
      const delayMs = rateLimitish ? rateLimitDelayMs(msg, attempt) : transientBackoffMs(attempt);
      // One label for every retriable class — 429, gateway and network blips share the loop.
      serr(`  [embed-retry] attempt ${attempt + 1}/${MAX_RATE_LIMIT_RETRIES}, waiting ${delayMs}ms...`);
      await abortableSleep(delayMs, signal);
    }
  }
  // Unreachable, but TypeScript needs it.
  return embedBatch(texts);
}

/**
 * #4616: embedBatchWithBackoff that keeps a partially refused batch. Usable
 * vectors come back aligned with `texts` (null where the gateway refused an
 * item) together with the refusal; a batch with nothing usable still throws.
 */
export async function embedBatchKeepingUsable(texts: string[], opts: EmbedBatchWithBackoffOpts = {}):
  Promise<{ vectors: (Float32Array | null)[]; refused?: EmbeddingZeroNormError }> {
  try {
    return { vectors: await embedBatchWithBackoff(texts, opts) };
  } catch (e) {
    if (!isEmbeddingZeroNormError(e) || e.failures.length === texts.length) throw e;
    return { vectors: e.vectors, refused: e };
  }
}

/**
 * Retriable embed errors: 429 rate limits plus transient gateway overload
 * (502/503/504). Shared by embedBatchWithBackoff (retry decision) and
 * embedPageTexts (fan-out decision). D4: structured detection first
 * (gateway-wrapped errors via cause chain); message-match as fallback for
 * providers whose wrappers strip `cause.status`.
 */
export function isEmbedRetriableError(e: unknown): boolean {
  if (isEmbeddingZeroNormError(e)) return false; // #4616: terminal per item, never retried
  if (isSingleInputOverLengthError(e)) return false; // #6772: the same input fails again, even behind a 5xx
  const msg = e instanceof Error ? e.message : String(e);
  return (
    detect429FromCause(e) ||
    detectGatewayErrorFromCause(e) ||
    /rate.?limit|429/i.test(msg) ||
    /bad gateway|502|503|504|service unavailable|gateway timeout/i.test(msg)
  );
}

/** #3374 — plain bounded backoff knobs for transient network errors. */
export const TRANSIENT_NET_BASE_MS = 1_000;
export const TRANSIENT_NET_MAX_MS = 15_000;

/**
 * #3374 — plain bounded exponential backoff for transient network errors.
 * There is no retry-after header to parse on a conn-reset, and the 429
 * fallback (60s) is wildly oversized for a socket blip: base × 2^attempt,
 * capped, with the same ±30% jitter as the rate-limit path.
 *
 * @internal exported for unit tests.
 */
export function transientBackoffMs(attempt: number, rng: () => number = Math.random): number {
  const raw = Math.min(TRANSIENT_NET_BASE_MS * 2 ** attempt, TRANSIENT_NET_MAX_MS);
  const jitterFactor = 1 + (rng() * 2 - 1) * RATE_LIMIT_JITTER;
  return Math.max(1, Math.floor(raw * jitterFactor));
}

/**
 * #3374 — transient NETWORK failures: socket timeouts, connection resets,
 * DNS blips (including Bun's `DNS_ETIMEOUT` code and `ETIMEOUT` getaddrinfo
 * message). Structured detection first (error `code` / TimeoutError name
 * through the cause chain, matching statusFromCause's walk), message-match
 * fallback for wrappers that strip the code. Deliberately does NOT match
 * caller-initiated aborts: the retry loop re-checks `signal.aborted` BEFORE
 * classification, and nothing abort-shaped appears in these patterns.
 *
 * @internal exported for unit tests.
 */
export function isTransientNetworkEmbedError(e: unknown): boolean {
  if (isEmbeddingZeroNormError(e)) return false;
  if (isSingleInputOverLengthError(e)) return false;
  const TRANSIENT_CODES = /^(DNS_ETIMEOUT|ETIMEOUT|ETIMEDOUT|ESOCKETTIMEDOUT|ECONNRESET|EPIPE|ECONNABORTED|EAI_AGAIN|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET)$/;
  let cur: unknown = e;
  for (let depth = 0; depth < 5 && cur !== undefined && cur !== null; depth++) {
    const obj = cur as { code?: unknown; name?: unknown; cause?: unknown };
    if (typeof obj.code === 'string' && TRANSIENT_CODES.test(obj.code)) return true;
    if (obj.name === 'TimeoutError') return true;
    cur = obj.cause;
  }
  const msg = e instanceof Error ? e.message : String(e);
  return /\b(DNS_ETIMEOUT|ETIMEOUT|ETIMEDOUT|ESOCKETTIMEDOUT|ECONNRESET|EPIPE|EAI_AGAIN)\b|socket hang up|fetch failed|connect(ion)? timeout|connection (reset|closed)|network (error|timeout)|request timed out|timed out/i.test(msg);
}