Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

16.7 KB
Raw Blame
import { createHash } from "node:crypto";

import { getSettings as defaultGetSettings } from "@/lib/db/settings";
import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
import {
  resolveVideoBridgeRuntimeSettings,
  resolveVisionBridgeRuntimeSettings,
  resolveVideoAudioTranscriptionRuntimeSettings,
  type VideoAnalysisMode,
} from "@/shared/constants/modalityBridgeDefaults";

import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base";
import type { BridgeCacheStore } from "./modalityBridge/bridgeCache";
import {
  extractVideoFocusHint,
  extractVideoParts,
  loadVideoPartBytes,
  replaceVideoParts,
  type DescribeVideoDependencies,
  type DescribedVideo,
  type VideoFusionTelemetry,
  type VideoPart,
} from "./videoBridgeHelpers";
import {
  combineModelIdentities,
  processVideoPart,
  type ProcessVideoPartDeps,
  type VideoAnalysisContext,
} from "./videoBridgePipeline";
import { getSharedVideoResultCacheFor } from "./videoBridgeResultCache";
import { appendVideoFallbackTranscripts } from "./videoBridgeFallbackTranscript";
import { type VisionModelConfig } from "./visionBridgeHelpers";
import { getBestVisionModel } from "./visionBridgeRouter";
import { createVideoSttAdapter, type VideoSttAdapterDependencies } from "./videoBridgeSttAdapter";
import {
  resolveVisionBridgeKeyScope,
  scopedCredentialCheck,
  type VisionBridgeKeyScope,
  type VisionBridgeKeyScopeDeps,
} from "./visionBridgeKeyScope";

export type { VideoAnalysisContext } from "./videoBridgePipeline";

/**
 * One replaced video part whose rendered text carried a transcript cue.
 * `redactedText` is the structured-redaction shadow (see
 * `DescribedVideo.descriptionRedacted`) for that same part — never derived
 * from the model-bound text, so it cannot be bypassed by cue content.
 *
 * #12150 fix round 1 (adversarial review): the downstream log-redaction
 * consumer (`applyVideoBridgeLogRedaction`, chatCore/attemptLogging.ts)
 * matches by CONTENT (`fullText`), not by `messageIndex`/`partIndex`.
 * Between this guardrail's preCall and the eventual log write, other
 * request-mutation stages (system-prompt injection when no existing system
 * message is found, context-relay handoff injection, reasoning-rule body
 * rewrites) can prepend/splice messages, silently invalidating any
 * positional index. `messageIndex`/`partIndex` are kept as advisory/
 * debugging metadata only — never used for matching.
 */
export interface VideoBridgeLogRedactionEntry {
  container: "messages" | "input";
  /** Advisory only (see interface doc) — may be stale by the time the log is written. */
  messageIndex: number;
  /** Advisory only (see interface doc) — may be stale by the time the log is written. */
  partIndex: number;
  /**
   * The exact, unredacted text placed into the replaced part
   * (`descriptions[i]`, identical to what `replaceVideoParts` writes to
   * `content[partIndex].text`). The downstream consumer matches parts by
   * `part.text === fullText`, so it finds the video part wherever a later
   * stage moved it, and never touches a part whose text differs.
   */
  fullText: string;
  redactedText: string;
}

/**
 * #12150 P1 final-review fix: re-anchor each redaction entry's `fullText` from
 * the FINAL pre-call guardrail payload. The video-bridge guardrail runs at
 * priority 7, but the PII masker (10) and credential masker (95) rewrite the
 * SAME chained payload afterward, in place — so by the end of the chain the
 * replaced part's text may differ from what video-bridge recorded, and the log
 * sink's content-match (`part.text === fullText`) would miss (fail open). Chain
 * guardrails only rewrite text in place — they never splice the message array —
 * so the advisory `(container, messageIndex, partIndex)` still resolves inside
 * the finished chain payload; reading the part text there yields the true
 * post-chain text the log sink will see. Falls back to the original `fullText`
 * when the index no longer resolves. Returns new entries; never mutates the
 * shared guardrail `meta` array. `redactedText` is unchanged (it is rendered
 * from the structured cues, independent of any masker rewrite).
 */
export function reanchorVideoBridgeRedaction(
  entries: readonly VideoBridgeLogRedactionEntry[],
  finalBody: unknown
): VideoBridgeLogRedactionEntry[] {
  const body = finalBody as Record<string, unknown> | null | undefined;
  return entries.map((entry) => {
    const container = body?.[entry.container];
    if (!Array.isArray(container)) return { ...entry };
    const message = container[entry.messageIndex] as { content?: unknown } | undefined;
    const content = message?.content;
    if (!Array.isArray(content)) return { ...entry };
    const part = content[entry.partIndex] as { text?: unknown } | undefined;
    if (!part || typeof part.text !== "string") return { ...entry };
    return { ...entry, fullText: part.text };
  });
}

type VideoBridgeBody = {
  model?: string;
  messages?: Array<{ role?: string; content?: unknown }>;
  input?: Array<{ role?: string; content?: unknown }>;
  [key: string]: unknown;
};

export interface VideoBridgeDependencies extends VideoSttAdapterDependencies {
  getSettings?: () => Promise<Record<string, unknown>>;
  getCapabilities?: (model: string) => { supportsVideo: boolean | null };
  describePart?: (part: VideoPart, analysis: VideoAnalysisContext) => Promise<DescribedVideo>;
  extractFrames?: DescribeVideoDependencies["extractFrames"];
  fetchRemote?: DescribeVideoDependencies["fetchRemote"];
  resultCache?: BridgeCacheStore;
  selectVisionModel?: (fixedModel?: string) => Promise<string | null>;
  /** Base credential check the caller API-key scope narrows (GHSA-3w2r). */
  hasUsableCredentials?: (model: string) => Promise<boolean | null>;
  /** Seams for the caller API-key scope (GHSA-3w2r). */
  keyScope?: VisionBridgeKeyScopeDeps;
  callVisionModel?: (
    imageDataUri: string,
    config: VisionModelConfig,
    apiKey?: string
  ) => Promise<string>;
}

/**
 * Frame vision model for a key-scoped request: picked only among the models the
 * key may use, in the key's selection-cache partition. A virtual `auto` /
 * `auto/*` model resolves to a concrete member, since the key policy is
 * checked against the model actually called.
 */
function selectScopedVideoVisionModel(
  fixedModel: string | undefined,
  keyScope: VisionBridgeKeyScope,
  hasUsableCredentials: (model: string) => Promise<boolean | null>
): Promise<string | null> {
  const concrete =
    fixedModel === "auto" || fixedModel?.startsWith("auto/") ? undefined : fixedModel;
  return getBestVisionModel(
    { fixedModel: concrete, cacheScope: keyScope.cacheScope },
    { hasUsableCredentials }
  );
}

function resolveVideoAnalysisContext(
  body: VideoBridgeBody,
  requestedAnalysisMode: VideoAnalysisMode
): VideoAnalysisContext {
  const focusHint = requestedAnalysisMode === "focused" ? extractVideoFocusHint(body) : undefined;
  return {
    analysisMode: focusHint ? "focused" : "full",
    ...(focusHint ? { focusHint } : {}),
    focusHintFingerprint: focusHint ? createHash("sha256").update(focusHint).digest("hex") : null,
    requestedAnalysisMode,
  };
}

export class VideoBridgeGuardrail extends BaseGuardrail {
  name = "video-bridge";
  priority = 7;

  private readonly deps: VideoBridgeDependencies;

  constructor(options?: { enabled?: boolean; deps?: VideoBridgeDependencies }) {
    super("video-bridge", { priority: 7, enabled: options?.enabled });
    this.deps = options?.deps ?? {};
  }

  async preCall(payload: unknown, context: GuardrailContext): Promise<GuardrailResult<unknown>> {
    if (!this.enabled || context.disabledGuardrails?.includes("video-bridge")) {
      return { block: false };
    }

    if (context.signal?.aborted) throw new Error("Video Bridge processing was aborted");

    const body = payload as VideoBridgeBody;
    const model = context.model || body.model;
    if (!model) return { block: false };

    const getSettings = this.deps.getSettings ?? defaultGetSettings;
    let persisted: Record<string, unknown> = {};
    try {
      persisted = await getSettings();
    } catch {
      // Early boot can run before the settings database is ready; defaults are safe.
    }
    const runtime = resolveVideoBridgeRuntimeSettings(persisted);
    if (!runtime.enabled) return { block: false };

    const parts = extractVideoParts(body);
    if (parts.length === 0) return { block: false };

    const capabilities = (this.deps.getCapabilities ?? getResolvedModelCapabilities)(model);
    if (capabilities.supportsVideo === true) return { block: false };

    // Raw cue fields stay sensitive even when describing the video fails or
    // maxVideos leaves it untouched. Retention must not depend on replacement.
    const rawTranscriptObserved = parts.some(
      (part) => part.transcript !== undefined || part.audioTranscript !== undefined
    );

    const analysis = resolveVideoAnalysisContext(body, runtime.analysisMode);
    const visionRuntime = resolveVisionBridgeRuntimeSettings(persisted);
    const configuredModel = runtime.model.trim() || visionRuntime.model.trim();
    const routingPlanModel = configuredModel || "auto";
    const cache = runtime.cacheEnabled
      ? (this.deps.resultCache ?? getSharedVideoResultCacheFor(runtime))
      : null;
    const successfulModels = new Set<string>();
    // GHSA-3w2r: frames are described (and the soundtrack transcribed) on behalf
    // of the caller's API key, so every vision / STT model and connection the
    // bridge picks must be one that key could use itself.
    const keyScope = await resolveVisionBridgeKeyScope(context.apiKeyInfo, this.deps.keyScope);
    const scopedCheck = keyScope
      ? scopedCredentialCheck(this.deps.hasUsableCredentials, keyScope)
      : undefined;
    let selectedModelPromise: Promise<string | null> | null = null;
    const selectVideoModel = (): Promise<string | null> => {
      if (!selectedModelPromise) {
        const select =
          this.deps.selectVisionModel ??
          ((fixedModel?: string) =>
            keyScope && scopedCheck
              ? selectScopedVideoVisionModel(fixedModel, keyScope, scopedCheck)
              : getBestVisionModel({ fixedModel }));
        selectedModelPromise = select(configuredModel || undefined);
      }
      return selectedModelPromise;
    };
    const pipelineDeps: ProcessVideoPartDeps = {
      broker: {
        loadVideoPartBytes,
        extractFrames: this.deps.extractFrames,
        fetchRemote: this.deps.fetchRemote,
      },
      transcription: {
        describePart: createVideoSttAdapter({
          cache,
          dependencies: this.deps,
          keyScope:
            keyScope && scopedCheck ? { scope: keyScope, hasUsableCredentials: scopedCheck } : null,
          principalId: context.apiKeyInfo?.id,
          settings: persisted,
        }),
      },
      // Whole-result cache predates STT consent/model identity. Do not let a visual-only
      // entry satisfy a consented request or an STT entry satisfy an opted-out request.
      cache: resolveVideoAudioTranscriptionRuntimeSettings(persisted).enabled ? null : cache,
      selectVideoModel,
      overrideDescribePart: this.deps.describePart,
      callVisionModel: this.deps.callVisionModel,
      ...(keyScope ? { visionKeyScope: keyScope } : {}),
    };

    const startedAt = Date.now();
    const descriptions: Array<string | null> = [];
    const fallbackParts: VideoPart[] = [];
    let totalFramesRequested = 0;
    let totalFramesExtracted = 0;
    let totalFramesUsed = 0;
    let totalDurationSeconds = 0;
    let totalCacheHits = 0;
    let totalSamplingCandidateCount = 0;
    let totalDedupDropped = 0;
    let focusWindowsApplied = 0;
    let focusHintsApplied = 0;
    let transcriptCuesApplied = 0;
    let contactSheetsUsed = 0;
    let audioFusionRuns = 0;
    let audioFusionPartials = 0;
    const audioFusionFailureCodes = new Set<string>();
    const recordFusionTelemetry = (fusion?: VideoFusionTelemetry): void => {
      if (!fusion) return;
      audioFusionRuns += 1;
      if (fusion.partial) audioFusionPartials += 1;
      for (const [source, code] of Object.entries(fusion.failures ?? {})) {
        audioFusionFailureCodes.add(`${source}:${code}`);
      }
    };
    let samplingPolicyEffective: "uniform" | "scene_aware" | "segment_aware" = "uniform";
    let failures = 0;
    const logRedactionEntries: VideoBridgeLogRedactionEntry[] = [];

    const attemptedParts = parts.slice(0, runtime.maxVideos);
    for (let index = 0; index < attemptedParts.length; index++) {
      const part = attemptedParts[index];
      const result = await processVideoPart({
        analysis,
        context,
        deps: pipelineDeps,
        part,
        partIndex: index,
        runtime,
        visionRuntime,
      });

      if (result.status === "aborted") {
        throw new Error("Video Bridge processing was aborted");
      }
      if (result.status === "failed") {
        failures += 1;
        fallbackParts.push(part);
        descriptions.push(
          capabilities.supportsVideo === false
            ? `[Video ${index + 1}]: (unavailable — video could not be described)`
            : null
        );
        continue;
      }

      descriptions.push(result.description);
      if (result.descriptionRedacted !== undefined) {
        logRedactionEntries.push({
          container: part.container,
          messageIndex: part.messageIndex,
          partIndex: part.partIndex,
          // The exact text `replaceVideoParts` is about to write into
          // content[partIndex].text (same `result.description` value pushed
          // to `descriptions` just above) — the content-address key the
          // downstream consumer matches on. See the interface doc.
          fullText: result.description,
          redactedText: result.descriptionRedacted,
        });
      }
      totalFramesRequested += result.framesRequested;
      totalFramesExtracted += result.framesExtracted;
      totalFramesUsed += result.framesUsed;
      totalDedupDropped += result.dedupDropped;
      if (result.hasFocusWindow) focusWindowsApplied += 1;
      if (analysis.analysisMode === "focused") focusHintsApplied += 1;
      totalDurationSeconds += result.durationSeconds;
      totalSamplingCandidateCount += result.samplingCandidateCount;
      transcriptCuesApplied += result.transcriptCuesApplied;
      if (result.contactSheetUsed) contactSheetsUsed += 1;
      recordFusionTelemetry(result.fusion);
      if (result.samplingPolicyEffective !== "uniform") {
        samplingPolicyEffective = result.samplingPolicyEffective;
      }
      for (const producerModel of result.producerModels) successfulModels.add(producerModel);
      totalCacheHits += result.frameCacheHits;
    }

    for (let index = attemptedParts.length; index < parts.length; index++) {
      fallbackParts.push(parts[index]);
      descriptions.push(
        capabilities.supportsVideo === false
          ? `[Video ${index + 1}]: (not processed because the per-request video limit was reached)`
          : null
      );
    }

    const videosProcessed = attemptedParts.length - failures;
    const videosReplaced = descriptions.filter((description) => description !== null).length;
    if (videosReplaced === 0 && !rawTranscriptObserved) return { block: false };

    const modifiedPayload = replaceVideoParts(body, parts, descriptions);
    logRedactionEntries.push(
      ...appendVideoFallbackTranscripts(modifiedPayload, fallbackParts, context.log)
    );

    return {
      block: false,
      modifiedPayload,
      meta: {
        analysisMode: analysis.analysisMode,
        analysisModeRequested: analysis.requestedAnalysisMode,
        cacheHits: totalCacheHits,
        durationSeconds: totalDurationSeconds,
        failures,
        framesExtracted: totalFramesExtracted,
        framesRequested: totalFramesRequested,
        framesUsed: totalFramesUsed,
        dedupDropped: totalDedupDropped,
        focusWindowsApplied,
        focusHintsApplied,
        transcriptCuesApplied,
        // Protect both rendered cues and raw fields on unreplaced video parts.
        // Frames without transcript fields retain their normal logging policy.
        videoBridgeObserved: rawTranscriptObserved || logRedactionEntries.length > 0,
        ...(logRedactionEntries.length > 0 ? { videoBridgeLogRedaction: logRedactionEntries } : {}),
        contactSheetsUsed,
        audioFusionRuns,
        audioFusionPartials,
        audioFusionFailureCodes: [...audioFusionFailureCodes].sort(),
        samplingCandidateCount: totalSamplingCandidateCount,
        samplingPolicyEffective,
        samplingPolicyRequested: runtime.samplingPolicy,
        processingTimeMs: Date.now() - startedAt,
        attempts: attemptedParts.length,
        videoModel: combineModelIdentities(successfulModels, routingPlanModel),
        videosProcessed,
        videosReplaced,
      },
    };
  }
}