Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

17.5 KB
Raw Blame
import { Sha256 } from "@aws-crypto/sha256-js";
import { SignatureV4 } from "@smithy/signature-v4";
import {
  APICallError,
  NoObjectGeneratedError,
  Output,
  TypeValidationError,
  type LanguageModelUsage,
  type StepResult,
  type ToolSet,
} from "ai";
import { z } from "zod";
import { decrypt } from "../../encryption";
import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
import {
  assertValidBedrockRegion,
  canUseBedrockDefaultCredentials,
  resolveBedrockProviderAuth,
} from "../llm/ai-sdk/providers/bedrock";
import { isPlainObject } from "../llm/ai-sdk/providers/utils";
import { generateLLMText } from "../llm/llmText";
import { LLMAdapter, type TraceSinkParams } from "../llm/types";
import { logger } from "../logger";
import { parseOutboundUrl } from "../outbound-url/validation";
import { writeInternalTraceViaOtelIngestion } from "../otel/internalTraceOtelWriter";
import type { TopicsModel } from "./model-config";

const TIMEOUT_MS = 60_000;

// Error identity and lifetime are preserved, including for frozen SDK errors.
const responseMetadataByError = new WeakMap<
  object,
  TopicTextResponseMetadata
>();

const RAW_FINISH_REASONS = new Set([
  "stop",
  "length",
  "content_filter",
  "tool_calls",
  "function_call",
  "end_turn",
  "max_tokens",
  "stop_sequence",
  "tool_use",
  "refusal",
  "guardrail_intervened",
  "content_filtered",
  "model_context_window_exceeded",
  "malformed_model_output",
  "pause_turn",
  "STOP",
  "MAX_TOKENS",
  "SAFETY",
  "RECITATION",
  "OTHER",
]);

/**
 * Structured Topics text through the project's LLM connection. GPT-5.6+ models
 * on Bedrock reuse a shared prompt prefix only with explicit cache breakpoints,
 * which Converse does not accept for them, so cached prompts on those models
 * use InvokeModel at the same regional endpoint and model ID.
 */
export async function generateTopicText<T>(
  params: TopicTextParams<T>,
): Promise<TopicTextResult> {
  const startTimeISO = new Date().toISOString();
  let result: TopicTextResult | undefined;
  let responseMetadata: TopicTextResponseMetadata | undefined;
  let failure: unknown;
  const recordResponse = (
    response: TopicTextResult,
    metadata?: TopicTextResponseMetadata,
  ) => {
    result = response;
    responseMetadata = metadata;
  };
  try {
    result =
      isBedrockGPT(params.model) && params.system.some((part) => part.cache)
        ? await invokeWithCache(params, recordResponse)
        : await generateWithConnection(params, recordResponse);
    return result;
  } catch (error) {
    failure = error;
    // Responses refusal blocks can fail SDK validation before onEnd runs.
    if (
      APICallError.isInstance(error) &&
      error.statusCode === 200 &&
      TypeValidationError.isInstance(error.cause) &&
      hasOpenAIRefusal(error.cause.value)
    ) {
      responseMetadata = {
        "langfuse.topics.response.finish_reason": "content-filter",
      };
    }
    if (responseMetadata && error !== null && typeof error === "object") {
      responseMetadataByError.set(error, responseMetadata);
    }
    if (NoObjectGeneratedError.isInstance(error)) {
      result = { output: error.text, usage: topicUsage(error.usage) };
    }
    throw error;
  } finally {
    await writeTopicTrace(params, startTimeISO, result, failure);
  }
}

/** Bounded response diagnostics for the existing Topics failure log; no content. */
export function getTopicTextResponseMetadata(error: unknown) {
  return error !== null && typeof error === "object"
    ? responseMetadataByError.get(error)
    : undefined;
}

async function writeTopicTrace<T>(
  params: TopicTextParams<T>,
  startTimeISO: string,
  result: TopicTextResult | undefined,
  failure: unknown,
) {
  const { trace } = params;
  if (!trace) return;
  try {
    /** Validation marks the trace; the caller validates after accounting for usage. */
    const validation = params.schema.safeParse(result?.output);
    await writeInternalTraceViaOtelIngestion({
      rootSpanId: trace.traceId,
      eventInputs: [
        {
          projectId: trace.targetProjectId,
          traceId: trace.traceId,
          spanId: trace.traceId,
          name: trace.traceName,
          traceName: trace.traceName,
          type: "GENERATION",
          environment: trace.environment,
          startTimeISO,
          endTimeISO: new Date().toISOString(),
          modelName: params.model.model,
          modelParameters: { maxOutputTokens: params.maxOutputTokens },
          input: JSON.stringify(topicMessages(params)),
          output: JSON.stringify(result?.output),
          providedUsageDetails: topicTraceUsage(result?.usage),
          metadata: trace.metadata ?? {},
          userId: trace.userId,
          sessionId: trace.sessionId,
          ...topicTraceError(failure ?? validation.error),
        },
      ],
    });
  } catch {
    logger.error("Failed to publish Topics model trace", {
      traceId: trace.traceId,
    });
  }
}

function topicTraceUsage(usage: TopicTextResult["usage"] | undefined) {
  if (!usage) return undefined;
  const details: Record<string, number | undefined> = {
    input: undefined,
    output: usage.outputTokens,
    total: usage.totalTokens,
    input_cached_tokens: usage.cacheReadTokens,
    input_cache_creation: usage.cacheWriteTokens,
  };
  /** Standard input excludes cached tokens, which have their own prices. */
  if (usage.inputTokens !== undefined) {
    details.input = Math.max(
      0,
      usage.inputTokens -
        (usage.cacheReadTokens ?? 0) -
        (usage.cacheWriteTokens ?? 0),
    );
  }
  return Object.fromEntries(
    Object.entries(details).filter(
      (entry): entry is [string, number] => entry[1] !== undefined,
    ),
  );
}

function topicTraceError(failure: unknown) {
  if (failure === undefined) return {};
  return {
    level: "ERROR" as const,
    statusMessage: failure instanceof Error ? failure.message : String(failure),
  };
}

/** The provider-bound messages of a Topics request; also used to measure its egress. */
export function topicMessages(params: {
  system: TopicPromptPart[];
  input: string;
}) {
  return [
    {
      role: "system" as const,
      content: params.system.map((part) => part.text).join("\n\n"),
    },
    { role: "user" as const, content: params.input },
  ];
}

function topicUsage(
  usage: LanguageModelUsage | undefined,
): TopicTextResult["usage"] {
  return {
    inputTokens: usage?.inputTokens,
    outputTokens: usage?.outputTokens,
    totalTokens: usage?.totalTokens,
    cacheReadTokens: usage?.inputTokenDetails?.cacheReadTokens,
    cacheWriteTokens: usage?.inputTokenDetails?.cacheWriteTokens,
  };
}

async function generateWithConnection<T>(
  params: TopicTextParams<T>,
  recordResponse: (
    result: TopicTextResult,
    metadata: TopicTextResponseMetadata,
  ) => void,
): Promise<TopicTextResult> {
  const nativeOpenAI = isFirstPartyOpenAI(params.model);
  const result = await generateLLMText({
    model: { adapter: params.model.adapter, id: params.model.model },
    connection: params.model.connection,
    openAIChatProvider: nativeOpenAI ? "native" : undefined,
    // The static system prompt comes first so providers with automatic prefix
    // caching can reuse it across traces.
    messages: topicMessages(params),
    output: Output.object({ schema: params.schema }),
    maxOutputTokens: params.maxOutputTokens,
    reasoning: topicReasoning(params.model),
    // Bedrock ignores portable reasoning for OpenAI models; their default effort
    // is not none.
    ...(isBedrockGPT(params.model)
      ? {
          providerOptions: {
            bedrock: {
              additionalModelRequestFields: { reasoning: { effort: "none" } },
            },
          },
        }
      : {}),
    // The AI SDK retries 429 and 5xx responses with exponential backoff.
    maxRetries: 2,
    timeout: params.timeoutMs ?? TIMEOUT_MS,
    // Retain wire refusal markers in memory; diagnostics only expose their classification.
    include: { responseBody: true },
    // onEnd runs before structured-output parsing and the output getter can throw.
    onEnd: ({ finalStep }) => {
      const text = finalStep.text;
      recordResponse(
        { output: text, usage: topicUsage(finalStep.usage) },
        responseAttributes(finalStep, text.length, params.maxOutputTokens),
      );
    },
  });
  const usage = topicUsage(result.usage);
  return {
    output: result.output,
    usage,
  };
}

function responseAttributes(
  step: StepResult<ToolSet>,
  textCharacters: number,
  maxOutputTokens: number,
): TopicTextResponseMetadata {
  return {
    "langfuse.topics.response.finish_reason": hasOpenAIRefusal(
      step.response.body,
    )
      ? "content-filter"
      : step.finishReason,
    "langfuse.topics.response.raw_finish_reason": RAW_FINISH_REASONS.has(
      step.rawFinishReason ?? "",
    )
      ? step.rawFinishReason!
      : "other",
    "langfuse.topics.response.text_characters": textCharacters,
    "langfuse.topics.response.text_part_count": step.content.filter(
      (part) => part.type === "text",
    ).length,
    "langfuse.topics.response.reasoning_part_count": step.content.filter(
      (part) => part.type === "reasoning",
    ).length,
    "langfuse.topics.response.max_output_tokens": maxOutputTokens,
    ...(step.usage.outputTokens !== undefined && {
      "langfuse.topics.response.output_tokens": step.usage.outputTokens,
    }),
  };
}

/** Inspect wire markers only; generated text and exception messages are not signals. */
function hasOpenAIRefusal(body: unknown): boolean {
  if (!isPlainObject(body)) return false;
  const choice = Array.isArray(body.choices) ? body.choices[0] : undefined;
  const message = isPlainObject(choice) ? choice.message : undefined;
  if (
    isPlainObject(message) &&
    typeof message.refusal === "string" &&
    message.refusal.length > 0
  )
    return true;
  return (
    Array.isArray(body.output) &&
    body.output.some(
      (item: unknown) =>
        isPlainObject(item) &&
        item.type === "message" &&
        item.role === "assistant" &&
        Array.isArray(item.content) &&
        item.content.some(
          (part: unknown) => isPlainObject(part) && part.type === "refusal",
        ),
    )
  );
}

function topicReasoning(
  model: TopicsModel,
): "none" | "minimal" | "low" | undefined {
  if (isFirstPartyOpenAI(model)) return openAIReasoning(model.model);
  /** Compatible endpoints choose their own supported reasoning settings. */
  if (
    model.adapter === LLMAdapter.OpenAI &&
    model.connection.config?.useResponsesApi !== true
  )
    return undefined;
  return "none";
}

/** Topics owns its OpenAI request options; saved evaluator options keep their routing. */
function isFirstPartyOpenAI(model: TopicsModel): boolean {
  if (model.adapter !== LLMAdapter.OpenAI) return false;
  if (!model.connection.baseURL) return true;
  try {
    const hostname = parseOutboundUrl(
      model.connection.baseURL.replace("{model}", model.model),
    ).hostname.replace(/\.$/, "");
    return (
      hostname === "api.openai.com" || hostname.endsWith(".api.openai.com")
    );
  } catch {
    return false;
  }
}

/** Selects low-latency reasoning only for known models; others use their defaults. */
function openAIReasoning(
  model: string,
): "none" | "minimal" | "low" | undefined {
  const id = model.replace(/-\d{4}-\d{2}-\d{2}$/, "");
  if (/^gpt-5(?:-(?:mini|nano))?$/.test(id)) return "minimal";
  if (
    /^gpt-5\.[1-6](?:-(?:mini|nano|luna|sol|terra))?$/.test(id) ||
    id === "gpt-6-luna" ||
    id === "gpt-6-sol"
  )
    return "none";
  // The API accepts none, but @ai-sdk/openai drops it for other GPT-6 models.
  if (/^gpt-6(?:\.\d+)?(?:-[a-z]+)?$/.test(id)) return "low";
  return undefined;
}

function isBedrockGPT(model: TopicsModel): boolean {
  if (model.adapter !== LLMAdapter.Bedrock) return false;
  /** Opaque application inference profiles cannot identify the underlying model. */
  const version = /(?:^|[/.])openai\.gpt-(\d+)(?:\.(\d+))?(?:[-:]|$)/.exec(
    model.model,
  );
  return Boolean(
    version &&
    (Number(version[1]) > 5 ||
      (Number(version[1]) === 5 && Number(version[2] ?? 0) >= 6)),
  );
}

function signingCredentials(
  auth: ReturnType<typeof resolveBedrockProviderAuth>,
) {
  const { credentialProvider } = auth;
  if (credentialProvider)
    return async () => {
      const resolved = await credentialProvider();
      return {
        accessKeyId: resolved.accessKeyId,
        secretAccessKey: resolved.secretAccessKey,
        sessionToken: resolved.sessionToken,
      };
    };
  if (auth.accessKeyId && auth.secretAccessKey)
    return {
      accessKeyId: auth.accessKeyId,
      secretAccessKey: auth.secretAccessKey,
    };
  return undefined;
}

/** SigV4 or bearer-token auth from the connection's stored Bedrock credential. */
async function bedrockRequestHeaders(
  model: TopicsModel,
  url: URL,
  region: string,
  body: string,
): Promise<Record<string, string>> {
  const headers = {
    host: url.hostname,
    "content-type": "application/json",
    accept: "application/json",
  };
  const auth = resolveBedrockProviderAuth({
    secretKey: decrypt(model.connection.secretKey),
    allowDefaultCredentials: canUseBedrockDefaultCredentials(
      model.connection.projectId,
    ),
  });
  if (auth.apiKey)
    return { ...headers, authorization: `Bearer ${auth.apiKey}` };
  const credentials = signingCredentials(auth);
  if (!credentials)
    throw new Error(
      "Invalid Bedrock credentials. Expected AWS access key JSON or a Bedrock API key.",
    );
  // AWS's own SigV4 signer, as used by the AWS SDK.
  const signed = await new SignatureV4({
    credentials,
    region,
    service: "bedrock",
    sha256: Sha256,
  }).sign({
    method: "POST",
    protocol: url.protocol,
    hostname: url.hostname,
    path: url.pathname,
    headers,
    body,
  });
  return signed.headers;
}

async function invokeWithCache(
  params: TopicTextParams<unknown>,
  recordResponse: (
    result: TopicTextResult,
    metadata?: TopicTextResponseMetadata,
  ) => void,
): Promise<TopicTextResult> {
  const { region } = BedrockConfigSchema.parse(params.model.connection.config);
  assertValidBedrockRegion(region);
  const { $schema: _, ...schema } = z.toJSONSchema(params.schema);
  const url = new URL(
    `https://bedrock-runtime.${region}.amazonaws.com/model/${encodeURIComponent(params.model.model)}/invoke`,
  );
  const body = JSON.stringify({
    messages: [
      {
        role: "system",
        content: params.system.map((part) => ({
          type: "text",
          text: part.text,
          ...(part.cache
            ? { prompt_cache_breakpoint: { mode: "explicit" } }
            : {}),
        })),
      },
      { role: "user", content: [{ type: "text", text: params.input }] },
    ],
    prompt_cache_options: { mode: "explicit", ttl: "30m" },
    reasoning_effort: "none",
    max_completion_tokens: params.maxOutputTokens,
    response_format: {
      type: "json_schema",
      json_schema: { name: "topics", schema, strict: true },
    },
  });
  // A single attempt; the host is derived from the validated region.
  const response = await fetch(url, {
    method: "POST",
    headers: await bedrockRequestHeaders(params.model, url, region, body),
    body,
    signal: AbortSignal.timeout(params.timeoutMs ?? TIMEOUT_MS),
  });
  // The status lets the Topics error classification tell throttling from auth or input errors.
  if (!response.ok)
    throw Object.assign(
      new Error(
        `Bedrock InvokeModel failed with ${response.status}: ${(await response.text()).slice(0, 500)}`,
      ),
      { status: response.status },
    );
  const json = (await response.json()) as {
    choices?: {
      finish_reason?: string;
      message?: { content?: string | null; refusal?: string | null };
    }[];
    usage?: {
      prompt_tokens?: number;
      completion_tokens?: number;
      total_tokens?: number;
      prompt_tokens_details?: {
        cached_tokens?: number;
        cache_write_tokens?: number;
      };
    };
  };
  const message = json.choices?.[0]?.message;
  const result: TopicTextResult = {
    output: message?.content ?? message?.refusal,
    usage: {
      inputTokens: json.usage?.prompt_tokens,
      outputTokens: json.usage?.completion_tokens,
      totalTokens: json.usage?.total_tokens,
      cacheReadTokens: json.usage?.prompt_tokens_details?.cached_tokens,
      cacheWriteTokens: json.usage?.prompt_tokens_details?.cache_write_tokens,
    },
  };
  // Retain paid usage and the original response if parsing or validation fails.
  recordResponse(
    result,
    hasOpenAIRefusal(json)
      ? { "langfuse.topics.response.finish_reason": "content-filter" }
      : undefined,
  );

  if (!message?.content || json.choices?.[0]?.finish_reason === "length")
    throw Object.assign(
      new Error(
        message?.refusal
          ? `The model refused: ${message.refusal}`
          : "The model returned no complete output.",
      ),
      { name: "AI_NoOutputGeneratedError" },
    );
  // A model can append text after a valid object; the caller treats that as unusable output.
  return { ...result, output: JSON.parse(message.content) as unknown };
}

/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */
export type TopicPromptPart = { text: string; cache?: boolean };

type TopicTextResponseMetadata = Record<string, string | number>;

type TopicTextResult = {
  output: unknown;
  usage: {
    inputTokens?: number;
    outputTokens?: number;
    totalTokens?: number;
    cacheReadTokens?: number;
    cacheWriteTokens?: number;
  };
};

type TopicTextParams<T> = {
  model: TopicsModel;
  system: TopicPromptPart[];
  input: string;
  schema: z.ZodType<T>;
  maxOutputTokens: number;
  timeoutMs?: number;
  trace?: TraceSinkParams;
};