Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

19.7 KB
Raw Blame
import type { NormalizerContext, ProviderAdapter } from "../types";
import {
  removeNullFields,
  stringifyToolResultContent,
  parseMetadata,
  isRichToolResult,
  getNestedProperty,
} from "../helpers";
import { z } from "zod";

/**
 * Detection schemas for OpenAI formats
 * These are permissive - only validate structural markers, not full API contracts
 */

// INPUT SCHEMAS (requests)
const OpenAIInputChatCompletionsSchema = z.looseObject({
  messages: z.array(z.any()),
  tools: z.array(z.any()).optional(),
});

const OpenAIInputMessagesSchema = z
  .array(
    z.looseObject({
      role: z.enum(["system", "user", "assistant", "tool", "function"]),
    }),
  )
  .refine(
    (data) => {
      // Reject if any message has top-level parts (Microsoft Agent/Gemini format)
      // OpenAI uses parts inside content, not at message level
      return !data.some(
        (msg) =>
          typeof msg === "object" &&
          msg !== null &&
          "parts" in msg &&
          Array.isArray((msg as Record<string, unknown>).parts),
      );
    },
    { message: "Messages with top-level parts are not OpenAI format" },
  );

// Role-less Responses input item types. normalizeMessage converts the common
// conversation items and preserves built-in tool items as JSON passthrough.
// Reference: https://platform.openai.com/docs/api-reference/responses/create
const RESPONSES_ITEM_TYPES = new Set([
  "reasoning",
  "function_call",
  "tool_call",
  "function_call_output",
  "file_search_call",
  "computer_call",
  "computer_call_output",
  "web_search_call",
  "code_interpreter_call",
  "image_generation_call",
  "local_shell_call",
  "local_shell_call_output",
  "shell_call",
  "shell_call_output",
  "apply_patch_call",
  "apply_patch_call_output",
  "mcp_list_tools",
  "mcp_approval_request",
  "mcp_approval_response",
  "mcp_call",
  "custom_tool_call",
  "custom_tool_call_output",
  "tool_search_call",
  "tool_search_output",
  "compaction",
  "item_reference",
]);

/**
 * Check that a Responses API request `input` array holds role-based messages
 * or typed Responses items normalizeMessage converts or preserves. Rejects string items
 * (embeddings-style input) and messages with top-level parts (Microsoft
 * Agent/Gemini format).
 */
function isResponsesApiInputArray(value: unknown): value is unknown[] {
  return (
    Array.isArray(value) &&
    value.length > 0 &&
    value.every((item) => {
      if (!item || typeof item !== "object") return false;
      const msg = item as Record<string, unknown>;
      if (Array.isArray(msg.parts)) return false;
      if (typeof msg.role === "string") return true;
      return typeof msg.type === "string" && RESPONSES_ITEM_TYPES.has(msg.type);
    })
  );
}

// Bare {input: "..."} is too generic to claim; a string input needs a
// distinctive Responses API request sibling.
const RESPONSES_REQUEST_SIBLINGS = [
  "tools",
  "tool_choice",
  "parallel_tool_calls",
  "instructions",
];

/**
 * Responses API request: {input: [...] | "...", tools?, instructions?, ...}
 * Shared by detect() and preprocessData so detection cannot drift from what
 * preprocessing consumes. Also used by extractAdditionalInput to hide `input`
 * from additional input only when it holds the conversation.
 */
function isOpenAIResponsesRequest(
  data: unknown,
): data is Record<string, unknown> {
  if (!data || typeof data !== "object" || Array.isArray(data)) return false;
  if ("messages" in data || "output" in data) return false;
  const obj = data as Record<string, unknown>;
  if (isResponsesApiInputArray(obj.input)) return true;
  return (
    typeof obj.input === "string" &&
    obj.input.length > 0 &&
    RESPONSES_REQUEST_SIBLINGS.some((key) => key in obj)
  );
}

const OpenAIInputResponsesSchema = z.custom<Record<string, unknown>>(
  isOpenAIResponsesRequest,
);

// OUTPUT SCHEMAS (responses)
const OpenAIOutputResponsesSchema = z.looseObject({
  output: z.array(z.any()),
  tools: z.array(z.any()).optional(),
});

const OpenAIOutputChoicesSchema = z.looseObject({
  model: z.string(),
  choices: z.array(z.any()),
});

const OpenAIOutputSingleMessageSchema = z.looseObject({
  role: z.string(),
  tool_calls: z.array(
    z.looseObject({
      type: z.string(),
      function: z.looseObject({
        name: z.string(),
      }),
    }),
  ),
});

/**
 * Extract thinking content from OpenAI Responses API reasoning item
 * Format: { type: "reasoning", content: [...], summary: [...] }
 */
function extractReasoningContent(item: Record<string, unknown>): {
  thinking?: Array<{ type: "thinking"; content: string; summary?: string }>;
  redacted_thinking?: Array<{ type: "redacted_thinking"; data: string }>;
} | null {
  if (item.type !== "reasoning") return null;

  const contentArr = Array.isArray(item.content) ? item.content : [];
  const summaryArr = Array.isArray(item.summary) ? item.summary : [];

  // Extract text from content array (may contain {type: "text", text: "..."})
  const contentText = contentArr
    .map((c: unknown) => {
      if (typeof c === "string") return c;
      if (c && typeof c === "object" && "text" in c) {
        return (c as { text: string }).text;
      }
      return "";
    })
    .filter(Boolean)
    .join("\n");

  // Extract summary text
  const summaryText = summaryArr
    .map((s: unknown) => {
      if (typeof s === "string") return s;
      if (s && typeof s === "object" && "text" in s) {
        return (s as { text: string }).text;
      }
      return "";
    })
    .filter(Boolean)
    .join("\n");

  if (!contentText && !summaryText) {
    return typeof item.encrypted_content === "string" &&
      item.encrypted_content !== ""
      ? {
          redacted_thinking: [
            {
              type: "redacted_thinking",
              data: item.encrypted_content,
            },
          ],
        }
      : null;
  }

  return {
    thinking: [
      {
        type: "thinking" as const,
        content: contentText || summaryText,
        ...(summaryText && contentText ? { summary: summaryText } : {}),
      },
    ],
  };
}

function normalizeMessage(msg: unknown): Record<string, unknown> {
  if (!msg || typeof msg !== "object") return {};

  // we want to do type-based conversions BEFORE removeNullFields
  // because removeNullFields moves unrecognized fields to json passthrough
  let working = msg as Record<string, unknown>;

  // Handle OpenAI Responses API reasoning item
  // Format: { type: "reasoning", content: [...], summary: [...] }
  const reasoningContent = extractReasoningContent(working);
  if (reasoningContent) {
    return {
      role: "assistant",
      content: "",
      ...reasoningContent,
    };
  }

  // Convert direct function call message to tool_calls array format
  // Format: { type: "function_call", name: "...", arguments: {...}, call_id: "..." }
  // Convert to: { role: "assistant", tool_calls: [{ id, name, arguments, type }] }
  if (
    (working.type === "function_call" || working.type === "tool_call") &&
    working.name &&
    typeof working.name === "string"
  ) {
    const toolCall: Record<string, unknown> = {
      id: working.call_id || working.id || "",
      name: working.name,
      arguments:
        typeof working.arguments === "string"
          ? working.arguments
          : JSON.stringify(working.arguments ?? {}),
      type: "function",
    };

    const {
      type: _type,
      name: _name,
      arguments: _args,
      call_id: _call_id,
      id: _id,
      status: _status,
      ...rest
    } = working;
    working = {
      ...rest,
      role: rest.role || "assistant",
      tool_calls: [toolCall],
    };
  }

  // Convert function_call_output to standard tool message format
  // Format: { type: "function_call_output", call_id: "...", output: "..." }
  // Convert to: { role: "tool", tool_call_id: "...", content: "..." }
  if (working.type === "function_call_output") {
    const content =
      typeof working.output === "string"
        ? working.output
        : JSON.stringify(working.output ?? "");

    const { type: _type, call_id, output: _output, ...rest } = working;

    working = {
      ...rest,
      role: "tool",
      tool_call_id: call_id,
      content,
    };
  }

  // Now apply removeNullFields after conversions
  let normalized = removeNullFields(working);

  // Extract text from OpenAI Agents output_text format
  // Format: content: [{type: "output_text", text: "..."}]
  if (
    normalized.content &&
    Array.isArray(normalized.content) &&
    normalized.content.length > 0
  ) {
    const firstItem = normalized.content[0];
    if (
      firstItem &&
      typeof firstItem === "object" &&
      (firstItem as Record<string, unknown>).type === "output_text" &&
      typeof (firstItem as Record<string, unknown>).text === "string"
    ) {
      // Extract just the text for simple output
      normalized.content = (firstItem as Record<string, unknown>).text;
    }
  }

  // Normalize camelCase toolCalls to snake_case tool_calls (VAPI framework does this, otherwise it matches OpenAI)
  if ("toolCalls" in normalized && Array.isArray(normalized.toolCalls)) {
    normalized.tool_calls = normalized.toolCalls;
    delete normalized.toolCalls;
  }

  // Flatten OpenAI nested tool_calls format: function.name → name, function.arguments → arguments
  // Format: { id, type: "function", function: { name, arguments } }
  // Convert to: { id, name, arguments, type }
  if (normalized.tool_calls && Array.isArray(normalized.tool_calls)) {
    const sanitizedToolCalls = (normalized.tool_calls as unknown[]).filter(
      (tc): tc is Record<string, unknown> =>
        Boolean(tc) && typeof tc === "object",
    );

    normalized.tool_calls = sanitizedToolCalls.map((tc) => {
      if (tc.function && typeof tc.function === "object") {
        const func = tc.function as Record<string, unknown>;
        return {
          id: tc.id,
          name: func.name,
          arguments:
            typeof func.arguments === "string"
              ? func.arguments
              : JSON.stringify(func.arguments ?? {}),
          type: tc.type || "function",
          index: tc.index,
        };
      }
      // Already flat format, just ensure arguments is stringified
      return {
        ...tc,
        arguments:
          typeof tc.arguments === "string"
            ? tc.arguments
            : JSON.stringify(tc.arguments ?? {}),
      };
    });
  }

  // For tool messages with rich object content, spread into message
  // so it goes to json passthrough field and renders as PrettyJsonView.
  // Rich = nested structure OR 3+ keys. Simple <=2 scalar keys.
  if (
    normalized.role === "tool" &&
    typeof normalized.content === "object" &&
    normalized.content !== null &&
    !Array.isArray(normalized.content)
  ) {
    if (isRichToolResult(normalized.content)) {
      // Rich object: spread for table rendering
      const { content, ...rest } = normalized;
      return { ...rest, ...content };
    }

    // Simple object: stringify for text rendering
    normalized.content = stringifyToolResultContent(normalized.content);
  }

  return normalized;
}

/**
 * Flatten tool definition from nested or flat format to standard format
 * Handles both Chat Completions {type, function: {name, ...}} and flat {name, ...}
 */
function flattenToolDefinition(tool: unknown): Record<string, unknown> {
  if (typeof tool !== "object" || !tool) return {};

  const t = tool as Record<string, unknown>;
  // Handle nested {type, function: {name, ...}} or flat {name, ...}
  const toolFunc = (t.function as Record<string, unknown> | undefined) ?? t;

  const toolDef: Record<string, unknown> = {
    name: toolFunc.name,
    description: toolFunc.description ?? "",
  };
  if (toolFunc.parameters != null) toolDef.parameters = toolFunc.parameters;
  return toolDef;
}

function normalizeMessagesWithTools(
  messages: unknown[],
  tools: unknown[],
): Array<Record<string, unknown>> {
  const normalizedTools = tools.map(flattenToolDefinition);
  return messages.map((message) => ({
    ...normalizeMessage(message),
    tools: normalizedTools,
  }));
}

function preprocessData(data: unknown): unknown {
  if (!data) return data;

  // OpenAI Chat Completions API: {tools, messages} OR Responses API: {tools, output}
  // References:
  // - https://platform.openai.com/docs/api-reference/chat/create
  // - https://platform.openai.com/docs/api-reference/responses
  if (
    typeof data === "object" &&
    !Array.isArray(data) &&
    "tools" in data &&
    (("messages" in data && !("output" in data)) || "output" in data)
  ) {
    const obj = data as Record<string, unknown>;
    const messagesArray = (obj.messages ?? obj.output) as unknown[];

    if (Array.isArray(messagesArray) && Array.isArray(obj.tools)) {
      // Attach tools to all messages
      return normalizeMessagesWithTools(messagesArray, obj.tools);
    }
  }

  // Responses API without tools: {output: [...]}
  if (
    typeof data === "object" &&
    !Array.isArray(data) &&
    "output" in data &&
    !("messages" in data) &&
    !("tools" in data)
  ) {
    const obj = data as Record<string, unknown>;
    if (Array.isArray(obj.output)) {
      return (obj.output as unknown[]).map(normalizeMessage);
    }
  }

  // Responses API request: {input: [...] | "...", tools?, instructions?, ...}
  // Reference: https://platform.openai.com/docs/api-reference/responses/create
  if (isOpenAIResponsesRequest(data)) {
    const items =
      typeof data.input === "string"
        ? [{ role: "user", content: data.input }]
        : (data.input as unknown[]);
    const messages =
      typeof data.instructions === "string" && data.instructions !== ""
        ? (
            [{ role: "system", content: data.instructions }] as unknown[]
          ).concat(items)
        : items;
    if (Array.isArray(data.tools)) {
      // Attach tools to all messages
      return normalizeMessagesWithTools(messages, data.tools);
    }
    return messages.map(normalizeMessage);
  }

  // Chat Completions response: {choices: [{message: {...}}]}
  if (typeof data === "object" && !Array.isArray(data) && "choices" in data) {
    const obj = data as Record<string, unknown>;
    if (Array.isArray(obj.choices) && obj.choices.length > 0) {
      const firstChoice = obj.choices[0] as Record<string, unknown>;
      if (
        firstChoice &&
        typeof firstChoice === "object" &&
        "message" in firstChoice
      ) {
        return normalizeMessage(firstChoice.message);
      }
    }
  }

  if (
    typeof data === "object" &&
    !Array.isArray(data) &&
    ["function_call", "tool_call", "function_call_output"].includes(
      String((data as Record<string, unknown>).type),
    )
  ) {
    return normalizeMessage(data);
  }

  // Array of messages
  if (Array.isArray(data)) {
    return data.map(normalizeMessage);
  }

  // Object with messages key
  if (typeof data === "object" && "messages" in data) {
    const obj = data as Record<string, unknown>;
    return {
      ...obj,
      messages: Array.isArray(obj.messages)
        ? obj.messages.map(normalizeMessage)
        : obj.messages,
    };
  }

  // Single message
  if (typeof data === "object" && "role" in data) {
    return normalizeMessage(data);
  }

  return data;
}

export const openAIAdapter: ProviderAdapter = {
  id: "openai",

  detect(ctx: NormalizerContext): boolean {
    const meta = parseMetadata(ctx.metadata);

    // REJECTIONS: Explicit rejection of LangGraph/LangChain/Semantic/Pydantic Kernel formats
    if (meta && typeof meta === "object") {
      // LangGraph
      if (
        "langgraph_step" in meta ||
        "langgraph_node" in meta ||
        "langgraph_path" in meta ||
        meta.framework === "langgraph" ||
        (Array.isArray(meta.tags) && meta.tags.includes("langgraph"))
      ) {
        return false;
      }

      // Semantic Kernel (scope.name starts with Microsoft.SemanticKernel)
      if (
        "scope" in meta &&
        meta.scope !== null &&
        typeof meta.scope === "object"
      ) {
        const scope = meta.scope as Record<string, unknown>;
        if (
          typeof scope.name === "string" &&
          scope.name.startsWith("Microsoft.SemanticKernel")
        ) {
          return false;
        }

        if (
          scope.name === "agent_framework" ||
          (typeof scope.name === "string" &&
            scope.name.includes("Microsoft.Extensions.AI"))
        ) {
          return false;
        }
      }

      const providerName = getNestedProperty(
        meta,
        "attributes",
        "gen_ai.provider.name",
      );
      if (providerName === "microsoft.agent_framework") return false;

      // Pydantic ai
      if (
        "scope" in meta &&
        meta.scope !== null &&
        typeof meta.scope === "object"
      ) {
        const scope = meta.scope as Record<string, unknown>;
        if (scope.name === "pydantic-ai") {
          return false;
        }
      }

      // LangChain (type without role)
      if (
        ctx.metadata &&
        typeof ctx.metadata === "object" &&
        "messages" in ctx.metadata
      ) {
        const messages = (ctx.metadata as Record<string, unknown>).messages;
        if (Array.isArray(messages)) {
          const hasLangChainType = messages.some((msg: unknown) => {
            // Null guard: ensure msg exists and is an object before accessing properties
            if (!msg || typeof msg !== "object") {
              return false;
            }
            const message = msg as Record<string, unknown>;
            return (
              message.type &&
              typeof message.type === "string" &&
              ["human", "ai", "tool", "system"].includes(message.type) &&
              !("role" in message)
            );
          });
          if (hasLangChainType) return false;
        }
      }
    }

    // HINTS: Fast checks for explicit OpenAI indicators
    if (ctx.framework === "openai") return true;
    if (ctx.observationName?.toLowerCase().includes("openai")) return true;
    if (meta?.ls_provider === "openai") return true;

    // Metadata attributes check
    if (meta && typeof meta === "object" && "attributes" in meta) {
      const attributes = (meta as Record<string, unknown>).attributes;
      if (
        attributes &&
        typeof attributes === "object" &&
        (attributes as Record<string, unknown>)["llm.system"] === "openai"
      ) {
        return true;
      }
    }

    // STRUCTURAL: Schema-based detection on metadata
    if (OpenAIInputChatCompletionsSchema.safeParse(ctx.metadata).success)
      return true;
    if (OpenAIInputMessagesSchema.safeParse(ctx.metadata).success) return true;
    if (OpenAIInputResponsesSchema.safeParse(ctx.metadata).success) return true;
    if (OpenAIOutputResponsesSchema.safeParse(ctx.metadata).success)
      return true;
    if (OpenAIOutputChoicesSchema.safeParse(ctx.metadata).success) return true;
    if (OpenAIOutputSingleMessageSchema.safeParse(ctx.metadata).success)
      return true;

    // finally, test on data if available. we might've done this already if we passed
    // data into metadata. we only do this last due to performance concerns.
    if (OpenAIInputChatCompletionsSchema.safeParse(ctx.data).success)
      return true;
    if (OpenAIInputMessagesSchema.safeParse(ctx.data).success) return true;
    if (OpenAIInputResponsesSchema.safeParse(ctx.data).success) return true;
    if (OpenAIOutputResponsesSchema.safeParse(ctx.data).success) return true;
    if (OpenAIOutputChoicesSchema.safeParse(ctx.data).success) return true;
    if (OpenAIOutputSingleMessageSchema.safeParse(ctx.data).success)
      return true;

    return false;
  },

  preprocess(
    data: unknown,
    _kind: "input" | "output",
    _ctx: NormalizerContext,
  ): unknown {
    return preprocessData(data);
  },

  getConsumedInputKeys(data: unknown): string[] {
    if (!isOpenAIResponsesRequest(data)) return [];

    return typeof data.instructions === "string" && data.instructions !== ""
      ? ["input", "instructions"]
      : ["input"];
  },
};