🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(openai): add structured review output and enhance customer response fixtures (#18342)
marliessophie23d175a45b6b
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.tsindex 3931ba8b7e..eef217ea42 100644--- a/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts+++ b/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts@@ -55,8 +55,138 @@ export const documentedResponsesFixtures: NormalizedIOFixture[] = [},];+const structuredReviewOutput = {+ hypotheses: [+ {+ hypothesis: "Normal account activity with misleading userId.",+ confidence: 0.99,+ evidence: "Some evidence.",+ },+ ],+ facts: [+ {+ fact: "Fact 1.",+ evidence: "search_fact_by_key, get_records_by_ids identity and human.",+ },+ {+ fact: "Fact 2",+ evidence:+ "answer_access_event_question acting-identity two-window lookup and trigger.",+ },+ ],+ dead_ends: [+ { hypothesis: "A.", evidence: "Evidence A." },+ { hypothesis: "B", evidence: "Evidence B" },+ ],+ next_step: {+ hypothesis:+ "No outstanding discriminator-changing test; record the benign account-compromise verdict.",+ evidence: "All five required checks attempted or not applicable",+ verdict: "Benign; the account-compromise match requirement is not met.",+ severity: "Not applicable",+ confidence: 0.99,+ },+};+// Anonymized customer payload; prose, names, IDs, and preview content replaced.export const customerFixtures: NormalizedIOFixture[] = [+ {+ name: "anonymized customer Responses structured output text with tool call",+ spanIO: {+ input: undefined,+ metadata: undefined,+ output: {+ output: [+ {+ id: "rs_sample_review",+ summary: [],+ type: "reasoning",+ content: [],+ encrypted_content: "gAAAAAB",+ },+ {+ id: "msg_sample_review",+ content: [+ {+ annotations: [],+ text: structuredReviewOutput,+ type: "output_text",+ logprobs: [],+ },+ ],+ role: "assistant",+ status: "completed",+ type: "message",+ phase: "commentary",+ },+ {+ arguments: {+ label: "Benign",+ scenario: "account_compromise",+ summary: "## Summary",+ },+ call_id: "call_sample_review",+ name: "conclude_benign_review",+ type: "function_call",+ id: "fc_sample_review",+ caller: null,+ namespace: null,+ status: "completed",+ },+ ],+ },+ },+ expected: {+ messages: [+ {+ source: "output",+ role: "assistant",+ id: "rs_sample_review",+ parts: [+ {+ type: "reasoning",+ content: { kind: "encrypted", data: "gAAAAAB" },+ },+ ],+ },+ {+ source: "output",+ role: "assistant",+ id: "msg_sample_review",+ parts: [+ {+ type: "data",+ value: structuredReviewOutput,+ providerMetadata: { annotations: [], logprobs: [] },+ },+ ],+ },+ {+ source: "output",+ role: "assistant",+ parts: [+ {+ type: "tool-call",+ toolCallId: "call_sample_review",+ toolName: "conclude_benign_review",+ input: {+ label: "Benign",+ scenario: "account_compromise",+ summary: "## Summary",+ },+ toolType: "function_call",+ providerMetadata: {+ caller: null,+ namespace: null,+ status: "completed",+ },+ },+ ],+ },+ ],+ toolDefinitions: [],+ },+ },{name: "anonymized customer Responses tool history",spanIO: {diff --git a/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.ts b/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.tsindex 2a449eb4f0..b2c6b28c1f 100644--- a/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.ts+++ b/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.ts@@ -1,6 +1,12 @@import { registeredProviders } from "../../../conventions";import type { JsonValue, NormalizedMessagePart } from "../../../types";-import { optionalString, parseArray, toJsonValue } from "../../utils/json";+import {+ omitKeys,+ optionalString,+ parseArray,+ toJsonValue,+ toProviderMetadata,+} from "../../utils/json";import { reasoningPart } from "./reasoning";/** Citation carriers are provider vocabulary (`citationKeys`); the first@@ -23,13 +29,35 @@ export function extractCitations(export function normalizeTextPart(value: Record<string, unknown>,): NormalizedMessagePart {- const text = optionalString(value.text ?? value.content) ?? "";+ const content = value.text ?? value.content;+ const text = optionalString(content) ?? "";if (value.thought === true) {return reasoningPart(text,optionalString(value.thoughtSignature ?? value.thought_signature),);}+ // LiteLLM+OpenAI Instrumentation may store parsed JSON in a text block's payload.+ if (+ content !== undefined &&+ content !== null &&+ typeof content !== "string"+ ) {+ const providerMetadata = toProviderMetadata(+ omitKeys(value, [+ "type",+ "text",+ "content",+ "providerMetadata",+ "providerOptions",+ ]),+ );+ return {+ type: "data",+ value: toJsonValue(content),+ ...(providerMetadata ? { providerMetadata } : {}),+ };+ }const citations = extractCitations(value);return {type: "text",