Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(openai): add structured review output and enhance customer response fixtures (#18342)

marliessophie23d175a45b6b
diff --git a/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts b/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts
index 3931ba8b7e..eef217ea42 100644
--- a/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts
+++ b/packages/shared/src/utils/normalized-io/conventions/providers/openai/fixtures.ts
@@ -55,8 +55,138 @@ export const documentedResponsesFixtures: NormalizedIOFixture[] = [
},
];
+const structuredReviewOutput = {
+ hypotheses: [
+ {
+ hypothesis: "Normal account activity with misleading userId.",
+ confidence: 0.99,
+ evidence: "Some evidence.",
+ },
+ ],
+ facts: [
+ {
+ fact: "Fact 1.",
+ evidence: "search_fact_by_key, get_records_by_ids identity and human.",
+ },
+ {
+ fact: "Fact 2",
+ evidence:
+ "answer_access_event_question acting-identity two-window lookup and trigger.",
+ },
+ ],
+ dead_ends: [
+ { hypothesis: "A.", evidence: "Evidence A." },
+ { hypothesis: "B", evidence: "Evidence B" },
+ ],
+ next_step: {
+ hypothesis:
+ "No outstanding discriminator-changing test; record the benign account-compromise verdict.",
+ evidence: "All five required checks attempted or not applicable",
+ verdict: "Benign; the account-compromise match requirement is not met.",
+ severity: "Not applicable",
+ confidence: 0.99,
+ },
+};
+
// Anonymized customer payload; prose, names, IDs, and preview content replaced.
export const customerFixtures: NormalizedIOFixture[] = [
+ {
+ name: "anonymized customer Responses structured output text with tool call",
+ spanIO: {
+ input: undefined,
+ metadata: undefined,
+ output: {
+ output: [
+ {
+ id: "rs_sample_review",
+ summary: [],
+ type: "reasoning",
+ content: [],
+ encrypted_content: "gAAAAAB",
+ },
+ {
+ id: "msg_sample_review",
+ content: [
+ {
+ annotations: [],
+ text: structuredReviewOutput,
+ type: "output_text",
+ logprobs: [],
+ },
+ ],
+ role: "assistant",
+ status: "completed",
+ type: "message",
+ phase: "commentary",
+ },
+ {
+ arguments: {
+ label: "Benign",
+ scenario: "account_compromise",
+ summary: "## Summary",
+ },
+ call_id: "call_sample_review",
+ name: "conclude_benign_review",
+ type: "function_call",
+ id: "fc_sample_review",
+ caller: null,
+ namespace: null,
+ status: "completed",
+ },
+ ],
+ },
+ },
+ expected: {
+ messages: [
+ {
+ source: "output",
+ role: "assistant",
+ id: "rs_sample_review",
+ parts: [
+ {
+ type: "reasoning",
+ content: { kind: "encrypted", data: "gAAAAAB" },
+ },
+ ],
+ },
+ {
+ source: "output",
+ role: "assistant",
+ id: "msg_sample_review",
+ parts: [
+ {
+ type: "data",
+ value: structuredReviewOutput,
+ providerMetadata: { annotations: [], logprobs: [] },
+ },
+ ],
+ },
+ {
+ source: "output",
+ role: "assistant",
+ parts: [
+ {
+ type: "tool-call",
+ toolCallId: "call_sample_review",
+ toolName: "conclude_benign_review",
+ input: {
+ label: "Benign",
+ scenario: "account_compromise",
+ summary: "## Summary",
+ },
+ toolType: "function_call",
+ providerMetadata: {
+ caller: null,
+ namespace: null,
+ status: "completed",
+ },
+ },
+ ],
+ },
+ ],
+ toolDefinitions: [],
+ },
+ },
{
name: "anonymized customer Responses tool history",
spanIO: {
diff --git a/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.ts b/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.ts
index 2a449eb4f0..b2c6b28c1f 100644
--- a/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.ts
+++ b/packages/shared/src/utils/normalized-io/core/normalize/message-parts/text.ts
@@ -1,6 +1,12 @@
import { registeredProviders } from "../../../conventions";
import type { JsonValue, NormalizedMessagePart } from "../../../types";
-import { optionalString, parseArray, toJsonValue } from "../../utils/json";
+import {
+ omitKeys,
+ optionalString,
+ parseArray,
+ toJsonValue,
+ toProviderMetadata,
+} from "../../utils/json";
import { reasoningPart } from "./reasoning";
/** Citation carriers are provider vocabulary (`citationKeys`); the first
@@ -23,13 +29,35 @@ export function extractCitations(
export function normalizeTextPart(
value: Record<string, unknown>,
): NormalizedMessagePart {
- const text = optionalString(value.text ?? value.content) ?? "";
+ const content = value.text ?? value.content;
+ const text = optionalString(content) ?? "";
if (value.thought === true) {
return reasoningPart(
text,
optionalString(value.thoughtSignature ?? value.thought_signature),
);
}
+ // LiteLLM+OpenAI Instrumentation may store parsed JSON in a text block's payload.
+ if (
+ content !== undefined &&
+ content !== null &&
+ typeof content !== "string"
+ ) {
+ const providerMetadata = toProviderMetadata(
+ omitKeys(value, [
+ "type",
+ "text",
+ "content",
+ "providerMetadata",
+ "providerOptions",
+ ]),
+ );
+ return {
+ type: "data",
+ value: toJsonValue(content),
+ ...(providerMetadata ? { providerMetadata } : {}),
+ };
+ }
const citations = extractCitations(value);
return {
type: "text",