Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

43.0 KB
Raw Blame
import { EventType } from "@ag-ui/core";
import { Buffer } from "node:buffer";
import { getInternalTracingHandler, logger } from "@langfuse/shared/src/server";

import {
  getInAppAgentInstrumentationObservationId,
  getInAppAgentInstrumentationTraceId,
  getInAppAgentLlmCallObservationId,
  parseInAppAgentToolApprovalEvent,
  type InAppAgentToolApprovalSource,
} from "@langfuse/shared/in-app-agent";
import type { AgUiEvent, AgUiMessage } from "@langfuse/shared/in-app-agent";
import {
  getToolFailureMessage,
  isRecord,
  normalizeToolOutput,
  parseJsonOrString,
} from "@langfuse/shared/in-app-agent/server/toolErrors";
import { ResumeForwardedPropsSchema, type AgUiRunAgentInput } from "./types";
import { compactTextMessageChunks } from "@langfuse/shared/in-app-agent/server/eventCompaction";
import type { InAppAgentUserAccess } from "@langfuse/shared/in-app-agent/server/mcpPolicy";
import {
  assertUnreachable,
  SkillsAvailableInputSchema,
  type SkillsAvailableInput,
} from "@langfuse/shared";

export type InAppAgentTracingConfig = {
  environment: string;
  metadata: Record<string, unknown>;
  user: {
    id: string;
    email?: string | null;
    projectRole?: InAppAgentUserAccess["projectRole"];
    // Global Langfuse admin flag. This bypasses project membership checks.
    isAdmin: boolean;
  };
  runId: string;
  runStartedAt: Date;
  targetProjectId: string;
  prompt?: InAppAgentPromptMetadata;
};

export type InAppAgentInstrumentationParams = {
  input: AgUiRunAgentInput;
  tracing?: InAppAgentTracingConfig;
  model?: string;
};

export type InAppAgentPromptMetadata = {
  name: string;
  version: number;
};

const IN_APP_AGENT_TURN_NAME = "agent-turn";
const IN_APP_AGENT_LLM_CALL_NAME = "invoke-model";
type InternalTracingHandler = ReturnType<typeof getInternalTracingHandler>;
type AgentRunToolCall = {
  id: string;
  name: string;
  arguments: string;
  type: "function";
};
type AgentRunToolDefinition = {
  type: "function";
  function: {
    name: string;
    description?: string;
    parameters?: unknown;
  };
};
type AgentRunSkillDefinition = {
  name: string;
  description?: string;
};
// Matches the ChatML thinking content part
// (packages/shared/src/utils/IORepresentation/chatML/types.ts) so the trace UI
// renders reasoning as collapsible "Thinking" blocks.
type AgentRunThinkingPart = {
  type: "thinking";
  content: string;
};
type AgentRunChatMessage = {
  role: string;
  name?: string;
  content?: unknown;
  thinking?: AgentRunThinkingPart[];
  tool_calls?: AgentRunToolCall[];
  tool_call_id?: string;
};
type ObservationBody = Parameters<
  InternalTracingHandler["recordObservation"]
>[1];
type ToolCallApprovalStatus = "approved" | "rejected";
type AgentRunToolSpan = {
  name: string;
  startTime: Date;
  args: string;
  argsComplete: boolean;
  output?: unknown;
  failureMessage?: string;
  parentMessageId?: string;
};
type ModelToolCall = {
  toolCallId: string;
  toolName: string;
  args: unknown;
};
type OpenModelCall = {
  callNumber: number;
  startTime: Date;
  input: unknown;
  completionStartTime?: Date;
  textDeltas: string[];
  toolCalls: ModelToolCall[];
  modelId?: string;
  streamErrorMessage?: string;
};
type ToolExecutionTimes = {
  startTime?: Date;
  endTime?: Date;
};
type ApprovalContinuation = {
  status: ToolCallApprovalStatus;
  continuationNumber: number;
  rootRunId: string;
  toolCallId: string;
  toolName: string;
  traceStartedAt?: Date;
  approvalRequestedAt?: Date;
  approvalDecidedAt?: Date;
  metadata: Record<string, unknown>;
};

export function createInAppAgentInstrumentation({
  input,
  tracing,
  model,
}: InAppAgentInstrumentationParams) {
  if (!tracing?.targetProjectId) {
    return undefined;
  }

  try {
    return new InAppAgentInstrumentation({
      input,
      metadata: tracing.metadata,
      userId: tracing.user.id,
      userEmail: tracing.user.email,
      userProjectRole: tracing.user.projectRole,
      userIsAdmin: tracing.user.isAdmin,
      runId: tracing.runId,
      runStartedAt: tracing.runStartedAt,
      targetProjectId: tracing.targetProjectId,
      environment: tracing.environment,
      prompt: tracing.prompt,
      model,
    });
  } catch (error) {
    logger.warn("Failed to initialize in-app agent Langfuse tracing", error);
    return undefined;
  }
}

export class InAppAgentInstrumentation {
  private readonly processTracedEvents: () => Promise<void>;
  private flushPromise?: Promise<void>;
  private readonly recordObservation: InternalTracingHandler["recordObservation"];
  private readonly runId: string;
  private readonly traceId: string;
  private readonly rootObservationId: string;
  private readonly userId: string;
  private readonly sessionId: string;
  private readonly traceStartTime: Date;
  private readonly agentRunStartTime: Date;
  private agentRunInput: unknown;
  private readonly approvalContinuation?: ApprovalContinuation;
  private readonly prompt?: InAppAgentPromptMetadata;
  private readonly toolSpans = new Map<string, AgentRunToolSpan>();
  private readonly toolCallApprovals = new Map<
    string,
    ToolCallApprovalStatus
  >();
  private readonly toolCallApprovalSources = new Map<
    string,
    InAppAgentToolApprovalSource
  >();
  private readonly toolExecutionTimes = new Map<string, ToolExecutionTimes>();
  private readonly toolCallToModelObservationId = new Map<string, string>();
  private openModelObservationId?: string;
  private lastModelObservationId?: string;
  private readonly metadata: Record<string, unknown>;
  private readonly targetProjectId: string;
  private skillsAvailable: SkillsAvailableInput = [];
  private readonly agentRunOutputMessages: AgentRunChatMessage[] = [];
  private readonly agentRunToolCalls: AgentRunToolCall[] = [];
  private output = "";
  private currentReasoning = "";
  private pendingThinking: AgentRunThinkingPart[] = [];
  private ended = false;
  private readonly model?: string;
  private nextModelCallNumber = 0;
  private openModelCall?: OpenModelCall;

  constructor(params: {
    input: AgUiRunAgentInput;
    metadata: Record<string, unknown>;
    userId: string;
    userEmail?: string | null;
    userProjectRole?: InAppAgentUserAccess["projectRole"];
    userIsAdmin: boolean;
    runId: string;
    runStartedAt: Date;
    targetProjectId: string;
    environment: string;
    prompt?: InAppAgentPromptMetadata;
    model?: string;
  }) {
    this.approvalContinuation = getApprovalContinuation(params.input);
    this.metadata = {
      ...params.metadata,
      ...(params.userEmail ? { langfuse_user_email: params.userEmail } : {}),
      ...(params.userProjectRole
        ? { langfuse_user_project_role: params.userProjectRole }
        : {}),
      langfuse_user_is_admin: params.userIsAdmin,
      ...(params.prompt
        ? {
            prompt_name: params.prompt.name,
            prompt_version: params.prompt.version,
          }
        : {}),
      ...(this.approvalContinuation
        ? {
            approval_continuation_count:
              this.approvalContinuation.continuationNumber,
          }
        : {}),
    };
    this.agentRunInput = getAgentRunInput(params.input);
    this.prompt = params.prompt;
    this.targetProjectId = params.targetProjectId;
    this.model = params.model;
    this.runId = params.runId;
    this.userId = params.userId;
    this.sessionId = params.input.threadId;
    const rootRunId = this.approvalContinuation?.rootRunId ?? params.runId;
    this.traceId = getInAppAgentInstrumentationTraceId(rootRunId);
    this.rootObservationId =
      getInAppAgentInstrumentationObservationId(rootRunId);
    this.agentRunStartTime = new Date();
    // V4 replacement keys include exact start_time; continuations must reuse
    // the persisted root timestamp rather than instrumentation initialization.
    this.traceStartTime =
      this.approvalContinuation?.traceStartedAt ?? params.runStartedAt;

    const traceSinkParams = {
      targetProjectId: params.targetProjectId,
      traceId: this.traceId,
      traceName: IN_APP_AGENT_TURN_NAME,
      environment: params.environment,
      userId: params.userId,
      sessionId: params.input.threadId,
      metadata: this.metadata,
      prompt: params.prompt,
    };
    const { recordObservation, processTracedEvents } =
      getInternalTracingHandler(traceSinkParams);
    this.processTracedEvents = processTracedEvents;
    this.recordObservation = recordObservation;

    // Create the root observation immediately so tool/generation children
    // emitted during the run have a parent that already exists (avoids
    // orphans that attach to the trace as siblings of the late agent).
    this.emitRootAgentObservation({ metadata: this.metadata });
    this.emitApprovalWaitObservation();
  }

  recordEvents(events: AgUiEvent[]) {
    if (this.ended) {
      return;
    }

    const compactedEvents = compactTextMessageChunks(events);

    for (const event of compactedEvents) {
      this.recordEvent(event);
    }
  }

  recordAvailableTools(tools: Record<string, unknown>) {
    if (this.ended) {
      return;
    }

    const availableTools = getAgentRunAvailableTools(tools);

    if (availableTools.length === 0) {
      return;
    }

    this.agentRunInput = addAvailableToolsToAgentRunInput(
      this.agentRunInput,
      availableTools,
    );
  }

  recordToolCallApproval(approval?: {
    toolCallId: string;
    status: ToolCallApprovalStatus;
  }) {
    if (this.ended || !approval) {
      return;
    }

    this.toolCallApprovals.set(approval.toolCallId, approval.status);
  }

  recordToolExecutionStart(toolCallId: string) {
    if (this.ended) {
      return;
    }

    const times = this.toolExecutionTimes.get(toolCallId) ?? {};
    if (times.startTime === undefined) {
      times.startTime = new Date();
    }
    this.toolExecutionTimes.set(toolCallId, times);
  }

  recordToolExecutionEnd(toolCallId: string) {
    if (this.ended) {
      return;
    }

    const times = this.toolExecutionTimes.get(toolCallId) ?? {};
    if (times.endTime === undefined) {
      times.endTime = new Date();
    }
    this.toolExecutionTimes.set(toolCallId, times);
  }

  recordModelCallStart(options: unknown) {
    if (this.ended) {
      return;
    }

    const now = new Date();
    const callNumber =
      this.openModelCall?.callNumber ?? ++this.nextModelCallNumber;
    this.openModelObservationId = getInAppAgentLlmCallObservationId(
      this.runId,
      callNumber,
    );
    this.openModelCall = {
      callNumber,
      startTime: this.openModelCall?.startTime ?? now,
      input: getModelCallInput(options),
      textDeltas: [],
      toolCalls: [],
    };
  }

  recordModelStreamPart(part: unknown) {
    if (this.ended || !this.openModelCall || !isRecord(part)) {
      return;
    }

    const type = part.type;
    if (typeof type !== "string") {
      return;
    }

    const now = new Date();

    if (type === "text-delta" || type === "reasoning-delta") {
      if (this.openModelCall.completionStartTime === undefined) {
        this.openModelCall.completionStartTime = now;
      }
      if (type === "text-delta" && typeof part.delta === "string") {
        this.openModelCall.textDeltas.push(part.delta);
      }
      return;
    }

    if (type === "tool-call") {
      const toolCallId = getStringValue(part.toolCallId);
      const toolName = getStringValue(part.toolName);
      if (toolCallId && toolName) {
        this.openModelCall.toolCalls.push({
          toolCallId,
          toolName,
          args:
            typeof part.input === "string"
              ? parseJsonOrString(part.input)
              : part.input,
        });
      }
      return;
    }

    if (type === "error") {
      this.openModelCall.streamErrorMessage = getErrorMessage(part.error);
      return;
    }

    if (type === "response-metadata") {
      this.openModelCall.modelId = getStringValue(part.modelId);
      return;
    }

    if (type !== "finish") {
      return;
    }

    this.emitModelGeneration(this.openModelCall, part, now);
    this.openModelCall = undefined;
  }

  recordAvailableSkills(skills: unknown[]) {
    if (this.ended) {
      return;
    }

    const catalogue = SkillsAvailableInputSchema.safeParse(
      skills.flatMap((skill) => {
        if (!isRecord(skill)) return [];
        const metadata = isRecord(skill.metadata) ? skill.metadata : {};
        const reference = SkillsAvailableInputSchema.element.safeParse({
          name: skill.name,
          // Foreign-project skills remain visible without resolving to a local version.
          version:
            metadata.langfuseProjectId === this.targetProjectId
              ? metadata.langfuseSkillVersion
              : undefined,
        });
        return reference.success ? [reference.data] : [];
      }),
    );
    this.skillsAvailable = catalogue.success ? catalogue.data : [];

    const availableSkills = getAgentRunAvailableSkills(skills);

    if (availableSkills.length === 0) {
      return;
    }

    this.agentRunInput = addAvailableSkillsToAgentRunInput(
      this.agentRunInput,
      availableSkills,
    );
  }

  endWithError(error: unknown) {
    if (this.ended) {
      return;
    }

    const message = error instanceof Error ? error.message : String(error);
    this.endOpenModelCall(message);
    this.endOpenToolSpans({ error: message }, message);
    this.flushTrailingThinking();
    this.emitRootAgentObservation({
      level: "ERROR",
      statusMessage: message,
      metadata: {
        ...this.metadata,
        error: message,
      },
    });
    this.ended = true;
  }

  end(params?: { aborted?: boolean; result?: unknown }) {
    if (this.ended) {
      return;
    }

    if (params?.aborted) {
      this.endOpenModelCall("aborted");
    } else {
      this.endOpenModelCall("Model call ended before provider finish");
    }
    this.endOpenToolSpans(params?.aborted ? { aborted: true } : undefined);
    this.flushTrailingThinking();
    const metadata = {
      ...this.metadata,
      ...(params?.aborted ? { aborted: true } : {}),
      ...(params?.result ? { result: params.result } : {}),
    };
    this.emitRootAgentObservation({ metadata });
    this.ended = true;
  }

  flush(): Promise<void> {
    if (this.flushPromise === undefined) {
      this.flushPromise = this.processTracedEvents().catch((error) => {
        logger.warn("Failed to flush in-app agent Langfuse tracing", error);
      });
    }

    return this.flushPromise;
  }

  private recordEvent(event: AgUiEvent) {
    if (
      event.type === EventType.TEXT_MESSAGE_CHUNK ||
      event.type === EventType.TEXT_MESSAGE_CONTENT
    ) {
      if (typeof event.delta === "string") {
        this.recordAssistantText(event.delta);
      }
      return;
    }

    if (
      event.type === EventType.REASONING_MESSAGE_CHUNK ||
      event.type === EventType.REASONING_MESSAGE_CONTENT
    ) {
      if (typeof event.delta === "string") {
        this.currentReasoning += event.delta;
      }
      return;
    }

    if (
      event.type === EventType.REASONING_MESSAGE_START ||
      event.type === EventType.REASONING_MESSAGE_END
    ) {
      this.flushCurrentReasoning();
      return;
    }

    if (event.type === EventType.TOOL_CALL_START) {
      this.startToolSpan(event);
      return;
    }

    const toolApproval = parseInAppAgentToolApprovalEvent(event);
    if (toolApproval) {
      this.toolCallApprovalSources.set(
        toolApproval.toolCallId,
        toolApproval.source,
      );
      return;
    }

    if (event.type === EventType.TOOL_CALL_ARGS) {
      this.appendToolArgs(event);
      return;
    }

    if (event.type === EventType.TOOL_CALL_RESULT) {
      this.recordToolResult(event);
      return;
    }

    if (event.type === EventType.TOOL_CALL_END) {
      this.endToolSpan(event);
      return;
    }

    if (event.type === EventType.RUN_ERROR) {
      this.endWithError(
        typeof event.message === "string"
          ? event.message
          : "Unknown in-app agent error",
      );
      return;
    }

    if (event.type === EventType.RUN_FINISHED) {
      this.end({ result: event.result });
      return;
    }

    if (
      event.type === EventType.RUN_STARTED ||
      event.type === EventType.TEXT_MESSAGE_START ||
      event.type === EventType.TEXT_MESSAGE_END ||
      event.type === EventType.STATE_SNAPSHOT ||
      event.type === EventType.STATE_DELTA ||
      event.type === EventType.MESSAGES_SNAPSHOT ||
      event.type === EventType.ACTIVITY_SNAPSHOT ||
      event.type === EventType.ACTIVITY_DELTA ||
      event.type === EventType.RAW ||
      event.type === EventType.CUSTOM ||
      event.type === EventType.STEP_STARTED ||
      event.type === EventType.STEP_FINISHED ||
      event.type === EventType.TOOL_CALL_CHUNK ||
      event.type === EventType.REASONING_START ||
      event.type === EventType.REASONING_END ||
      event.type === EventType.REASONING_ENCRYPTED_VALUE ||
      // eslint-disable-next-line @typescript-eslint/no-deprecated
      event.type === EventType.THINKING_START ||
      // eslint-disable-next-line @typescript-eslint/no-deprecated
      event.type === EventType.THINKING_END ||
      // eslint-disable-next-line @typescript-eslint/no-deprecated
      event.type === EventType.THINKING_TEXT_MESSAGE_START ||
      // eslint-disable-next-line @typescript-eslint/no-deprecated
      event.type === EventType.THINKING_TEXT_MESSAGE_CONTENT ||
      // eslint-disable-next-line @typescript-eslint/no-deprecated
      event.type === EventType.THINKING_TEXT_MESSAGE_END
    ) {
      return;
    }

    return assertUnreachable(event.type);
  }

  private startToolSpan(event: AgUiEvent) {
    if (typeof event.toolCallId !== "string") {
      return;
    }

    const name =
      typeof event.toolCallName === "string" ? event.toolCallName : "tool-call";
    // Execution start = AG-UI tool-call start (or later clamp to generation end).
    // Mastra tool-call chunks are request-time and must not set this.
    this.toolSpans.set(event.toolCallId, {
      name,
      startTime: new Date(),
      args: "",
      argsComplete: false,
      ...(typeof event.parentMessageId === "string"
        ? { parentMessageId: event.parentMessageId }
        : {}),
    });
  }

  private appendToolArgs(event: AgUiEvent) {
    if (
      typeof event.toolCallId !== "string" ||
      typeof event.delta !== "string"
    ) {
      return;
    }

    const tool = this.toolSpans.get(event.toolCallId);
    if (tool) {
      tool.args += event.delta;
    }
  }

  private recordToolResult(event: AgUiEvent) {
    if (typeof event.toolCallId !== "string") {
      return;
    }

    const tool = this.toolSpans.get(event.toolCallId);
    if (tool) {
      tool.output = event.content;
      tool.failureMessage = getToolFailureMessage(event.error, event.content);
      this.endToolSpanIfComplete(event.toolCallId, tool);
    }
  }

  private endToolSpan(event: AgUiEvent) {
    if (typeof event.toolCallId !== "string") {
      return;
    }

    const tool = this.toolSpans.get(event.toolCallId);
    if (!tool) {
      return;
    }

    tool.argsComplete = true;
    this.endToolSpanIfComplete(event.toolCallId, tool);
  }

  private endToolSpanIfComplete(toolCallId: string, tool: AgentRunToolSpan) {
    if (!tool.argsComplete || tool.output === undefined) {
      return;
    }

    this.createToolObservation(toolCallId, tool);
    this.toolSpans.delete(toolCallId);
  }

  private createToolObservation(
    toolCallId: string,
    tool: AgentRunToolSpan,
    options?: {
      metadata?: Record<string, unknown>;
      statusMessage?: string;
    },
  ) {
    const input = parseJsonOrUndefined(tool.args);
    const output =
      tool.output === undefined ? undefined : normalizeToolOutput(tool.output);
    const failureMessage =
      options?.statusMessage ??
      tool.failureMessage ??
      getToolFailureMessage(undefined, output);
    const toolCallApproval = this.toolCallApprovals.get(toolCallId);
    const toolCallApprovalSource = this.toolCallApprovalSources.get(toolCallId);
    const executionTimes = this.toolExecutionTimes.get(toolCallId);
    const rawStartTime = executionTimes?.startTime ?? tool.startTime;
    const endTime =
      toolCallApproval === "rejected"
        ? rawStartTime
        : (executionTimes?.endTime ?? new Date());
    const startTime =
      rawStartTime.getTime() > endTime.getTime() ? endTime : rawStartTime;
    const body: ObservationBody = {
      id: toolCallId,
      traceId: this.traceId,
      parentObservationId: this.getToolParentObservationId(toolCallId),
      name: tool.name,
      startTime,
      endTime,
      completionStartTime: startTime,
      input,
      output:
        output === undefined
          ? undefined
          : {
              role: "tool",
              name: tool.name,
              tool_call_id: toolCallId,
              content: output,
            },
      ...(failureMessage
        ? { level: "ERROR", statusMessage: failureMessage }
        : {}),
      metadata: {
        ...(options?.metadata ?? {}),
        toolCallId,
        ...(toolCallApproval ? { toolCallApproval } : {}),
        ...(toolCallApprovalSource ? { toolCallApprovalSource } : {}),
        ...(tool.argsComplete ? {} : { argsComplete: false }),
        ...(tool.parentMessageId
          ? { parentMessageId: tool.parentMessageId }
          : {}),
      },
    };

    this.recordToolCall(toolCallId, tool, output);
    this.toolCallApprovals.delete(toolCallId);
    this.toolCallApprovalSources.delete(toolCallId);
    this.toolExecutionTimes.delete(toolCallId);

    this.enqueueObservation("TOOL", body);
  }

  private emitRootAgentObservation(params: {
    metadata: Record<string, unknown>;
    level?: "ERROR";
    statusMessage?: string;
  }) {
    this.enqueueObservation("AGENT", {
      id: this.rootObservationId,
      traceId: this.traceId,
      name: IN_APP_AGENT_TURN_NAME,
      tags: ["in-app-agent"],
      startTime: this.traceStartTime,
      endTime: new Date(),
      input: this.getTraceInput(),
      output: this.getAgentRunOutput(),
      metadata: params.metadata,
      ...(this.prompt
        ? {
            promptName: this.prompt.name,
            promptVersion: this.prompt.version,
          }
        : {}),
      ...(params.level ? { level: params.level } : {}),
      ...(params.statusMessage ? { statusMessage: params.statusMessage } : {}),
    });
  }

  private emitModelGeneration(
    call: OpenModelCall,
    finish: Record<string, unknown> | undefined,
    endTime: Date,
    options?: {
      level?: "ERROR";
      statusMessage?: string;
    },
  ) {
    const usageDetails = getModelUsageDetails(finish?.usage);
    const finishReason = getModelFinishReason(finish);
    const model = call.modelId ?? this.model;
    const id = getInAppAgentLlmCallObservationId(this.runId, call.callNumber);
    this.lastModelObservationId = id;
    this.openModelObservationId = undefined;
    for (const toolCall of call.toolCalls) {
      this.toolCallToModelObservationId.set(toolCall.toolCallId, id);
    }
    const text = call.textDeltas.join("");
    const output =
      text || call.toolCalls.length > 0
        ? {
            ...(text ? { text } : {}),
            ...(call.toolCalls.length > 0
              ? { tool_calls: call.toolCalls }
              : {}),
          }
        : undefined;

    this.enqueueObservation("GENERATION", {
      id,
      traceId: this.traceId,
      parentObservationId: this.rootObservationId,
      name: IN_APP_AGENT_LLM_CALL_NAME,
      startTime: call.startTime,
      endTime,
      ...(call.completionStartTime
        ? { completionStartTime: call.completionStartTime }
        : {}),
      input: call.input,
      output,
      // Model only travels with usage so Langfuse does not invent tokenizer-
      // estimated costs for generations that never reported tokens.
      ...(usageDetails
        ? {
            usageDetails,
            ...(model ? { model } : {}),
          }
        : {}),
      ...(options?.level ? { level: options.level } : {}),
      ...(options?.statusMessage
        ? { statusMessage: options.statusMessage }
        : {}),
      metadata: {
        ...(finishReason ? { finish_reason: finishReason } : {}),
        ...(options?.statusMessage ? { error: options.statusMessage } : {}),
      },
    });
  }

  private getTraceInput() {
    if (!this.approvalContinuation || !isRecord(this.agentRunInput)) {
      return this.agentRunInput;
    }

    const messages = Array.isArray(this.agentRunInput.messages)
      ? this.agentRunInput.messages.filter((message) => {
          if (!isRecord(message)) {
            return false;
          }

          return (
            message.role === "developer" ||
            message.role === "system" ||
            message.role === "user"
          );
        })
      : [];

    return {
      ...this.agentRunInput,
      messages,
    };
  }

  private endOpenModelCall(statusMessage: string) {
    if (!this.openModelCall) {
      return;
    }

    const message = this.openModelCall.streamErrorMessage ?? statusMessage;
    this.emitModelGeneration(this.openModelCall, undefined, new Date(), {
      level: "ERROR",
      statusMessage: message,
    });
    this.openModelCall = undefined;
  }

  private endOpenToolSpans(
    metadata?: Record<string, unknown>,
    statusMessage?: string,
  ) {
    // Approval interrupts complete the stream without a tool result. Drop those
    // incomplete spans — the generation already records the tool_call request,
    // and the continuation run emits the real tool observation after approval.
    const forceClose =
      Boolean(statusMessage) ||
      Boolean(metadata?.aborted) ||
      Boolean(metadata?.error);

    for (const [toolCallId, tool] of this.toolSpans.entries()) {
      if (tool.output === undefined && !forceClose) {
        this.toolSpans.delete(toolCallId);
        this.toolExecutionTimes.delete(toolCallId);
        continue;
      }

      this.createToolObservation(toolCallId, tool, {
        metadata,
        statusMessage,
      });
      this.toolSpans.delete(toolCallId);
    }
  }

  private getToolParentObservationId(toolCallId: string): string {
    return (
      this.toolCallToModelObservationId.get(toolCallId) ??
      this.openModelObservationId ??
      this.lastModelObservationId ??
      this.rootObservationId
    );
  }

  private enqueueObservation(
    type: Parameters<InternalTracingHandler["recordObservation"]>[0],
    body: ObservationBody,
  ) {
    this.recordObservation(type, {
      ...body,
      ...((type === "GENERATION" || type === "TOOL") &&
      this.skillsAvailable.length > 0
        ? { skillsAvailable: this.skillsAvailable }
        : {}),
      userId: this.userId,
      sessionId: this.sessionId,
      metadata: { ...this.metadata, ...body.metadata },
    });
  }

  // Reasoning that no assistant message followed (e.g. aborted or errored
  // turns) still gets recorded as its own thinking-only assistant message.
  private flushTrailingThinking() {
    const thinking = this.takePendingThinking();
    if (!thinking) {
      return;
    }

    this.agentRunOutputMessages.push({
      role: "assistant",
      content: "",
      thinking,
    });
  }

  private flushCurrentReasoning() {
    if (!this.currentReasoning) {
      return;
    }

    this.pendingThinking.push({
      type: "thinking",
      content: this.currentReasoning,
    });
    this.currentReasoning = "";
  }

  // Reasoning always precedes the assistant action it produced, so pending
  // thinking parts attach to the next assistant output message.
  private takePendingThinking(): AgentRunThinkingPart[] | undefined {
    this.flushCurrentReasoning();

    if (this.pendingThinking.length === 0) {
      return undefined;
    }

    return this.pendingThinking.splice(0);
  }

  private recordAssistantText(delta: string) {
    this.output += delta;

    const thinking = this.takePendingThinking();
    const lastMessage = this.agentRunOutputMessages.at(-1);
    if (
      !thinking &&
      lastMessage?.role === "assistant" &&
      !lastMessage.tool_calls?.length
    ) {
      lastMessage.content = `${typeof lastMessage.content === "string" ? lastMessage.content : ""}${delta}`;
      return;
    }

    this.agentRunOutputMessages.push({
      role: "assistant",
      content: delta,
      ...(thinking ? { thinking } : {}),
    });
  }

  private recordToolCall(
    toolCallId: string,
    tool: {
      name: string;
      args: string;
      startTime: Date;
    },
    output: unknown,
  ) {
    const toolCall: AgentRunToolCall = {
      id: toolCallId,
      name: tool.name,
      arguments: tool.args || "{}",
      type: "function",
    };

    const thinking = this.takePendingThinking();
    this.agentRunToolCalls.push(toolCall);
    this.agentRunOutputMessages.push({
      role: "assistant",
      content: "",
      ...(thinking ? { thinking } : {}),
      tool_calls: [toolCall],
    });

    if (output !== undefined) {
      this.agentRunOutputMessages.push({
        role: "tool",
        tool_call_id: toolCallId,
        content: output,
      });
    }
  }

  private getAgentRunOutput() {
    const messages = this.getAgentRunOutputMessages();

    if (messages.length === 0) {
      return undefined;
    }

    const priorMessages = messages.slice(
      0,
      messages.length - this.agentRunOutputMessages.length,
    );
    const priorToolCalls = priorMessages.flatMap(
      (message) => message.tool_calls ?? [],
    );
    const toolCalls = [...priorToolCalls, ...this.agentRunToolCalls];

    return {
      messages,
      ...(this.output ? { text: this.output } : {}),
      ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
    };
  }

  private getAgentRunOutputMessages(): AgentRunChatMessage[] {
    const priorMessages =
      this.approvalContinuation && isRecord(this.agentRunInput)
        ? getPriorAgentRunOutputMessages(this.agentRunInput.messages)
        : [];

    const messages = [...priorMessages, ...this.agentRunOutputMessages];
    return messages.map((message) => {
      if (message.role !== "tool") {
        return message;
      }

      return {
        ...message,
        content: `Output recorded in TOOL observation ${message.tool_call_id}.`,
      };
    });
  }

  private emitApprovalWaitObservation() {
    const continuation = this.approvalContinuation;
    if (!continuation) {
      return;
    }

    const requestedAt =
      continuation.approvalRequestedAt ??
      continuation.approvalDecidedAt ??
      this.agentRunStartTime;
    const decidedAt = continuation.approvalDecidedAt ?? this.agentRunStartTime;
    const endTime =
      decidedAt.getTime() < requestedAt.getTime() ? requestedAt : decidedAt;

    this.enqueueObservation("SPAN", {
      id: `${this.runId}-approval-wait`,
      traceId: this.traceId,
      parentObservationId: this.rootObservationId,
      name: `waiting for user approval for tool ${continuation.toolName}`,
      startTime: requestedAt,
      endTime,
      input: {
        tool_call_id: continuation.toolCallId,
        tool_name: continuation.toolName,
      },
      output: `User ${continuation.status} tool ${continuation.toolName}`,
      metadata: continuation.metadata,
    });
  }
}

function getApprovalContinuation(
  input: AgUiRunAgentInput,
): ApprovalContinuation | undefined {
  const forwardedProps = ResumeForwardedPropsSchema.safeParse(
    input.forwardedProps,
  );

  if (!forwardedProps.success) {
    return undefined;
  }

  const {
    approved,
    approvalRequest,
    continuationNumber = 1,
    rootRunId = approvalRequest.runId,
    traceStartedAt,
    approvalRequestedAt,
    approvalDecidedAt,
  } = forwardedProps.data.command.resume;
  const status = approved ? "approved" : "rejected";

  return {
    status,
    continuationNumber,
    rootRunId,
    toolCallId: approvalRequest.toolCallId,
    toolName: approvalRequest.toolName,
    ...(traceStartedAt ? { traceStartedAt: new Date(traceStartedAt) } : {}),
    ...(approvalRequestedAt
      ? { approvalRequestedAt: new Date(approvalRequestedAt) }
      : {}),
    ...(approvalDecidedAt
      ? { approvalDecidedAt: new Date(approvalDecidedAt) }
      : {}),
    metadata: {
      continuation_type: "tool_approval",
      continuation_number: continuationNumber,
      parent_run_id: approvalRequest.runId,
      approval_status: status,
      approval_tool_call_id: approvalRequest.toolCallId,
      approval_tool_name: approvalRequest.toolName,
      ...(approvalRequestedAt
        ? { approval_requested_at: approvalRequestedAt }
        : {}),
      ...(approvalDecidedAt ? { approval_decided_at: approvalDecidedAt } : {}),
    },
  };
}

function getPriorAgentRunOutputMessages(
  messages: unknown,
): AgentRunChatMessage[] {
  if (!Array.isArray(messages)) {
    return [];
  }

  return messages.filter(
    (message): message is AgentRunChatMessage =>
      isRecord(message) &&
      (message.role === "assistant" || message.role === "tool"),
  );
}

function getAgentRunInput(input: AgUiRunAgentInput): unknown {
  const messages = getAgentRunMessages(getCurrentTurnMessages(input.messages));
  const context = getAgentRunContext(input);

  if (!context) {
    return { messages };
  }

  return {
    messages,
    context,
  };
}

function getCurrentTurnMessages(messages: AgUiMessage[]): AgUiMessage[] {
  const lastUserMessageIndex = messages.findLastIndex(
    (message) => message.role === "user",
  );

  if (lastUserMessageIndex === -1) {
    return messages;
  }

  return messages.filter(
    (message, index) =>
      message.role === "developer" ||
      message.role === "system" ||
      index >= lastUserMessageIndex,
  );
}

function addAvailableToolsToAgentRunInput(
  input: unknown,
  tools: AgentRunToolDefinition[],
) {
  if (!isRecord(input)) {
    return input;
  }

  return {
    ...input,
    tools,
  };
}

function addAvailableSkillsToAgentRunInput(
  input: unknown,
  skills: AgentRunSkillDefinition[],
) {
  if (!isRecord(input)) {
    return input;
  }

  return {
    ...input,
    skills,
  };
}

function getAgentRunAvailableTools(
  tools: Record<string, unknown>,
): AgentRunToolDefinition[] {
  return Object.entries(tools).map(([name, tool]) => {
    const toolRecord = isRecord(tool) ? tool : {};
    const description = getStringValue(toolRecord.description);
    const parameters = getSerializableToolParameters(toolRecord);

    return {
      type: "function" as const,
      function: {
        name,
        ...(description ? { description } : {}),
        ...(parameters ? { parameters } : {}),
      },
    };
  });
}

function getAgentRunAvailableSkills(
  skills: unknown[],
): AgentRunSkillDefinition[] {
  return skills.flatMap((skill) => {
    const skillRecord = isRecord(skill) ? skill : {};
    const name = getStringValue(skillRecord.name);

    if (!name) {
      return [];
    }

    const description = getStringValue(skillRecord.description);

    return [
      {
        name,
        ...(description ? { description } : {}),
      },
    ];
  });
}

function getAgentRunMessages(messages: AgUiMessage[]): AgentRunChatMessage[] {
  return messages.flatMap((message): AgentRunChatMessage[] => {
    if (message.role === "developer" || message.role === "system") {
      return [
        {
          role: message.role,
          ...(message.name ? { name: message.name } : {}),
          content: message.content,
        },
      ];
    }

    if (message.role === "user") {
      return [
        {
          role: "user",
          ...(message.name ? { name: message.name } : {}),
          content: normalizeUserMessageContent(message.content),
        },
      ];
    }

    if (message.role === "assistant") {
      const toolCalls = message.toolCalls?.map((toolCall) => ({
        id: toolCall.id,
        name: toolCall.function.name,
        arguments: toolCall.function.arguments,
        type: "function" as const,
      }));

      if (!message.content && !toolCalls?.length) {
        return [];
      }

      return [
        {
          role: "assistant",
          ...(message.name ? { name: message.name } : {}),
          content: message.content ?? "",
          ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),
        },
      ];
    }

    if (message.role === "tool") {
      return [
        {
          role: "tool",
          tool_call_id: message.toolCallId,
          content: parseJsonOrString(message.content),
        },
      ];
    }

    if (message.role === "activity" || message.role === "reasoning") {
      return [];
    }

    return assertUnreachable(message);
  });
}

function normalizeUserMessageContent(
  content: Extract<AgUiMessage, { role: "user" }>["content"],
): unknown {
  if (typeof content === "string") {
    return content;
  }

  if (content.every((part) => part.type === "text")) {
    return content.map((part) => part.text).join("");
  }

  return content;
}

function getAgentRunContext(input: AgUiRunAgentInput) {
  if (input.context.length === 0) {
    return undefined;
  }

  return Object.fromEntries(
    input.context.map((item) => [
      item.description,
      parseContextValue(item.description, item.value),
    ]),
  );
}

function parseContextValue(description: string, value: string): unknown {
  const parsed = parseJsonOrString(value);

  if (description === "browser_languages" && typeof parsed === "string") {
    return parsed
      .split(",")
      .map((language) => language.trim())
      .filter(Boolean);
  }

  return parsed;
}

function getFiniteNumber(value: unknown): number | undefined {
  return typeof value === "number" && Number.isFinite(value)
    ? value
    : undefined;
}

function getModelCallInput(options: unknown): unknown {
  if (!isRecord(options)) {
    return undefined;
  }

  const serializedOptions = toSerializableJson({
    maxOutputTokens: options.maxOutputTokens,
    temperature: options.temperature,
    stopSequences: options.stopSequences,
    topP: options.topP,
    topK: options.topK,
    presencePenalty: options.presencePenalty,
    frequencyPenalty: options.frequencyPenalty,
    responseFormat: options.responseFormat,
    seed: options.seed,
    tools: options.tools,
    toolChoice: options.toolChoice,
    providerOptions: options.providerOptions,
  });

  // Only change telemetry, never the provider-bound prompt. Stable tool-result
  // IDs let session transcripts attach the full TOOL observation output.
  const messages = Array.isArray(options.prompt)
    ? options.prompt.map((message: unknown) => {
        if (!isRecord(message) || message.role !== "tool") {
          return message;
        }

        if (!Array.isArray(message.content)) {
          return {
            ...message,
            content: "Output recorded in the corresponding TOOL observation.",
          };
        }

        return {
          ...message,
          content: message.content.map((part: unknown) => {
            if (!isRecord(part) || part.type !== "tool-result") {
              return part;
            }

            // Allowlist fields: Mastra also duplicates the payload in
            // providerOptions.mastra.modelOutput.
            return {
              type: part.type,
              toolCallId: part.toolCallId,
              toolName: part.toolName,
              output: {
                type: "text",
                value: `Output recorded in TOOL observation ${part.toolCallId}.`,
              },
            };
          }),
        };
      })
    : options.prompt;
  return {
    ...(messages === undefined ? {} : { messages }),
    ...(isRecord(serializedOptions) ? serializedOptions : {}),
  };
}

function getModelUsageDetails(
  usage: unknown,
): Record<string, number> | undefined {
  if (!isRecord(usage)) {
    return undefined;
  }

  const nestedInputTokens = isRecord(usage.inputTokens)
    ? usage.inputTokens
    : undefined;
  const nestedOutputTokens = isRecord(usage.outputTokens)
    ? usage.outputTokens
    : undefined;
  const inputTokens =
    getFiniteNumber(usage.inputTokens) ??
    getFiniteNumber(nestedInputTokens?.total);
  const outputTokens =
    getFiniteNumber(usage.outputTokens) ??
    getFiniteNumber(nestedOutputTokens?.total);
  if (inputTokens === undefined && outputTokens === undefined) {
    return undefined;
  }

  const cacheReadTokens =
    getFiniteNumber(usage.cachedInputTokens) ??
    getFiniteNumber(nestedInputTokens?.cacheRead) ??
    0;
  const cacheWriteTokens =
    getFiniteNumber(usage.cacheCreationInputTokens) ??
    getFiniteNumber(nestedInputTokens?.cacheWrite) ??
    0;
  const uncachedInputTokens = getFiniteNumber(nestedInputTokens?.noCache);

  // Provider inputTokens includes cache reads/writes, while Langfuse model
  // prices bill `input` as non-cached input tokens with separate cache keys.
  return {
    input:
      uncachedInputTokens ??
      Math.max(0, (inputTokens ?? 0) - cacheReadTokens - cacheWriteTokens),
    output: outputTokens ?? 0,
    cache_read_input_tokens: cacheReadTokens,
    cache_creation_input_tokens: cacheWriteTokens,
    total:
      getFiniteNumber(usage.totalTokens) ??
      (inputTokens ?? 0) + (outputTokens ?? 0),
  };
}

function getModelFinishReason(
  event: Record<string, unknown> | undefined,
): string | undefined {
  if (!event) {
    return undefined;
  }

  const finishReason = event.finishReason;
  if (typeof finishReason === "string") {
    return finishReason;
  }

  if (isRecord(finishReason)) {
    return (
      getStringValue(finishReason.unified) ?? getStringValue(finishReason.raw)
    );
  }

  return undefined;
}

function getSerializableToolParameters(tool: Record<string, unknown>): unknown {
  const parameters =
    tool.parameters ??
    tool.inputSchema ??
    tool.parameters_json_schema ??
    tool.input_schema;

  return toSerializableJson(parameters);
}

function toSerializableJson(
  value: unknown,
  seen = new WeakSet<object>(),
  depth = 0,
): unknown {
  if (value instanceof URL) {
    return value.toString();
  }

  if (value instanceof Uint8Array) {
    return Buffer.from(value).toString("base64");
  }

  if (value === null || typeof value === "string") {
    return value;
  }

  if (typeof value === "boolean") {
    return value;
  }

  if (typeof value === "number") {
    return Number.isFinite(value) ? value : undefined;
  }

  if (Array.isArray(value)) {
    if (depth >= 8) {
      return undefined;
    }

    return value
      .map((item) => toSerializableJson(item, seen, depth + 1))
      .filter((item) => item !== undefined);
  }

  if (!isRecord(value)) {
    return undefined;
  }

  if (Object.getPrototypeOf(value) !== Object.prototype) {
    return undefined;
  }

  if (seen.has(value)) {
    return undefined;
  }

  if (depth >= 8) {
    return undefined;
  }

  seen.add(value);

  const entries = Object.entries(value).flatMap(([key, item]) => {
    const serializableItem =
      key === "data" && item instanceof Uint8Array
        ? `data:${getStringValue(value.mediaType) ?? "application/octet-stream"};base64,${Buffer.from(item).toString("base64")}`
        : toSerializableJson(item, seen, depth + 1);

    return serializableItem === undefined
      ? []
      : [[key, serializableItem] as const];
  });

  seen.delete(value);

  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
}

function parseJsonOrUndefined(value: string): unknown {
  if (!value) {
    return undefined;
  }

  return parseJsonOrString(value);
}

function getStringValue(value: unknown): string | undefined {
  return typeof value === "string" && value.trim() ? value : undefined;
}

function getErrorMessage(error: unknown): string {
  if (error instanceof Error) {
    return error.message;
  }

  return typeof error === "string" ? error : "Model stream failed";
}