Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

15.6 KB
Raw Blame
import { type Model, type Observation } from "@langfuse/shared";
import { SpanStatusCode, type Span } from "@opentelemetry/api";
import {
  assembleTranscriptContext,
  buildTracesPath,
  getProductBaseUrl,
  getTracer,
  orderObservations,
  recordDistribution,
  recordIncrement,
  renderGenericTranscript,
  renderTranscript,
  topicsTranscriptConfig,
  transcriptBlockTypes,
  type Transcript,
  type TranscriptContext,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
import { tokenCountAsync } from "../tokenisation/async-usage";

// A fixed tokenizer makes payload sizes comparable across models and projects.
// gpt-4o selects o200k_base, also used by GPT-5 mini and nano.
// This model is only an in-memory tokenizer configuration, never persisted.
const TOKENIZER_MODEL: Model = {
  id: "trace-batch-token-estimate",
  modelName: "gpt-4o",
  tokenizerId: "openai",
  tokenizerConfig: { tokenizerModel: "gpt-4o" },
  createdAt: new Date(0),
  updatedAt: new Date(0),
  projectId: null,
  matchPattern: "",
  startDate: null,
  inputPrice: null,
  outputPrice: null,
  totalPrice: null,
  unit: "TOKENS",
};

const TOKEN_METRICS = [
  "transcript_message_tokens",
  "transcript_current_turn_tokens",
  "transcript_history_tokens",
] as const;

function recordTranscriptDistribution(
  metric: string,
  value: number,
  tags?: { [tag: string]: string | number },
): void {
  if (env.LANGFUSE_TRACE_BATCH_TRANSCRIPT_METRICS_ENABLED !== "true") return;
  if (tags) {
    recordDistribution(`langfuse.trace_batch.${metric}`, value, tags);
    return;
  }
  recordDistribution(`langfuse.trace_batch.${metric}`, value);
}

function recordTokens(
  span: Span,
  metric: (typeof TOKEN_METRICS)[number],
  tokens: number,
) {
  recordTranscriptDistribution(metric, tokens, { tokenizer: "o200k_base" });
  span.setAttribute(`langfuse.trace_batch.${metric}`, tokens);
}

function recordContentAndStructureCounts(
  transcript: Transcript | null,
  observationCount: number,
  span: Span,
) {
  let total = 0;
  let toolResponses = 0;
  let historyMessages = 0;
  let historyParts = 0;
  let currentTurnMessages = 0;
  let currentTurnParts = 0;
  let currentTurnToolCalls = 0;
  let currentTurnToolResults = 0;
  for (const thread of transcript?.threads ?? []) {
    for (const [section, messages] of [
      ["history", thread.conversationHistory],
      ["current_turn", thread.currentTurn.messages],
    ] as const) {
      if (section === "history") historyMessages += messages.length;
      else currentTurnMessages += messages.length;
      for (const message of messages) {
        for (const part of message.parts) {
          if (section === "history") historyParts++;
          else {
            currentTurnParts++;
            if (part.type === "tool-call") currentTurnToolCalls++;
            if (message.role === "tool" || part.type === "tool-result")
              currentTurnToolResults++;
          }
          // Use the same serialized-part basis for numerator and denominator;
          // exclude message wrappers and observation provenance from both.
          const characters = JSON.stringify(part).length;
          total += characters;
          if (message.role === "tool" || part.type === "tool-result") {
            toolResponses += characters;
          }
        }
      }
    }
  }
  for (const [metric, characters] of [
    ["transcript_content_characters", total],
    ["transcript_tool_response_characters", toolResponses],
  ] as const) {
    recordTranscriptDistribution(metric, characters);
    span.setAttribute(`langfuse.trace_batch.${metric}`, characters);
  }
  for (const [metric, count] of [
    ["transcript_observation_count", observationCount],
    ["transcript_history_message_count", historyMessages],
    ["transcript_history_part_count", historyParts],
    ["transcript_current_turn_message_count", currentTurnMessages],
    ["transcript_current_turn_part_count", currentTurnParts],
    ["transcript_current_turn_tool_call_count", currentTurnToolCalls],
    ["transcript_current_turn_tool_result_count", currentTurnToolResults],
  ] as const) {
    recordTranscriptDistribution(metric, count);
    span.setAttribute(`langfuse.trace_batch.${metric}`, count);
  }
}

async function recordTokenEstimates(transcript: Transcript, span: Span) {
  const currentTurn = transcript.threads.flatMap(
    ({ currentTurn }) => currentTurn.messages,
  );
  const history = transcript.threads.flatMap(
    ({ conversationHistory }) => conversationHistory,
  );
  let total = 0;
  let unavailable = false;
  // Count each message in just one partition. Submit sequentially to keep one
  // pending tokenizer request per batch in the pool shared with ingestion.
  for (const [metric, messages] of [
    ["transcript_current_turn_tokens", currentTurn],
    ["transcript_history_tokens", history],
  ] as const) {
    const tokens = messages.length
      ? await tokenCountAsync({
          model: TOKENIZER_MODEL,
          // Both partitions exclude provenance and use identical JSON framing.
          text: {
            messages: messages.map(({ role, parts }) => ({ role, parts })),
          },
        })
      : 0;
    if (tokens === undefined) {
      if (!unavailable) {
        recordIncrement("langfuse.trace_batch.token_estimation_unavailable", 1);
        span.setAttribute(
          "langfuse.trace_batch.token_estimation",
          "unavailable",
        );
      }
      unavailable = true;
    } else {
      recordTokens(span, metric, tokens);
      total += tokens;
    }
  }
  if (!unavailable) {
    recordTokens(span, "transcript_message_tokens", total);
  }
}

function recordTopicsRendering(
  transcript: Transcript | null,
  observations: Observation[],
  span: Span,
): string | undefined {
  try {
    const { text, stats } = renderTranscript(
      transcript,
      observations,
      topicsTranscriptConfig,
    );
    const metric = (name: string, value: number) => {
      recordTranscriptDistribution(name, value);
      span.setAttribute(`langfuse.trace_batch.${name}`, value);
    };
    metric("topics_transcript_characters", text.length);
    metric("topics_transcript_blocks_cut", stats.blocksCut);
    metric("topics_transcript_history_characters", stats.historyCharacters);
    metric(
      "topics_transcript_current_turn_characters",
      stats.currentTurnCharacters,
    );
    const messageCharacters =
      stats.historyCharacters + stats.currentTurnCharacters;
    metric(
      "topics_transcript_history_share",
      messageCharacters ? stats.historyCharacters / messageCharacters : 0,
    );
    for (const block of transcriptBlockTypes) {
      if (stats.blockCharacters[block].raw === 0) continue;
      for (const stage of ["raw", "clipped"] as const) {
        const characters = stats.blockCharacters[block][stage];
        recordTranscriptDistribution(
          "topics_transcript_block_characters",
          characters,
          { block, stage },
        );
        span.setAttribute(
          `langfuse.trace_batch.topics_transcript_block_${block}_${stage}_characters`,
          characters,
        );
      }
    }
    return text;
  } catch {
    recordIncrement("langfuse.trace_batch.topics_transcript_failed", 1);
    span.setAttribute("langfuse.trace_batch.topics_transcript", "failed");
    return undefined;
  }
}

async function recordTopicsTokens(text: string, span: Span) {
  const tokens = text
    ? await tokenCountAsync({ model: TOKENIZER_MODEL, text })
    : 0;
  if (tokens === undefined) {
    recordIncrement(
      "langfuse.trace_batch.topics_transcript_token_estimation_unavailable",
      1,
    );
    span.setAttribute("langfuse.trace_batch.topics_transcript", "unavailable");
    return;
  }
  recordTranscriptDistribution("topics_transcript_tokens", tokens, {
    tokenizer: "o200k_base",
  });
  span.setAttribute("langfuse.trace_batch.topics_transcript_tokens", tokens);
}

async function recordTranscriptJsonTokens(text: string | null, span: Span) {
  const tokens =
    text === null ? 0 : await tokenCountAsync({ model: TOKENIZER_MODEL, text });
  if (tokens === undefined) {
    recordIncrement(
      "langfuse.trace_batch.transcript_json_token_estimation_unavailable",
      1,
    );
    span.setAttribute("langfuse.trace_batch.transcript_json", "unavailable");
    return;
  }
  recordTranscriptDistribution("transcript_json_tokens", tokens, {
    tokenizer: "o200k_base",
  });
  span.setAttribute("langfuse.trace_batch.transcript_json_tokens", tokens);
}

async function recordGenericTranscriptTokens(text: string, span: Span) {
  const tokens = text
    ? await tokenCountAsync({ model: TOKENIZER_MODEL, text })
    : 0;
  if (tokens === undefined) {
    recordIncrement(
      "langfuse.trace_batch.generic_transcript_token_estimation_unavailable",
      1,
    );
    span.setAttribute("langfuse.trace_batch.generic_transcript", "unavailable");
    return;
  }
  recordTranscriptDistribution("generic_transcript_tokens", tokens, {
    tokenizer: "o200k_base",
  });
  span.setAttribute("langfuse.trace_batch.generic_transcript_tokens", tokens);
}

function recordGenericRendering(
  transcript: Transcript | null,
  observations: Observation[],
  span: Span,
): string | undefined {
  try {
    const text = renderGenericTranscript(transcript, observations);
    recordTranscriptDistribution("generic_transcript_characters", text.length);
    span.setAttribute(
      "langfuse.trace_batch.generic_transcript_characters",
      text.length,
    );
    return text;
  } catch {
    recordIncrement("langfuse.trace_batch.generic_transcript_failed", 1);
    span.setAttribute("langfuse.trace_batch.generic_transcript", "failed");
    return undefined;
  }
}

export function recordTraceBatchTranscript(
  observations: Observation[],
  onTranscript?: (context: TranscriptContext | null) => Promise<void>,
): Promise<void> {
  // Inherit the batch parent without activating this span while the next trace streams.
  const span = getTracer("trace-batch").startSpan("trace-batch-transcript", {
    attributes: {
      "langfuse.project.id": observations[0]?.projectId,
      "langfuse.trace.id": observations[0]?.traceId ?? undefined,
      "langfuse.trace_batch.observation_count": observations.length,
      "langfuse.trace_batch.experiment_id":
        env.LANGFUSE_TRACE_BATCH_EXPERIMENT_ID,
      "langfuse.trace_batch.tokenizer": "o200k_base",
    },
  });
  try {
    if (span.isRecording() && env.NEXTAUTH_URL && observations[0]?.traceId) {
      const { projectId, traceId } = observations[0];
      span.setAttribute(
        "langfuse.trace.url",
        new URL(
          buildTracesPath({ projectId, query: { peek: traceId } }).slice(1),
          getProductBaseUrl(),
        ).toString(),
      );
    }
    const startedAt = performance.now();
    let phaseTimings = { normalizationMs: 0, matchingMs: 0 };
    const orderedObservations = orderObservations(observations);
    const context = assembleTranscriptContext(orderedObservations, {
      onTimings: (timings) => {
        phaseTimings = timings;
      },
    });
    const transcript = context?.transcript ?? null;
    const assemblyDurationMs = performance.now() - startedAt;
    recordTranscriptDistribution(
      "transcript_assembly_duration_ms",
      assemblyDurationMs,
      { has_transcript: String(transcript !== null) },
    );
    span.setAttributes({
      "langfuse.trace_batch.transcript_assembly_duration_ms":
        assemblyDurationMs,
      "langfuse.trace_batch.has_transcript": transcript !== null,
    });
    for (const [phase, duration] of [
      ["normalization", phaseTimings.normalizationMs],
      ["matching", phaseTimings.matchingMs],
    ] as const) {
      recordTranscriptDistribution(
        "transcript_assembly_phase_duration_ms",
        duration,
        { phase },
      );
      span.setAttribute(
        `langfuse.trace_batch.transcript_assembly_${phase}_duration_ms`,
        duration,
      );
    }
    const threadCount = transcript?.threads.length ?? 0;
    recordTranscriptDistribution("transcript_thread_count", threadCount);
    span.setAttribute(
      "langfuse.trace_batch.transcript_thread_count",
      threadCount,
    );
    recordContentAndStructureCounts(
      transcript,
      orderedObservations.length,
      span,
    );
    const comparisonRenderStart = performance.now();
    const transcriptJson =
      transcript === null ? null : JSON.stringify(transcript);
    const transcriptJsonCharacters = transcriptJson?.length ?? 0;
    recordTranscriptDistribution(
      "transcript_json_characters",
      transcriptJsonCharacters,
    );
    span.setAttribute(
      "langfuse.trace_batch.transcript_json_characters",
      transcriptJsonCharacters,
    );
    if (span.isRecording())
      span.setAttribute(
        "langfuse.trace_batch.transcript_characters",
        transcriptJsonCharacters,
      );

    const genericText = recordGenericRendering(
      transcript,
      orderedObservations,
      span,
    );

    const topicsText = recordTopicsRendering(
      transcript,
      orderedObservations,
      span,
    );
    const comparisonRenderDurationMs =
      performance.now() - comparisonRenderStart;
    recordTranscriptDistribution(
      "transcript_comparison_render_duration_ms",
      comparisonRenderDurationMs,
    );
    span.setAttribute(
      "langfuse.trace_batch.transcript_comparison_render_duration_ms",
      comparisonRenderDurationMs,
    );

    if (transcript === null) {
      for (const metric of TOKEN_METRICS) recordTokens(span, metric, 0);
    }

    // Text estimates run sequentially while the next trace streams.
    return (
      (async () => {
        if (transcript !== null) {
          try {
            await recordTokenEstimates(transcript, span);
          } catch {
            // A missing experiment metric must not retry all reads in the batch.
            recordIncrement("langfuse.trace_batch.token_estimation_failed", 1);
            span.setAttribute(
              "langfuse.trace_batch.token_estimation",
              "failed",
            );
          }
        }
        const comparisonTokenizationStart = performance.now();
        try {
          await recordTranscriptJsonTokens(transcriptJson, span);
        } catch {
          recordIncrement("langfuse.trace_batch.transcript_json_failed", 1);
          span.setAttribute("langfuse.trace_batch.transcript_json", "failed");
        }
        if (genericText !== undefined) {
          try {
            await recordGenericTranscriptTokens(genericText, span);
          } catch {
            recordIncrement(
              "langfuse.trace_batch.generic_transcript_failed",
              1,
            );
            span.setAttribute(
              "langfuse.trace_batch.generic_transcript",
              "failed",
            );
          }
        }
        if (topicsText !== undefined) {
          try {
            await recordTopicsTokens(topicsText, span);
          } catch {
            recordIncrement("langfuse.trace_batch.topics_transcript_failed", 1);
            span.setAttribute(
              "langfuse.trace_batch.topics_transcript",
              "failed",
            );
          }
        }
        const comparisonTokenizationDurationMs =
          performance.now() - comparisonTokenizationStart;
        recordTranscriptDistribution(
          "transcript_comparison_tokenization_duration_ms",
          comparisonTokenizationDurationMs,
        );
        span.setAttribute(
          "langfuse.trace_batch.transcript_comparison_tokenization_duration_ms",
          comparisonTokenizationDurationMs,
        );
      })()
        .finally(() => span.end())
        // Topics summarizes from the same assembled transcript.
        .then(() => onTranscript?.(context))
    );
  } catch (error) {
    span.setStatus({
      code: SpanStatusCode.ERROR,
      message: "Transcript assembly failed",
    });
    span.end();
    throw error;
  }
}