Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

18.8 KB
Raw Blame
import { randomBytes } from "node:crypto";

import type { Telemetry } from "ai";

import {
  type Attributes,
  type Tracer,
  context,
  ROOT_CONTEXT,
  SpanKind,
  SpanStatusCode,
  trace,
  type Span,
} from "@opentelemetry/api";
import { resourceFromAttributes } from "@opentelemetry/resources";
import {
  BasicTracerProvider,
  InMemorySpanExporter,
  SimpleSpanProcessor,
} from "@opentelemetry/sdk-trace-base";
import type { TelemetryOptions } from "ai";

import { stringifyValue } from "../../../utils/stringChecks";
import { traceException } from "../../instrumentation";
import { logger } from "../../logger";
import {
  AI_FEATURE_OTEL_SDK_NAME,
  buildEvaluationAttributes,
  LangfuseOtelSpanAttributes,
} from "../../otel/attributes";
import { publishInternalOtelSpans } from "../../otel/internalTraceOtelWriter";
import type { InternalTraceExperimentContext } from "../internalTraceEvents";
import type { TraceSinkParams } from "../types";

const W3C_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
const INTERNAL_SDK_NAME = "langfuse-internal-ai-sdk";
const INTERNAL_TRACER_SCOPE = "langfuse-internal-llm-completion";

export type AiSdkTelemetryCapture = {
  /** Pass to `generateText`/`streamText` as the `telemetry` option. */
  telemetry: TelemetryOptions;
  /**
   * Runs `fn` with the internal root span active, so AI SDK telemetry spans
   * nest under it. Requires a registered OTel context manager for propagation
   * across awaits (present in web and worker runtimes).
   */
  run: <T>(fn: () => T) => T;
  /**
   * Records the completion result as the root span's (and trace's) output.
   * Call before `flush`. The root observation's input/output feed eval
   * variable mapping for experiment run items.
   */
  setRootOutput: (output: unknown) => void;
  /**
   * Marks the root span (and thus the internal trace) as failed. Call before
   * `flush`, so a failed completion is visible as an ERROR-level root
   * observation instead of a bare 0-duration span.
   */
  setRootError: (error: unknown) => void;
  /**
   * Ends the root span and publishes all captured spans to the regular OTel
   * ingestion pipeline (same S3 + queue path as the public
   * /api/public/otel/v1/traces endpoint). Idempotent; never throws.
   */
  flush: () => Promise<void>;
};

/**
 * Per-call isolated OTel capture for AI SDK executions.
 *
 * A dedicated `BasicTracerProvider` (never globally registered) buffers spans
 * in memory. Its id generator returns `traceSinkParams.traceId` for root
 * spans, so the internal root span carries the Langfuse trace ID without any
 * span rewriting, and all child spans inherit it via context. After the call,
 * spans are serialized to OTLP JSON and submitted through
 * `OtelIngestionProcessor.publishToOtelIngestionQueue` — internal traces get
 * exactly the same ingestion treatment as user traces.
 *
 * For experiment run items (`eventsWriter.experimentContext` present), the
 * `langfuse.experiment.*` attributes are set on every captured span with the
 * root span as `experiment_item_root_observation_id`, so the OTel ingestion
 * pipeline materializes the linkage required for queue-side scheduling of
 * experiment observation evals.
 *
 * Returns `undefined` (no tracing) when the environment is not
 * langfuse-prefixed (the eval-loop safeguard) unless this is an AI-feature
 * product trace (`aiFeatureOtelIngestion`), or when the trace ID is not a
 * valid W3C trace ID.
 */
export function createAiSdkTelemetryCapture(params: {
  traceSinkParams: TraceSinkParams;
  /** Recorded as the root span's (and trace's) input. */
  rootInput?: unknown;
  /** Trace-visible generation input. Pass media-reference messages instead of the exact provider egress payload when needed. */
  generationInput?: unknown;
}): AiSdkTelemetryCapture | undefined {
  const { traceSinkParams, rootInput, generationInput } = params;
  const isAiFeatureProductTrace = Boolean(
    traceSinkParams.aiFeatureOtelIngestion,
  );

  // Safeguard: All internal traces must use LangfuseInternalTraceEnvironment enum values
  // This prevents infinite eval loops (user trace → eval → eval trace → another eval)
  // See corresponding check in worker/src/features/evaluation/evalService.ts createEvalJobs()
  // AI-feature product traces are the exception: they use environment
  // `production` so observation evals can match them.
  if (
    !isAiFeatureProductTrace &&
    !traceSinkParams.environment?.startsWith("langfuse")
  ) {
    logger.warn(
      "Skipping trace creation: internal traces must use LangfuseInternalTraceEnvironment enum",
      {
        environment: traceSinkParams.environment,
        traceId: traceSinkParams.traceId,
      },
    );
    return undefined;
  }

  const traceId = traceSinkParams.traceId.toLowerCase();
  if (!W3C_TRACE_ID_PATTERN.test(traceId) || traceId === "0".repeat(32)) {
    logger.warn(
      "Skipping trace creation: trace id is not a valid W3C trace id",
      { traceId: traceSinkParams.traceId },
    );
    return undefined;
  }

  const exporter = new InMemorySpanExporter();
  const tracerProvider = new BasicTracerProvider({
    // The environment must reach every captured span (not just the root), or
    // the resulting observations fall back to the "default" environment and
    // leak into user-facing views.
    resource: resourceFromAttributes({
      [LangfuseOtelSpanAttributes.ENVIRONMENT]: traceSinkParams.environment,
    }),
    spanProcessors: [new SimpleSpanProcessor(exporter)],
    // Root spans (no parent in context) receive the Langfuse trace ID; child
    // spans inherit it through context propagation.
    idGenerator: {
      generateTraceId: () => traceId,
      generateSpanId: () => randomBytes(8).toString("hex"),
    },
  });
  const tracer = tracerProvider.getTracer(INTERNAL_TRACER_SCOPE);

  const serializedInput =
    rootInput !== undefined ? stringifyValue(rootInput) : undefined;
  const evaluationAttributes = traceSinkParams.evaluationContext
    ? buildEvaluationAttributes(traceSinkParams.evaluationContext)
    : undefined;

  const experimentContext = traceSinkParams.eventsWriter?.experimentContext;
  // Customers pay per observation and only use an evaluator trace to inspect
  // input, output, and cost, so the model call is written on this root span
  // instead of adding another one. Prompt experiments still need the child span.
  const writeGenerationOnRoot =
    evaluationAttributes !== undefined && experimentContext === undefined;
  const childSpanMetadata = traceSinkParams.metadata
    ? Object.fromEntries(
        Object.entries(traceSinkParams.metadata).filter(
          ([key]) => key !== "structured_output_schema",
        ),
      )
    : undefined;
  const promptAttributes = traceSinkParams.prompt
    ? {
        // Link the LLM generation to the resolved Langfuse prompt.
        [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME]:
          traceSinkParams.prompt.name,
        [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION]:
          traceSinkParams.prompt.version,
      }
    : undefined;

  const rootSpan = tracer.startSpan(
    traceSinkParams.traceName,
    {
      attributes: {
        [LangfuseOtelSpanAttributes.TRACE_NAME]: traceSinkParams.traceName,
        [LangfuseOtelSpanAttributes.ENVIRONMENT]: traceSinkParams.environment,
        ...(traceSinkParams.userId
          ? {
              [LangfuseOtelSpanAttributes.TRACE_USER_ID]:
                traceSinkParams.userId,
            }
          : {}),
        ...(evaluationAttributes ?? {}),
        ...(writeGenerationOnRoot ? (promptAttributes ?? {}) : {}),
        ...(traceSinkParams.metadata
          ? {
              [LangfuseOtelSpanAttributes.TRACE_METADATA]: JSON.stringify(
                traceSinkParams.metadata,
              ),
            }
          : {}),
        ...(writeGenerationOnRoot && childSpanMetadata
          ? {
              [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]:
                JSON.stringify(childSpanMetadata),
            }
          : {}),
        ...(serializedInput !== undefined
          ? {
              [LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: serializedInput,
            }
          : {}),
      },
    },
    // ROOT_CONTEXT detaches from the server's own observability trace.
    ROOT_CONTEXT,
  );
  const activeContext = trace.setSpan(ROOT_CONTEXT, rootSpan);

  const experimentAttributes = experimentContext
    ? buildExperimentAttributes(
        experimentContext,
        rootSpan.spanContext().spanId,
      )
    : undefined;
  if (experimentAttributes) {
    rootSpan.setAttributes(experimentAttributes);
  }

  const generationTelemetry = createGenerationSpanTelemetry({
    tracer,
    recordedInput: generationInput,
    existingSpan: writeGenerationOnRoot ? rootSpan : undefined,
    attributes: {
      // Experiment linkage goes on every span so every materialized event
      // remains associated with the run item root.
      ...(experimentAttributes ?? {}),
      ...(evaluationAttributes ?? {}),
      ...(promptAttributes ?? {}),
      [LangfuseOtelSpanAttributes.TRACE_NAME]: traceSinkParams.traceName,
      [LangfuseOtelSpanAttributes.ENVIRONMENT]: traceSinkParams.environment,
      ...(traceSinkParams.userId
        ? {
            [LangfuseOtelSpanAttributes.TRACE_USER_ID]: traceSinkParams.userId,
          }
        : {}),
      // Keep useful trace context on child spans, but leave the potentially
      // large output schema on the root trace only.
      ...(childSpanMetadata
        ? {
            [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]:
              JSON.stringify(childSpanMetadata),
          }
        : {}),
    },
  });

  let flushed = false;

  const setRootOutput = (output: unknown): void => {
    if (flushed || output === undefined) return;
    const serializedOutput = stringifyValue(output);

    rootSpan.setAttribute(
      LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT,
      serializedOutput,
    );
  };

  const setRootError = (error: unknown): void => {
    if (flushed) return;
    rootSpan.setAttribute("error.type", getErrorType(error));
    rootSpan.setStatus({
      code: SpanStatusCode.ERROR,
      message: error instanceof Error ? error.message : String(error),
    });
    if (error instanceof Error) rootSpan.recordException(error);
  };

  const flush = async (): Promise<void> => {
    if (flushed) return;
    flushed = true;

    try {
      rootSpan.end();
      await tracerProvider.forceFlush();

      const spans = exporter.getFinishedSpans();
      // Without a registered OTel context manager, AI SDK spans lose the
      // active root span and land on a random trace ID. Publish only spans on
      // the expected trace so nothing leaks into foreign traces.
      const matchingSpans = spans.filter(
        (span) => span.spanContext().traceId === traceId,
      );

      if (matchingSpans.length < spans.length) {
        logger.warn(
          "Dropping AI SDK telemetry spans outside the internal trace; is an OTel context manager registered?",
          {
            traceId,
            droppedSpans: spans.length - matchingSpans.length,
          },
        );
      }
      if (matchingSpans.length === 0) return;

      await publishInternalOtelSpans({
        spans: matchingSpans,
        projectId: traceSinkParams.targetProjectId,
        sdkName: isAiFeatureProductTrace
          ? AI_FEATURE_OTEL_SDK_NAME
          : INTERNAL_SDK_NAME,
        isLangfuseInternal: !isAiFeatureProductTrace,
      });
    } catch (e) {
      traceException(e);
      logger.error("Failed to publish AI SDK internal telemetry", {
        error: e,
        traceId,
        projectId: traceSinkParams.targetProjectId,
      });
    } finally {
      await tracerProvider.shutdown().catch(() => undefined);
    }
  };

  return {
    telemetry: {
      isEnabled: true,
      integrations: [generationTelemetry],
    },
    run: (fn) => context.with(activeContext, fn),
    setRootOutput,
    setRootError,
    flush,
  };
}

/**
 * Minimal AI SDK telemetry integration for Langfuse-internal LLM completions.
 */
function createGenerationSpanTelemetry(params: {
  tracer: Tracer;
  /**
   * Extra attributes for every generation span (Langfuse prompt link,
   * experiment linkage).
   */
  attributes?: Attributes;
  /**
   * Safe input to record instead of the provider-bound messages. The latter
   * may contain short-lived signed media URLs that must not enter traces.
   */
  recordedInput?: unknown;
  /**
   * Evaluator executions write the model call onto the trace root instead of
   * adding a child span. The caller ends that span.
   */
  existingSpan?: Span;
}): Telemetry {
  const { tracer, attributes, recordedInput, existingSpan } = params;
  const openSpans = new Map<string, Span>();

  const generationAttributes = (
    event: Parameters<NonNullable<Telemetry["onLanguageModelCallStart"]>>[0],
  ): Attributes => ({
    "gen_ai.operation.name": "chat",
    "gen_ai.provider.name": event.provider,
    "gen_ai.request.model": event.modelId,
    ...definedNumberAttributes({
      "gen_ai.request.max_tokens": event.maxOutputTokens,
      "gen_ai.request.temperature": event.temperature,
      "gen_ai.request.top_p": event.topP,
    }),
    ...(recordedInput !== undefined || event.messages !== undefined
      ? {
          "gen_ai.input.messages": safeJsonStringify(
            recordedInput ?? event.messages,
          ),
        }
      : {}),
    ...(event.tools && event.tools.length > 0
      ? { "gen_ai.tool.definitions": safeJsonStringify(event.tools) }
      : {}),
    ...attributes,
  });

  const endAllOpenSpans = (error?: unknown): void => {
    for (const span of openSpans.values()) {
      if (span === existingSpan) continue;
      if (error !== undefined) {
        span.setAttribute("error.type", getErrorType(error));
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: error instanceof Error ? error.message : String(error),
        });

        if (error instanceof Error) span.recordException(error);
      }
      span.end();
    }
    openSpans.clear();
  };

  return {
    onLanguageModelCallStart(event) {
      if (existingSpan) {
        existingSpan.setAttributes(generationAttributes(event));
        openSpans.set(event.callId, existingSpan);
        return;
      }

      // Defensive: a lingering span for this call id means its end event never
      // fired (e.g. a retried attempt) — close it before starting the next.
      openSpans.get(event.callId)?.end();

      const span = tracer.startSpan(
        `chat ${event.modelId}`,
        {
          kind: SpanKind.CLIENT,
          attributes: generationAttributes(event),
        },
        context.active(),
      );

      openSpans.set(event.callId, span);
    },

    onLanguageModelCallEnd(event) {
      const span = openSpans.get(event.callId);

      if (!span) return;

      if (span !== existingSpan) openSpans.delete(event.callId);

      const outputMessages = safeJsonStringify([
        { role: "assistant", content: event.content },
      ]);
      span.setAttributes({
        "gen_ai.response.finish_reasons": [event.finishReason],
        ...(event.responseId ? { "gen_ai.response.id": event.responseId } : {}),
        ...definedNumberAttributes({
          "gen_ai.usage.input_tokens": event.usage.inputTokens,
          "gen_ai.usage.output_tokens": event.usage.outputTokens,
          "gen_ai.usage.cache_read.input_tokens":
            event.usage.inputTokenDetails?.cacheReadTokens,
          "gen_ai.usage.cache_creation.input_tokens":
            event.usage.inputTokenDetails?.cacheWriteTokens,
        }),
        "gen_ai.output.messages": outputMessages,
        // The root span already has observation input. Without observation
        // output, ingestion ignores this model response when a later step fails.
        ...(span === existingSpan
          ? {
              [LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: outputMessages,
            }
          : {}),
      });
      if (span !== existingSpan) span.end();
    },

    onError(event) {
      // The root span's error is recorded by setRootError. Marking it here too
      // would store the same exception twice.
      endAllOpenSpans(getTelemetryError(event));
    },

    onAbort() {
      endAllOpenSpans();
    },

    // Run the provider call with the generation span active, so any future
    // nested instrumentation parents correctly.
    executeLanguageModelCall({ callId, execute }) {
      const span = openSpans.get(callId);
      if (!span) return execute();

      return context.with(trace.setSpan(context.active(), span), execute);
    },
  };
}

function definedNumberAttributes(
  attributes: Record<string, number | undefined>,
): Attributes {
  return Object.fromEntries(
    Object.entries(attributes).filter(([, value]) => value !== undefined),
  ) as Attributes;
}

function safeJsonStringify(value: unknown): string {
  try {
    return JSON.stringify(value);
  } catch {
    return "[Unserializable content]";
  }
}

function getErrorType(error: unknown): string {
  if (
    error !== null &&
    typeof error === "object" &&
    "name" in error &&
    typeof error.name === "string" &&
    error.name.length > 0
  ) {
    return error.name;
  }

  return "_OTHER";
}

function getTelemetryError(event: unknown): unknown {
  return event !== null && typeof event === "object" && "error" in event
    ? event.error
    : event;
}
/**
 * Maps the internal experiment context to the `langfuse.experiment.*` span
 * attributes that `OtelIngestionProcessor.extractExperimentFields` reads to
 * produce experiment-linked event records.
 */
function buildExperimentAttributes(
  experimentContext: InternalTraceExperimentContext,
  rootSpanId: string,
): Record<string, string> {
  return {
    [LangfuseOtelSpanAttributes.EXPERIMENT_ID]: experimentContext.id,
    [LangfuseOtelSpanAttributes.EXPERIMENT_NAME]: experimentContext.name,
    [LangfuseOtelSpanAttributes.EXPERIMENT_DATASET_ID]:
      experimentContext.datasetId,
    [LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_ID]: experimentContext.itemId,
    [LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_VERSION]:
      experimentContext.itemVersion,
    [LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_ROOT_OBSERVATION_ID]:
      rootSpanId,
    ...(experimentContext.description
      ? {
          [LangfuseOtelSpanAttributes.EXPERIMENT_DESCRIPTION]:
            experimentContext.description,
        }
      : {}),
    ...(experimentContext.itemExpectedOutput !== undefined &&
    experimentContext.itemExpectedOutput !== null
      ? {
          [LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_EXPECTED_OUTPUT]:
            stringifyValue(experimentContext.itemExpectedOutput),
        }
      : {}),
    ...(experimentContext.metadata
      ? {
          [LangfuseOtelSpanAttributes.EXPERIMENT_METADATA]: JSON.stringify(
            experimentContext.metadata,
          ),
        }
      : {}),
    ...(experimentContext.itemMetadata
      ? {
          [LangfuseOtelSpanAttributes.EXPERIMENT_ITEM_METADATA]: JSON.stringify(
            experimentContext.itemMetadata,
          ),
        }
      : {}),
  };
}