Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

6.8 KB
Raw Blame
/* eslint-disable @repo/no-exotic-operators */
/**
 * Tests for mapping Flue (https://flueframework.com) OpenTelemetry spans to the
 * Langfuse data model.
 *
 * Flue's @flue/opentelemetry adapter converts its observe() event stream into
 * OTel spans. Model-turn spans use the GenAI semantic conventions; workflow,
 * operation, tool and delegated-task spans carry flue.* attributes only. This
 * suite verifies observation types and input/output extraction for each.
 *
 * Flow tested: ResourceSpan -> OtelIngestionProcessor.processToEvent()
 */
import { describe, it, expect } from "vitest";
import {
  OtelIngestionProcessor,
  type OtelIngestionProcessorConfig,
} from "@langfuse/shared/src/server";

const FLUE_SCOPE = "@flue/opentelemetry";

function createNanoTimestamp(nanoTime: bigint): {
  low: number;
  high: number;
  unsigned: boolean;
} {
  const low = Number(nanoTime & BigInt(0xffffffff));
  const high = Number(nanoTime >> BigInt(32));
  return { low, high, unsigned: true };
}

function createBufferId(hexString: string): { type: "Buffer"; data: number[] } {
  const buffer = Buffer.from(hexString, "hex");
  return { type: "Buffer", data: Array.from(buffer) };
}

type Attr = { key: string; value: { stringValue: string } };

function attrs(record: Record<string, string>): Attr[] {
  return Object.entries(record).map(([key, value]) => ({
    key,
    value: { stringValue: value },
  }));
}

/**
 * Build a single-span ResourceSpan under the Flue instrumentation scope and
 * return the processed event for that span.
 */
function processFlueSpan(params: {
  name: string;
  attributes: Record<string, string>;
  config?: Partial<OtelIngestionProcessorConfig>;
}): any {
  const resourceSpan = {
    resource: {
      attributes: attrs({ "service.name": "flue-app" }),
    },
    scopeSpans: [
      {
        scope: { name: FLUE_SCOPE, version: "1.0.0-beta.1", attributes: [] },
        spans: [
          {
            traceId: createBufferId("a11bbc4370b42e3646f4fb2e9ff33b53"),
            spanId: createBufferId("57f0255417974100"),
            name: params.name,
            kind: 1,
            startTimeUnixNano: createNanoTimestamp(BigInt(1714488530686000000)),
            endTimeUnixNano: createNanoTimestamp(BigInt(1714488530687000000)),
            attributes: attrs(params.attributes),
          },
        ],
      },
    ],
  };

  const processor = new OtelIngestionProcessor({
    projectId: "test-project",
    publicKey: "",
    sdkName: "",
    sdkVersion: "",
    ...params.config,
  });
  const events = processor.processToEvent([resourceSpan]);
  expect(events).toHaveLength(1);
  return events[0];
}

describe("Flue OTel span mapping", () => {
  it("adds request attribution to processed OTel events", () => {
    const event = processFlueSpan({
      name: "flue.operation prompt",
      attributes: {
        "flue.operation.id": "op_01",
        "flue.operation.kind": "prompt",
      },
      config: {
        publicKey: "pk-lf-public",
        sdkName: "python",
        sdkVersion: "3.4.0",
      },
    });

    expect(event).toMatchObject({
      ingestionApiKey: "pk-lf-public",
      ingestionSdkName: "python",
      ingestionSdkVersion: "3.4.0",
    });
  });

  it("maps a model-turn span to a GENERATION with flue.turn.input/output", () => {
    const event = processFlueSpan({
      name: "chat gpt-4o-mini",
      attributes: {
        "gen_ai.operation.name": "chat",
        "gen_ai.request.model": "gpt-4o-mini",
        "gen_ai.provider.name": "openai",
        "flue.turn.input": JSON.stringify({
          systemPrompt: "You are a helpful assistant.",
          messages: [{ role: "user", content: "Weather in Berlin?" }],
        }),
        "flue.turn.output": JSON.stringify({
          role: "assistant",
          content: [{ type: "text", text: "Berlin: sunny, 72 F" }],
        }),
      },
    });

    expect(event.type).toBe("GENERATION");
    expect(event.input).toEqual({
      systemPrompt: "You are a helpful assistant.",
      messages: [{ role: "user", content: "Weather in Berlin?" }],
    });
    expect(event.output).toEqual({
      role: "assistant",
      content: [{ type: "text", text: "Berlin: sunny, 72 F" }],
    });
    // content attributes should not be duplicated into metadata
    expect(event.metadata?.attributes?.["flue.turn.input"]).toBeUndefined();
    expect(event.metadata?.attributes?.["flue.turn.output"]).toBeUndefined();
  });

  it("maps a tool span to a TOOL observation with flue.tool.arguments/result", () => {
    const event = processFlueSpan({
      name: "flue.tool lookup_weather",
      attributes: {
        "flue.tool.name": "lookup_weather",
        "flue.tool.call_id": "call_eMtDs10B2Ijsp2DawyuUeE0r",
        "flue.tool.arguments": JSON.stringify({ city: "Berlin" }),
        "flue.tool.result": JSON.stringify({
          content: [{ type: "text", text: "Berlin: sunny, 72 F" }],
        }),
      },
    });

    expect(event.type).toBe("TOOL");
    expect(event.input).toEqual({ city: "Berlin" });
    expect(event.output).toEqual({
      content: [{ type: "text", text: "Berlin: sunny, 72 F" }],
    });
    expect(event.metadata?.attributes?.["flue.tool.arguments"]).toBeUndefined();
    expect(event.metadata?.attributes?.["flue.tool.result"]).toBeUndefined();
  });

  it("maps a delegated-task span to an AGENT observation with prompt/result", () => {
    const event = processFlueSpan({
      name: "flue.task summarizer",
      attributes: {
        "flue.task.id": "task_01",
        "flue.task.agent": "summarizer",
        "flue.task.prompt": "Summarize the meeting notes.",
        "flue.task.result": "The team shipped the integration.",
      },
    });

    expect(event.type).toBe("AGENT");
    expect(event.input).toBe("Summarize the meeting notes.");
    expect(event.output).toBe("The team shipped the integration.");
  });

  it("maps a workflow span to a SPAN with payload/result", () => {
    const event = processFlueSpan({
      name: "flue.workflow tools",
      attributes: {
        "flue.workflow.name": "tools",
        "flue.workflow.payload": JSON.stringify({ city: "Berlin" }),
        "flue.workflow.result": JSON.stringify({
          message: "The current weather in Berlin is sunny.",
        }),
      },
    });

    expect(event.type).toBe("SPAN");
    expect(event.input).toEqual({ city: "Berlin" });
    expect(event.output).toEqual({
      message: "The current weather in Berlin is sunny.",
    });
  });

  it("maps an operation span to a SPAN with result as output and no input", () => {
    const event = processFlueSpan({
      name: "flue.operation prompt",
      attributes: {
        "flue.operation.id": "op_01",
        "flue.operation.kind": "prompt",
        "flue.operation.result": JSON.stringify({ text: "done" }),
      },
    });

    expect(event.type).toBe("SPAN");
    expect(event.input).toBeNull();
    expect(event.output).toEqual({ text: "done" });
  });
});