Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

10.3 KB
Raw Blame
import { describe, it, expect } from "vitest";
import { langgraphAdapter, normalizeInput } from "@langfuse/shared";

describe("LangGraph Adapter", () => {
  it("should detect LangGraph via various metadata patterns", () => {
    expect(
      langgraphAdapter.detect({
        metadata: { langgraph_node: "agent", langgraph_step: 3 },
      }),
    ).toBe(true);

    expect(
      langgraphAdapter.detect({ metadata: { framework: "langgraph" } }),
    ).toBe(true);

    expect(langgraphAdapter.detect({ framework: "langgraph" })).toBe(true);

    expect(
      langgraphAdapter.detect({
        metadata: {
          messages: [
            { type: "human", content: "Hello" },
            { type: "ai", content: "Hi" },
          ],
        },
      }),
    ).toBe(true);

    expect(
      langgraphAdapter.detect({
        metadata: { scope: { name: "langfuse-sdk" } },
      }),
    ).toBe(false);

    // Regression: a null scope must not throw (typeof null === "object")
    expect(() =>
      langgraphAdapter.detect({ metadata: { scope: null } }),
    ).not.toThrow();
    expect(langgraphAdapter.detect({ metadata: { scope: null } })).toBe(false);
  });

  it("should normalize LangGraph messages with type-to-role conversion and tool_calls", () => {
    const input = {
      messages: [
        {
          content: "Search the web",
          type: "human",
          id: "msg-1",
          name: null,
        },
        {
          content: "",
          additional_kwargs: {
            tool_calls: [
              {
                id: "call_abc",
                type: "function",
                function: {
                  name: "Web-Search",
                  arguments: { query: "example" },
                },
              },
            ],
          },
          type: "ai",
          id: "run-123",
        },
        {
          content: { temperature: 72, conditions: "sunny" },
          type: "tool",
          name: "Web-Search",
          tool_call_id: "call_abc",
        },
      ],
    };

    const result = normalizeInput(input, { framework: "langgraph" });
    expect(result.success).toBe(true);
    if (!result.data) throw new Error("Expected data to be defined");

    expect(result.data).toHaveLength(3);
    expect(result.data[0].role).toBe("user");
    expect(result.data[0].content).toBe("Search the web");
    expect(result.data[0].json?.json?._originalType).toBe("human");

    expect(result.data[1].role).toBe("assistant");
    expect(result.data[1].json?.json?._originalType).toBe("ai");
    // Tool calls should be in validated tool_calls field with flat format
    const toolCalls = result.data[1].tool_calls;
    expect(toolCalls).toBeDefined();
    expect(toolCalls?.[0].id).toBe("call_abc");
    expect(toolCalls?.[0].name).toBe("Web-Search");
    expect(toolCalls?.[0].arguments).toBe('{"query":"example"}');

    expect(result.data[2].role).toBe("tool");
    expect(typeof result.data[2].content).toBe("string");
    expect(result.data[2].content).toBe(
      '{"temperature":72,"conditions":"sunny"}',
    );
  });

  it("should filter out tool definitions and attach them to messages", () => {
    // This is the format from langgraph-2025-08-22.json
    const input = [
      {
        role: "user",
        content: "Be helpful!",
      },
      {
        role: "user",
        content: "Search the web for 'example' and summarize.",
      },
      {
        role: "tool",
        content: {
          type: "function",
          function: {
            name: "Web-Search",
            description: "Dummy web search tool.",
            parameters: {
              properties: {
                query: {
                  type: "string",
                },
              },
              required: ["query"],
              type: "object",
            },
          },
        },
      },
    ];

    const result = normalizeInput(input, { framework: "langgraph" });
    expect(result.success).toBe(true);

    // Tool definition should be filtered out
    expect(result.data).toHaveLength(2);

    // Both messages should have tools attached
    expect(result.data?.[0].role).toBe("user");
    expect(result.data?.[0].content).toBe("Be helpful!");
    expect(result.data?.[0].tools).toBeDefined();
    expect(result.data?.[0].tools?.[0].name).toBe("Web-Search");
    expect(result.data?.[0].tools?.[0].description).toBe(
      "Dummy web search tool.",
    );

    expect(result.data?.[1].role).toBe("user");
    expect(result.data?.[1].content).toBe(
      "Search the web for 'example' and summarize.",
    );
    expect(result.data?.[1].tools).toBeDefined();
    expect(result.data?.[1].tools?.[0].name).toBe("Web-Search");
  });

  it("should attach root tools from normalized wrapped LangGraph input", () => {
    const input = {
      messages: [
        {
          role: "system",
          content: "You are a helpful assistant.",
        },
        {
          role: "user",
          content: "What's the weather in Tokyo?",
        },
      ],
      tools: [
        {
          type: "function",
          name: "get_weather",
          description: "Get the weather for a given location.",
          inputSchema: {
            type: "object",
            properties: {
              location: {
                type: "string",
                description: "The location to get the weather for.",
              },
            },
            required: ["location"],
          },
        },
      ],
    };

    const result = normalizeInput(input, { framework: "langgraph" });

    expect(result.success).toBe(true);
    expect(result.data).toHaveLength(2);
    expect(result.data?.[0].tools).toEqual([
      {
        name: "get_weather",
        description: "Get the weather for a given location.",
        parameters: {
          type: "object",
          properties: {
            location: {
              type: "string",
              description: "The location to get the weather for.",
            },
          },
          required: ["location"],
        },
      },
    ]);
    expect(result.data?.[1].tools?.[0].name).toBe("get_weather");
  });

  it("should skip null entries in tool_calls arrays", () => {
    const input = [
      {
        role: "assistant",
        content: "",
        tool_calls: [
          null,
          {
            id: "call_valid",
            type: "function",
            function: {
              name: "search_docs",
              arguments: { topic: "Langfuse" },
            },
          },
        ],
      },
    ];

    const result = normalizeInput(input, { framework: "langgraph" });
    expect(result.success).toBe(true);

    const toolCalls = result.data?.[0].tool_calls;
    expect(toolCalls?.length).toBe(1);
    expect(toolCalls?.[0].name).toBe("search_docs");
    expect(toolCalls?.[0].arguments).toBe('{"topic":"Langfuse"}');
  });

  it("should clean up additional_kwargs with null values", () => {
    const output = {
      role: "assistant",
      content: "",
      additional_kwargs: {
        tool_calls: [
          {
            id: "call_123",
            function: {
              name: "Web-Search",
              arguments: { query: "example" },
            },
            type: "function",
          },
        ],
        refusal: null,
      },
    };

    const result = normalizeInput(output, { framework: "langgraph" });
    expect(result.success).toBe(true);

    // Tool calls should be extracted and flattened
    expect(result.data?.[0].tool_calls).toBeDefined();
    expect(result.data?.[0].tool_calls?.[0].name).toBe("Web-Search");
    expect(result.data?.[0].tool_calls?.[0].arguments).toBe(
      '{"query":"example"}',
    );

    // additional_kwargs should be removed entirely (only had tool_calls and null refusal)
    expect(result.data?.[0].json?.json?.additional_kwargs).toBeUndefined();
  });

  it("should not mutate original object during normalization", () => {
    const additionalKwargs = {
      tool_calls: [{ id: "call_123", function: { name: "Web-Search" } }],
    };
    const output = {
      role: "assistant",
      content: "",
      additional_kwargs: additionalKwargs,
    };

    normalizeInput(output, { framework: "langgraph" });

    // Original must be unchanged (JSON view depends on it)
    expect(additionalKwargs).toEqual({
      tool_calls: [{ id: "call_123", function: { name: "Web-Search" } }],
    });
  });

  it("should filter tool definitions from google vertex langchain trace", () => {
    // Format from google-vertex-tools-langchain-2025-09-19.json
    const input = [
      {
        role: "system",
        content: "You are a helpful assistant.",
      },
      {
        role: "user",
        content: "Hello, I'm interested in learning more.",
      },
      {
        role: "tool",
        content: {
          type: "function",
          function: {
            name: "transition_to_service_a",
            description: "Transition to Service A",
            parameters: {
              properties: {
                reason: {
                  description: "Reason for transition",
                  type: "string",
                },
              },
              required: ["reason"],
              type: "object",
            },
          },
        },
      },
      {
        role: "tool",
        content: {
          type: "function",
          function: {
            name: "get_customer_info",
            description: "Retrieve customer info",
            parameters: {
              type: "object",
              properties: {
                customer_id: {
                  type: "string",
                },
              },
            },
          },
        },
      },
      {
        role: "assistant",
        content: "How can I help you today?",
      },
    ];

    const ctx = {
      metadata: {
        ls_provider: "google_vertexai",
      },
    };

    const result = normalizeInput(input, ctx);

    expect(result.success).toBe(true);

    // Tool definitions should be filtered out - only 3 messages remain
    expect(result.data).toHaveLength(3);

    // Check no tool definition messages remain
    const toolMessages = result.data?.filter((msg: any) => msg.role === "tool");
    expect(toolMessages).toHaveLength(0);

    // Tools should be attached to messages
    expect(result.data?.[0].tools).toBeDefined();
    expect(result.data?.[0].tools).toHaveLength(2);
    expect(result.data?.[0].tools?.[0].name).toBe("transition_to_service_a");
    expect(result.data?.[0].tools?.[1].name).toBe("get_customer_info");
  });
});