🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): add Langfuse agent/tools template dashboard (#16597)
Trang Nguyenb3467603de46
diff --git a/packages/shared/scripts/seeder/scenarios/event-mirror.ts b/packages/shared/scripts/seeder/scenarios/event-mirror.tsindex 9dfbddc26a..43ed53f3a8 100644--- a/packages/shared/scripts/seeder/scenarios/event-mirror.ts+++ b/packages/shared/scripts/seeder/scenarios/event-mirror.ts@@ -126,6 +126,9 @@ export const observationToEvent = (prompt_id: observation.prompt_id ?? null,prompt_name: observation.prompt_name ?? null,prompt_version: observation.prompt_version ?? null,+ tool_definitions: observation.tool_definitions ?? {},+ tool_calls: observation.tool_calls ?? [],+ tool_call_names: observation.tool_call_names ?? [],metadata_names: names,metadata_values: values,start_time: toMicros(observation.start_time) ?? Date.now() * 1000,diff --git a/packages/shared/scripts/seeder/scenarios/support-agent.ts b/packages/shared/scripts/seeder/scenarios/support-agent.tsindex 773292b139..193fdfee3a 100644--- a/packages/shared/scripts/seeder/scenarios/support-agent.ts+++ b/packages/shared/scripts/seeder/scenarios/support-agent.ts@@ -4,6 +4,9 @@ import {createTracesCh,createObservationsCh,createEventsCh,+ convertCallsToArrays,+ convertDefinitionsToMap,+ extractToolsFromObservation,ObservationRecordInsertType,} from "../../../src/server";import { ObservationType } from "../../../src/domain";@@ -57,6 +60,8 @@ type DemoObs = {/** time-to-first-token in ms — generations only */ttft?: number;modelParameters?: Record<string, unknown>;+ level?: "DEFAULT" | "WARNING" | "ERROR";+ statusMessage?: string;};const CUSTOMER_MESSAGE =@@ -87,7 +92,7 @@ const PLAN: DemoObs[] = [type: "AGENT",name: "support-copilot",start: 0,- end: 11052,+ end: 11752,input: { message: CUSTOMER_MESSAGE, customer_id: "cus_LqT4v8" },output: {reply: FINAL_REPLY,@@ -139,13 +144,72 @@ const PLAN: DemoObs[] = [sentiment: "frustrated",},},+ {+ key: "plan-context",+ parentKey: "root",+ type: "GENERATION",+ name: "plan-context",+ start: 826,+ end: 1510,+ model: "gpt-5.4-mini",+ usage: [648, 57],+ ttft: 171,+ modelParameters: { temperature: 0, max_tokens: 256, tool_choice: "auto" },+ input: {+ messages: [+ {+ role: "system",+ content:+ "Decide which context sources to load for this support request. " ++ "Call every relevant tool in parallel.",+ },+ {+ role: "user",+ content:+ '{"intent":"billing.duplicate_charge","customer_id":"cus_LqT4v8"}',+ },+ ],+ },+ // Three PARALLEL tool calls in one response — the undercount shape from+ // the linked report: counting observations sees 1, summing toolCalls sees 3.+ output: {+ content: null,+ tool_calls: [+ {+ id: "call_ctx01",+ type: "function",+ function: {+ name: "crm_get_customer",+ arguments: '{"customer_id":"cus_LqT4v8"}',+ },+ },+ {+ id: "call_ctx02",+ type: "function",+ function: {+ name: "billing_list_invoices",+ arguments:+ '{"customer_id":"cus_LqT4v8","period":"2026-06..2026-07"}',+ },+ },+ {+ id: "call_ctx03",+ type: "function",+ function: {+ name: "tickets_search",+ arguments: '{"query":"duplicate charge cus_LqT4v8","limit":5}',+ },+ },+ ],+ },+ },{key: "load-context",parentKey: "root",type: "SPAN",name: "load-context",- start: 858,- end: 1512,+ start: 1558,+ end: 2212,input: { customer_id: "cus_LqT4v8" },output: { sources: ["crm", "billing", "tickets"], cache_hit: false },},@@ -154,8 +218,8 @@ const PLAN: DemoObs[] = [parentKey: "load-context",type: "TOOL",name: "crm.get-customer",- start: 881,- end: 1129,+ start: 1581,+ end: 1829,input: { customer_id: "cus_LqT4v8" },output: {name: "Maya Chen",@@ -173,8 +237,8 @@ const PLAN: DemoObs[] = [parentKey: "load-context",type: "TOOL",name: "billing.list-invoices",- start: 886,- end: 1494,+ start: 1586,+ end: 2194,input: { customer_id: "cus_LqT4v8", period: "2026-06..2026-07" },output: {invoices: [@@ -199,8 +263,8 @@ const PLAN: DemoObs[] = [parentKey: "load-context",type: "TOOL",name: "tickets.search",- start: 893,- end: 1263,+ start: 1593,+ end: 1963,input: { query: "duplicate charge cus_LqT4v8", limit: 5 },output: {hits: 1,@@ -221,8 +285,8 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "GENERATION",name: "llm.chat",- start: 1561,- end: 3389,+ start: 2261,+ end: 4089,model: "gpt-5.4",usage: [1846, 94],ttft: 243,@@ -264,8 +328,8 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "TOOL",name: "stripe.find-charges",- start: 3441,- end: 4118,+ start: 4141,+ end: 4818,input: { customer_id: "cus_LqT4v8", period: "2026-07", amount_usd: 49 },output: {charges: [@@ -294,8 +358,8 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "GENERATION",name: "llm.chat",- start: 4172,- end: 5924,+ start: 4872,+ end: 6624,model: "gpt-5.4",usage: [2413, 72],ttft: 212,@@ -337,8 +401,8 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "TOOL",name: "stripe.create-refund",- start: 5978,- end: 6893,+ start: 6678,+ end: 7593,input: {charge_id: "ch_3PqK9b",reason: "duplicate",@@ -358,8 +422,8 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "GENERATION",name: "llm.chat",- start: 6947,- end: 8731,+ start: 7647,+ end: 9431,model: "gpt-5.4",usage: [2987, 141],ttft: 264,@@ -392,8 +456,8 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "GENERATION",name: "draft-response",- start: 8790,- end: 10377,+ start: 9490,+ end: 11077,model: "gpt-5.4",usage: [1312, 187],ttft: 231,@@ -423,8 +487,8 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "GUARDRAIL",name: "guardrail.output",- start: 10426,- end: 10557,+ start: 11126,+ end: 11257,input: { text: FINAL_REPLY, checks: ["policy", "tone", "pii"] },output: { verdict: "pass", tone: "empathetic", policy_violations: [] },metadata: { provider: "internal", policy: "support-v2" },@@ -434,27 +498,161 @@ const PLAN: DemoObs[] = [parentKey: "root",type: "TOOL",name: "zendesk.send-reply",- start: 10609,- end: 10881,+ start: 11309,+ end: 11581,input: { thread_id: "thread_7Hf3kX", body: FINAL_REPLY },output: { message_id: "msg_9uTb4w", status: "sent" },metadata: { provider: "zendesk" },},];+/**+ * --fail variant: the first refund attempt hits a Stripe idempotency+ * conflict (ERROR-level TOOL) and the agent retries with a fresh key —+ * one extra ReAct turn, one extra tool call. Feeds the tool-error and+ * retry shapes on the agent dashboard.+ */+const FAILURE_SHIFT_MS = 1666;+const FAILURE_NODES: DemoObs[] = [+ {+ key: "refund-fail",+ parentKey: "root",+ type: "TOOL",+ name: "stripe.create-refund",+ start: 6678,+ end: 7167,+ level: "ERROR",+ statusMessage: "Stripe error 409: idempotency_key conflict",+ input: {+ charge_id: "ch_3PqK9b",+ reason: "duplicate",+ idempotency_key: "refund-cus_LqT4v8-20260709",+ },+ output: {+ error: {+ type: "idempotency_error",+ code: 409,+ message:+ "Keys for idempotent requests can only be used with the same " ++ "parameters they were first used with.",+ },+ },+ metadata: { provider: "stripe", api_version: "2026-06-01" },+ },+ {+ key: "llm-retry",+ parentKey: "root",+ type: "GENERATION",+ name: "llm.chat",+ start: 7221,+ end: 8291,+ model: "gpt-5.4",+ usage: [2653, 68],+ ttft: 219,+ modelParameters: {+ temperature: 0.3,+ max_tokens: 1024,+ tool_choice: "auto",+ },+ input: {+ messages: [+ { role: "system", content: AGENT_SYSTEM_PROMPT },+ {+ role: "tool",+ tool_call_id: "call_xT19vB",+ content:+ '{"error":{"type":"idempotency_error","code":409,' ++ '"message":"Keys for idempotent requests can only be used with ' ++ 'the same parameters they were first used with."}}',+ },+ ],+ },+ output: {+ content: null,+ tool_calls: [+ {+ id: "call_rT44xC",+ type: "function",+ function: {+ name: "stripe_create_refund",+ arguments:+ '{"charge_id":"ch_3PqK9b","reason":"duplicate",' ++ '"idempotency_key":"refund-cus_LqT4v8-20260709-r2"}',+ },+ },+ ],+ },+ },+];++/**+ * buildPlan returns the happy-path PLAN, or a --fail variant. Failures+ * alternate by seed so the tool-error widgets show more than one tool:+ * - "refund": failed refund attempt + retry turn spliced in (shifts the tail)+ * - "tickets": tickets.search returns a fast Zendesk 502 (same timings; the+ * agent proceeds without prior-ticket context)+ */+function buildPlan(fail: boolean, failTarget: "refund" | "tickets"): DemoObs[] {+ if (!fail) return PLAN;+ if (failTarget === "tickets") {+ return PLAN.map((p) =>+ p.key === "tickets"+ ? {+ ...p,+ level: "ERROR" as const,+ statusMessage: "Zendesk 502: upstream search unavailable",+ output: {+ error: {+ type: "upstream_error",+ code: 502,+ message: "Search backend unavailable, request not retried.",+ },+ },+ }+ : p,+ );+ }+ const refundIndex = PLAN.findIndex((p) => p.key === "refund");+ return [+ ...PLAN.slice(0, refundIndex).map((p) =>+ p.key === "root" ? { ...p, end: p.end + FAILURE_SHIFT_MS } : p,+ ),+ ...FAILURE_NODES,+ ...PLAN.slice(refundIndex).map((p) => ({+ ...p,+ start: p.start + FAILURE_SHIFT_MS,+ end: p.end + FAILURE_SHIFT_MS,+ })),+ ];+}+const run = async (ctx: ScenarioContext,params: Record<string, string | number | boolean>,): Promise<SeedSummary> => {const startedAt = Date.now();const withV4 = params["v4"] as boolean;+ const fail = params["fail"] as boolean;+ const daysAgo = Number(params["days-ago"] ?? 0);+ const seedNum = Number(params["seed"] ?? 42);+ const plan = buildPlan(fail, seedNum % 2 === 0 ? "refund" : "tickets");// The prefix IS the trace id (no "-trace" suffix): the id shows in the// trace header, so a demo seeded with a hex-looking prefix (e.g.// --id-prefix 0198f2ab41c7e93d) reads like a production trace on camera.const traceId = ctx.idPrefix;const sessionId = `${ctx.idPrefix}-thread`;- const traceTimestamp = utcDayStartMs();+ // Deterministic business-hours offset so multi-day seeds don't stack every+ // trace at midnight UTC (varies by seed and day, 08:xx-17:xx).+ const intradayMs =+ ((8 + ((seedNum * 7 + daysAgo * 5) % 10)) * 60 + ((seedNum * 13) % 60)) *+ 60_000;+ const traceTimestamp = utcDayStartMs() - daysAgo * 86_400_000 + intradayMs;+ // Seed-keyed whole-trace duration scale (0.75x-1.35x): uniform scaling keeps+ // parent/child containment intact while making latency charts vary across+ // seeded runs instead of drawing flat lines.+ const latencyScale = 0.75 + ((seedNum * 37) % 61) / 100;+ const scaleMs = (offsetMs: number) => Math.round(offsetMs * latencyScale);if (ctx.dryRun) {return {@@ -467,8 +665,8 @@ const run = async (sessionIds: [sessionId],counts: {traces: 1,- observations: PLAN.length,- events: withV4 ? PLAN.length + 1 : 0,+ observations: plan.length,+ events: withV4 ? plan.length + 1 : 0,},verified: {},links: [@@ -480,7 +678,7 @@ const run = async (};}- const root = PLAN[0];+ const root = plan[0];const trace = createTrace({id: traceId,project_id: ctx.projectId,@@ -507,16 +705,29 @@ const run = async (});const keyToId = new Map<string, string>(- PLAN.map((p, i) => [p.key, `${ctx.idPrefix}-obs-${i}`]),+ plan.map((p, i) => [p.key, `${ctx.idPrefix}-obs-${i}`]),);- const observations: ObservationRecordInsertType[] = PLAN.map((p) => {+ const observations: ObservationRecordInsertType[] = plan.map((p) => {const prices = p.model ? MODEL_PRICES[p.model] : null;const [usageInput, usageOutput] = p.usage ?? [0, 0];const inputCost = prices ? usageInput * prices.input : 0;const outputCost = prices ? usageOutput * prices.output : 0;+ // Direct ClickHouse writes bypass ingestion, so run the same tool+ // extraction the ingestion pipeline applies — otherwise the tool_calls /+ // tool_call_names columns (dashboard tool-call measures) stay empty even+ // though the generation outputs contain OpenAI-style tool_calls.+ const { toolDefinitions, toolArguments } = extractToolsFromObservation(+ p.input,+ p.output,+ );+ const toolCallArrays = convertCallsToArrays(toolArguments);+return createObservation({+ tool_definitions: convertDefinitionsToMap(toolDefinitions),+ tool_calls: toolCallArrays.tool_calls,+ tool_call_names: toolCallArrays.tool_call_names,id: keyToId.get(p.key)!,trace_id: traceId,project_id: ctx.projectId,@@ -525,12 +736,14 @@ const run = async (parent_observation_id:p.parentKey === null ? null : (keyToId.get(p.parentKey) ?? null),name: p.name,- start_time: traceTimestamp + p.start,- end_time: traceTimestamp + p.end,+ start_time: traceTimestamp + scaleMs(p.start),+ end_time: traceTimestamp + scaleMs(p.end),completion_start_time:- p.ttft !== undefined ? traceTimestamp + p.start + p.ttft : null,- level: "DEFAULT",- status_message: null,+ p.ttft !== undefined+ ? traceTimestamp + scaleMs(p.start) + scaleMs(p.ttft)+ : null,+ level: p.level ?? "DEFAULT",+ status_message: p.statusMessage ?? null,version: null,input: p.input !== undefined ? JSON.stringify(p.input) : null,output: p.output !== undefined ? JSON.stringify(p.output) : null,@@ -672,6 +885,20 @@ export const supportAgentScenario: ScenarioDefinition = {default: false,description: "also mirror into v4 events_full/events_core",},+ {+ flag: "days-ago",+ type: "number",+ default: 0,+ description:+ "anchor the trace N days in the past (deterministic business-hours time-of-day) — run once per day to spread demo data over a date range",+ },+ {+ flag: "fail",+ type: "boolean",+ default: false,+ description:+ "failing-tool variant: the first refund attempt errors (ERROR-level TOOL, Stripe 409) and the agent retries — one extra ReAct turn and tool call",+ },],run,};diff --git a/worker/src/constants/langfuse-dashboards.json b/worker/src/constants/langfuse-dashboards.jsonindex a69ec18e6e..86557172bc 100644--- a/worker/src/constants/langfuse-dashboards.json+++ b/worker/src/constants/langfuse-dashboards.json@@ -475,6 +475,149 @@"chartType": "BAR_TIME_SERIES","chartConfig": {"type": "BAR_TIME_SERIES"},"owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000001ad076tqc9kr4",+ "createdAt": "2026-08-26T00:00:00.000Z",+ "updatedAt": "2026-08-26T00:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "Total Tool Calls",+ "description": "Total number of tool calls across observations, including parallel tool calls within a single observation",+ "view": "OBSERVATIONS",+ "dimensions": [],+ "metrics": [{"agg": "sum", "measure": "toolCalls"}],+ "filters": [],+ "chartType": "NUMBER",+ "chartConfig": {"type": "NUMBER", "row_limit": 100},+ "owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000002ad076rurfiln",+ "createdAt": "2026-08-26T00:00:00.000Z",+ "updatedAt": "2026-08-26T00:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "Total Tool Calls (over time)",+ "description": "Tool call volume over time, including parallel tool calls within a single observation",+ "view": "OBSERVATIONS",+ "dimensions": [],+ "metrics": [{"agg": "sum", "measure": "toolCalls"}],+ "filters": [],+ "chartType": "BAR_TIME_SERIES",+ "chartConfig": {"type": "BAR_TIME_SERIES"},+ "owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000003ad07afj8ag4o",+ "createdAt": "2026-08-26T00:00:00.000Z",+ "updatedAt": "2026-08-27T12:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "Top 20 Called Tools",+ "description": "Invocations per tool, including repeated calls within one observation, limited to the top 20",+ "view": "OBSERVATIONS",+ "dimensions": [{"field": "calledToolNames"}],+ "metrics": [{"agg": "sum", "measure": "toolCallInvocations"}],+ "filters": [],+ "chartType": "HORIZONTAL_BAR",+ "chartConfig": {"type": "HORIZONTAL_BAR", "row_limit": 20},+ "owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000007ad07terrbytl",+ "createdAt": "2026-08-26T12:00:00.000Z",+ "updatedAt": "2026-08-26T12:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "Tool Errors by Tool",+ "description": "Failed tool executions (ERROR level) per tool, limited to the top 20",+ "view": "OBSERVATIONS",+ "dimensions": [{"field": "name"}],+ "metrics": [{"agg": "count", "measure": "count"}],+ "filters": [+ {"column": "type", "type": "stringOptions", "operator": "any of", "value": ["TOOL"]},+ {"column": "level", "type": "stringOptions", "operator": "any of", "value": ["ERROR"]}+ ],+ "chartType": "HORIZONTAL_BAR",+ "chartConfig": {"type": "HORIZONTAL_BAR", "row_limit": 20},+ "owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000008ad07p95bytl0",+ "createdAt": "2026-08-26T12:00:00.000Z",+ "updatedAt": "2026-08-26T12:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "P 95 Tool Latency by Tool",+ "description": "P95 execution latency of TOOL observations per tool, limited to the top 20",+ "view": "OBSERVATIONS",+ "dimensions": [{"field": "name"}],+ "metrics": [{"agg": "p95", "measure": "latency"}],+ "filters": [+ {"column": "type", "type": "stringOptions", "operator": "any of", "value": ["TOOL"]}+ ],+ "chartType": "HORIZONTAL_BAR",+ "chartConfig": {"type": "HORIZONTAL_BAR", "row_limit": 20},+ "owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000009ad07p95ts0tl",+ "createdAt": "2026-08-26T12:30:00.000Z",+ "updatedAt": "2026-08-26T12:30:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "P 95 Tool Latency by Tool (over time)",+ "description": "P95 execution latency of TOOL observations over time, one series per tool",+ "view": "OBSERVATIONS",+ "dimensions": [{"field": "name"}],+ "metrics": [{"agg": "p95", "measure": "latency"}],+ "filters": [+ {"column": "type", "type": "stringOptions", "operator": "any of", "value": ["TOOL"]}+ ],+ "chartType": "LINE_TIME_SERIES",+ "chartConfig": {"type": "LINE_TIME_SERIES"},+ "owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000004ad07qwlf51rv",+ "createdAt": "2026-08-26T00:00:00.000Z",+ "updatedAt": "2026-08-26T00:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "Observations by Type",+ "description": "Distribution of observations by type, e.g. agents, tools, generations, and spans",+ "view": "OBSERVATIONS",+ "dimensions": [{"field": "type"}],+ "metrics": [{"agg": "count", "measure": "count"}],+ "filters": [],+ "chartType": "PIE",+ "chartConfig": {"type": "PIE", "row_limit": 100},+ "owner": "LANGFUSE"+ },+ {+ "id": "cmtdm68000005ad07q12mamip",+ "createdAt": "2026-08-26T00:00:00.000Z",+ "updatedAt": "2026-08-26T00:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "P 95 Latency by Observation Type",+ "description": "P95 latency segmented by observation type",+ "view": "OBSERVATIONS",+ "dimensions": [{"field": "type"}],+ "metrics": [{"agg": "p95", "measure": "latency"}],+ "filters": [],+ "chartType": "LINE_TIME_SERIES",+ "chartConfig": {"type": "LINE_TIME_SERIES"},+ "owner": "LANGFUSE"}],"dashboards": [@@ -550,6 +693,29 @@{"x": 4, "y": 0, "id": "f7d74d8d-09a5-4a5e-a8de-f48dc6ba6a4b", "type": "widget", "x_size": 4, "y_size": 5, "widgetId": "cmawk5sik00igad07kjetg17j"}]}+ },+ {+ "id": "cmtdm68000006ad07dzdb73zw",+ "createdAt": "2026-08-26T00:00:00.000Z",+ "updatedAt": "2026-08-26T13:00:00.000Z",+ "createdBy": null,+ "updatedBy": null,+ "projectId": null,+ "name": "Langfuse Agent Dashboard",+ "description": "Monitor agent tool usage: total tool calls, most-called tools, tool errors and latency, and observation type breakdowns.",+ "owner": "LANGFUSE",+ "definition": {+ "widgets": [+ {"x": 0, "y": 0, "id": "20d6ebcc-0b92-4cf1-92ff-934e13dbdc90", "type": "widget", "x_size": 3, "y_size": 4, "widgetId": "cmtdm68000001ad076tqc9kr4"},+ {"x": 3, "y": 0, "id": "11c3f92f-76f8-46c9-a89e-8211812dd6d9", "type": "widget", "x_size": 5, "y_size": 4, "widgetId": "cmtdm68000003ad07afj8ag4o"},+ {"x": 8, "y": 0, "id": "8c5d70ff-9ca1-4879-893f-187c5a62ea1d", "type": "widget", "x_size": 4, "y_size": 4, "widgetId": "cmtdm68000004ad07qwlf51rv"},+ {"x": 0, "y": 4, "id": "7c8ecb81-c586-4819-8c5f-0cd2155fda2d", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000002ad076rurfiln"},+ {"x": 6, "y": 4, "id": "b82d4e57-3a19-4f06-9c3d-1a7e6f25c842", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000008ad07p95bytl0"},+ {"x": 0, "y": 8, "id": "3f6a1c2e-91b4-4c8a-a7d2-5e0b8c94d711", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000007ad07terrbytl"},+ {"x": 6, "y": 8, "id": "df9d14b2-0133-4d1c-a83a-f1437470fc0e", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000005ad07q12mamip"},+ {"x": 0, "y": 12, "id": "5d1f8a03-6c2e-47b9-8e54-90ab37c1d6f2", "type": "widget", "x_size": 12, "y_size": 4, "widgetId": "cmtdm68000009ad07p95ts0tl"}+ ]+ }}]-}\ No newline at end of file+}diff --git a/worker/src/scripts/langfuseDashboards.test.ts b/worker/src/scripts/langfuseDashboards.test.tsnew file mode 100644index 0000000000..7ccc361b9c--- /dev/null+++ b/worker/src/scripts/langfuseDashboards.test.ts@@ -0,0 +1,17 @@+import { describe, expect, it } from "vitest";+import langfuseDashboards from "../constants/langfuse-dashboards.json";++describe("langfuse template dashboards", () => {+ it("ranks Top 20 Called Tools by per-tool invocations, not observation count", () => {+ const widget = langfuseDashboards.widgets.find(+ (entry) => entry.name === "Top 20 Called Tools",+ );++ expect(widget).toMatchObject({+ view: "OBSERVATIONS",+ dimensions: [{ field: "calledToolNames" }],+ metrics: [{ agg: "sum", measure: "toolCallInvocations" }],+ });+ expect(widget?.description).not.toMatch(/observations that called/i);+ });+});