Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): add Langfuse agent/tools template dashboard (#16597)

Trang Nguyenb3467603de46
diff --git a/packages/shared/scripts/seeder/scenarios/event-mirror.ts b/packages/shared/scripts/seeder/scenarios/event-mirror.ts
index 9dfbddc26a..43ed53f3a8 100644
--- a/packages/shared/scripts/seeder/scenarios/event-mirror.ts
+++ b/packages/shared/scripts/seeder/scenarios/event-mirror.ts
@@ -126,6 +126,9 @@ export const observationToEvent = (
prompt_id: observation.prompt_id ?? null,
prompt_name: observation.prompt_name ?? null,
prompt_version: observation.prompt_version ?? null,
+ tool_definitions: observation.tool_definitions ?? {},
+ tool_calls: observation.tool_calls ?? [],
+ tool_call_names: observation.tool_call_names ?? [],
metadata_names: names,
metadata_values: values,
start_time: toMicros(observation.start_time) ?? Date.now() * 1000,
diff --git a/packages/shared/scripts/seeder/scenarios/support-agent.ts b/packages/shared/scripts/seeder/scenarios/support-agent.ts
index 773292b139..193fdfee3a 100644
--- a/packages/shared/scripts/seeder/scenarios/support-agent.ts
+++ b/packages/shared/scripts/seeder/scenarios/support-agent.ts
@@ -4,6 +4,9 @@ import {
createTracesCh,
createObservationsCh,
createEventsCh,
+ convertCallsToArrays,
+ convertDefinitionsToMap,
+ extractToolsFromObservation,
ObservationRecordInsertType,
} from "../../../src/server";
import { ObservationType } from "../../../src/domain";
@@ -57,6 +60,8 @@ type DemoObs = {
/** time-to-first-token in ms — generations only */
ttft?: number;
modelParameters?: Record<string, unknown>;
+ level?: "DEFAULT" | "WARNING" | "ERROR";
+ statusMessage?: string;
};
const CUSTOMER_MESSAGE =
@@ -87,7 +92,7 @@ const PLAN: DemoObs[] = [
type: "AGENT",
name: "support-copilot",
start: 0,
- end: 11052,
+ end: 11752,
input: { message: CUSTOMER_MESSAGE, customer_id: "cus_LqT4v8" },
output: {
reply: FINAL_REPLY,
@@ -139,13 +144,72 @@ const PLAN: DemoObs[] = [
sentiment: "frustrated",
},
},
+ {
+ key: "plan-context",
+ parentKey: "root",
+ type: "GENERATION",
+ name: "plan-context",
+ start: 826,
+ end: 1510,
+ model: "gpt-5.4-mini",
+ usage: [648, 57],
+ ttft: 171,
+ modelParameters: { temperature: 0, max_tokens: 256, tool_choice: "auto" },
+ input: {
+ messages: [
+ {
+ role: "system",
+ content:
+ "Decide which context sources to load for this support request. " +
+ "Call every relevant tool in parallel.",
+ },
+ {
+ role: "user",
+ content:
+ '{"intent":"billing.duplicate_charge","customer_id":"cus_LqT4v8"}',
+ },
+ ],
+ },
+ // Three PARALLEL tool calls in one response — the undercount shape from
+ // the linked report: counting observations sees 1, summing toolCalls sees 3.
+ output: {
+ content: null,
+ tool_calls: [
+ {
+ id: "call_ctx01",
+ type: "function",
+ function: {
+ name: "crm_get_customer",
+ arguments: '{"customer_id":"cus_LqT4v8"}',
+ },
+ },
+ {
+ id: "call_ctx02",
+ type: "function",
+ function: {
+ name: "billing_list_invoices",
+ arguments:
+ '{"customer_id":"cus_LqT4v8","period":"2026-06..2026-07"}',
+ },
+ },
+ {
+ id: "call_ctx03",
+ type: "function",
+ function: {
+ name: "tickets_search",
+ arguments: '{"query":"duplicate charge cus_LqT4v8","limit":5}',
+ },
+ },
+ ],
+ },
+ },
{
key: "load-context",
parentKey: "root",
type: "SPAN",
name: "load-context",
- start: 858,
- end: 1512,
+ start: 1558,
+ end: 2212,
input: { customer_id: "cus_LqT4v8" },
output: { sources: ["crm", "billing", "tickets"], cache_hit: false },
},
@@ -154,8 +218,8 @@ const PLAN: DemoObs[] = [
parentKey: "load-context",
type: "TOOL",
name: "crm.get-customer",
- start: 881,
- end: 1129,
+ start: 1581,
+ end: 1829,
input: { customer_id: "cus_LqT4v8" },
output: {
name: "Maya Chen",
@@ -173,8 +237,8 @@ const PLAN: DemoObs[] = [
parentKey: "load-context",
type: "TOOL",
name: "billing.list-invoices",
- start: 886,
- end: 1494,
+ start: 1586,
+ end: 2194,
input: { customer_id: "cus_LqT4v8", period: "2026-06..2026-07" },
output: {
invoices: [
@@ -199,8 +263,8 @@ const PLAN: DemoObs[] = [
parentKey: "load-context",
type: "TOOL",
name: "tickets.search",
- start: 893,
- end: 1263,
+ start: 1593,
+ end: 1963,
input: { query: "duplicate charge cus_LqT4v8", limit: 5 },
output: {
hits: 1,
@@ -221,8 +285,8 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "GENERATION",
name: "llm.chat",
- start: 1561,
- end: 3389,
+ start: 2261,
+ end: 4089,
model: "gpt-5.4",
usage: [1846, 94],
ttft: 243,
@@ -264,8 +328,8 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "TOOL",
name: "stripe.find-charges",
- start: 3441,
- end: 4118,
+ start: 4141,
+ end: 4818,
input: { customer_id: "cus_LqT4v8", period: "2026-07", amount_usd: 49 },
output: {
charges: [
@@ -294,8 +358,8 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "GENERATION",
name: "llm.chat",
- start: 4172,
- end: 5924,
+ start: 4872,
+ end: 6624,
model: "gpt-5.4",
usage: [2413, 72],
ttft: 212,
@@ -337,8 +401,8 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "TOOL",
name: "stripe.create-refund",
- start: 5978,
- end: 6893,
+ start: 6678,
+ end: 7593,
input: {
charge_id: "ch_3PqK9b",
reason: "duplicate",
@@ -358,8 +422,8 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "GENERATION",
name: "llm.chat",
- start: 6947,
- end: 8731,
+ start: 7647,
+ end: 9431,
model: "gpt-5.4",
usage: [2987, 141],
ttft: 264,
@@ -392,8 +456,8 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "GENERATION",
name: "draft-response",
- start: 8790,
- end: 10377,
+ start: 9490,
+ end: 11077,
model: "gpt-5.4",
usage: [1312, 187],
ttft: 231,
@@ -423,8 +487,8 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "GUARDRAIL",
name: "guardrail.output",
- start: 10426,
- end: 10557,
+ start: 11126,
+ end: 11257,
input: { text: FINAL_REPLY, checks: ["policy", "tone", "pii"] },
output: { verdict: "pass", tone: "empathetic", policy_violations: [] },
metadata: { provider: "internal", policy: "support-v2" },
@@ -434,27 +498,161 @@ const PLAN: DemoObs[] = [
parentKey: "root",
type: "TOOL",
name: "zendesk.send-reply",
- start: 10609,
- end: 10881,
+ start: 11309,
+ end: 11581,
input: { thread_id: "thread_7Hf3kX", body: FINAL_REPLY },
output: { message_id: "msg_9uTb4w", status: "sent" },
metadata: { provider: "zendesk" },
},
];
+/**
+ * --fail variant: the first refund attempt hits a Stripe idempotency
+ * conflict (ERROR-level TOOL) and the agent retries with a fresh key —
+ * one extra ReAct turn, one extra tool call. Feeds the tool-error and
+ * retry shapes on the agent dashboard.
+ */
+const FAILURE_SHIFT_MS = 1666;
+const FAILURE_NODES: DemoObs[] = [
+ {
+ key: "refund-fail",
+ parentKey: "root",
+ type: "TOOL",
+ name: "stripe.create-refund",
+ start: 6678,
+ end: 7167,
+ level: "ERROR",
+ statusMessage: "Stripe error 409: idempotency_key conflict",
+ input: {
+ charge_id: "ch_3PqK9b",
+ reason: "duplicate",
+ idempotency_key: "refund-cus_LqT4v8-20260709",
+ },
+ output: {
+ error: {
+ type: "idempotency_error",
+ code: 409,
+ message:
+ "Keys for idempotent requests can only be used with the same " +
+ "parameters they were first used with.",
+ },
+ },
+ metadata: { provider: "stripe", api_version: "2026-06-01" },
+ },
+ {
+ key: "llm-retry",
+ parentKey: "root",
+ type: "GENERATION",
+ name: "llm.chat",
+ start: 7221,
+ end: 8291,
+ model: "gpt-5.4",
+ usage: [2653, 68],
+ ttft: 219,
+ modelParameters: {
+ temperature: 0.3,
+ max_tokens: 1024,
+ tool_choice: "auto",
+ },
+ input: {
+ messages: [
+ { role: "system", content: AGENT_SYSTEM_PROMPT },
+ {
+ role: "tool",
+ tool_call_id: "call_xT19vB",
+ content:
+ '{"error":{"type":"idempotency_error","code":409,' +
+ '"message":"Keys for idempotent requests can only be used with ' +
+ 'the same parameters they were first used with."}}',
+ },
+ ],
+ },
+ output: {
+ content: null,
+ tool_calls: [
+ {
+ id: "call_rT44xC",
+ type: "function",
+ function: {
+ name: "stripe_create_refund",
+ arguments:
+ '{"charge_id":"ch_3PqK9b","reason":"duplicate",' +
+ '"idempotency_key":"refund-cus_LqT4v8-20260709-r2"}',
+ },
+ },
+ ],
+ },
+ },
+];
+
+/**
+ * buildPlan returns the happy-path PLAN, or a --fail variant. Failures
+ * alternate by seed so the tool-error widgets show more than one tool:
+ * - "refund": failed refund attempt + retry turn spliced in (shifts the tail)
+ * - "tickets": tickets.search returns a fast Zendesk 502 (same timings; the
+ * agent proceeds without prior-ticket context)
+ */
+function buildPlan(fail: boolean, failTarget: "refund" | "tickets"): DemoObs[] {
+ if (!fail) return PLAN;
+ if (failTarget === "tickets") {
+ return PLAN.map((p) =>
+ p.key === "tickets"
+ ? {
+ ...p,
+ level: "ERROR" as const,
+ statusMessage: "Zendesk 502: upstream search unavailable",
+ output: {
+ error: {
+ type: "upstream_error",
+ code: 502,
+ message: "Search backend unavailable, request not retried.",
+ },
+ },
+ }
+ : p,
+ );
+ }
+ const refundIndex = PLAN.findIndex((p) => p.key === "refund");
+ return [
+ ...PLAN.slice(0, refundIndex).map((p) =>
+ p.key === "root" ? { ...p, end: p.end + FAILURE_SHIFT_MS } : p,
+ ),
+ ...FAILURE_NODES,
+ ...PLAN.slice(refundIndex).map((p) => ({
+ ...p,
+ start: p.start + FAILURE_SHIFT_MS,
+ end: p.end + FAILURE_SHIFT_MS,
+ })),
+ ];
+}
+
const run = async (
ctx: ScenarioContext,
params: Record<string, string | number | boolean>,
): Promise<SeedSummary> => {
const startedAt = Date.now();
const withV4 = params["v4"] as boolean;
+ const fail = params["fail"] as boolean;
+ const daysAgo = Number(params["days-ago"] ?? 0);
+ const seedNum = Number(params["seed"] ?? 42);
+ const plan = buildPlan(fail, seedNum % 2 === 0 ? "refund" : "tickets");
// The prefix IS the trace id (no "-trace" suffix): the id shows in the
// trace header, so a demo seeded with a hex-looking prefix (e.g.
// --id-prefix 0198f2ab41c7e93d) reads like a production trace on camera.
const traceId = ctx.idPrefix;
const sessionId = `${ctx.idPrefix}-thread`;
- const traceTimestamp = utcDayStartMs();
+ // Deterministic business-hours offset so multi-day seeds don't stack every
+ // trace at midnight UTC (varies by seed and day, 08:xx-17:xx).
+ const intradayMs =
+ ((8 + ((seedNum * 7 + daysAgo * 5) % 10)) * 60 + ((seedNum * 13) % 60)) *
+ 60_000;
+ const traceTimestamp = utcDayStartMs() - daysAgo * 86_400_000 + intradayMs;
+ // Seed-keyed whole-trace duration scale (0.75x-1.35x): uniform scaling keeps
+ // parent/child containment intact while making latency charts vary across
+ // seeded runs instead of drawing flat lines.
+ const latencyScale = 0.75 + ((seedNum * 37) % 61) / 100;
+ const scaleMs = (offsetMs: number) => Math.round(offsetMs * latencyScale);
if (ctx.dryRun) {
return {
@@ -467,8 +665,8 @@ const run = async (
sessionIds: [sessionId],
counts: {
traces: 1,
- observations: PLAN.length,
- events: withV4 ? PLAN.length + 1 : 0,
+ observations: plan.length,
+ events: withV4 ? plan.length + 1 : 0,
},
verified: {},
links: [
@@ -480,7 +678,7 @@ const run = async (
};
}
- const root = PLAN[0];
+ const root = plan[0];
const trace = createTrace({
id: traceId,
project_id: ctx.projectId,
@@ -507,16 +705,29 @@ const run = async (
});
const keyToId = new Map<string, string>(
- PLAN.map((p, i) => [p.key, `${ctx.idPrefix}-obs-${i}`]),
+ plan.map((p, i) => [p.key, `${ctx.idPrefix}-obs-${i}`]),
);
- const observations: ObservationRecordInsertType[] = PLAN.map((p) => {
+ const observations: ObservationRecordInsertType[] = plan.map((p) => {
const prices = p.model ? MODEL_PRICES[p.model] : null;
const [usageInput, usageOutput] = p.usage ?? [0, 0];
const inputCost = prices ? usageInput * prices.input : 0;
const outputCost = prices ? usageOutput * prices.output : 0;
+ // Direct ClickHouse writes bypass ingestion, so run the same tool
+ // extraction the ingestion pipeline applies — otherwise the tool_calls /
+ // tool_call_names columns (dashboard tool-call measures) stay empty even
+ // though the generation outputs contain OpenAI-style tool_calls.
+ const { toolDefinitions, toolArguments } = extractToolsFromObservation(
+ p.input,
+ p.output,
+ );
+ const toolCallArrays = convertCallsToArrays(toolArguments);
+
return createObservation({
+ tool_definitions: convertDefinitionsToMap(toolDefinitions),
+ tool_calls: toolCallArrays.tool_calls,
+ tool_call_names: toolCallArrays.tool_call_names,
id: keyToId.get(p.key)!,
trace_id: traceId,
project_id: ctx.projectId,
@@ -525,12 +736,14 @@ const run = async (
parent_observation_id:
p.parentKey === null ? null : (keyToId.get(p.parentKey) ?? null),
name: p.name,
- start_time: traceTimestamp + p.start,
- end_time: traceTimestamp + p.end,
+ start_time: traceTimestamp + scaleMs(p.start),
+ end_time: traceTimestamp + scaleMs(p.end),
completion_start_time:
- p.ttft !== undefined ? traceTimestamp + p.start + p.ttft : null,
- level: "DEFAULT",
- status_message: null,
+ p.ttft !== undefined
+ ? traceTimestamp + scaleMs(p.start) + scaleMs(p.ttft)
+ : null,
+ level: p.level ?? "DEFAULT",
+ status_message: p.statusMessage ?? null,
version: null,
input: p.input !== undefined ? JSON.stringify(p.input) : null,
output: p.output !== undefined ? JSON.stringify(p.output) : null,
@@ -672,6 +885,20 @@ export const supportAgentScenario: ScenarioDefinition = {
default: false,
description: "also mirror into v4 events_full/events_core",
},
+ {
+ flag: "days-ago",
+ type: "number",
+ default: 0,
+ description:
+ "anchor the trace N days in the past (deterministic business-hours time-of-day) — run once per day to spread demo data over a date range",
+ },
+ {
+ flag: "fail",
+ type: "boolean",
+ default: false,
+ description:
+ "failing-tool variant: the first refund attempt errors (ERROR-level TOOL, Stripe 409) and the agent retries — one extra ReAct turn and tool call",
+ },
],
run,
};
diff --git a/worker/src/constants/langfuse-dashboards.json b/worker/src/constants/langfuse-dashboards.json
index a69ec18e6e..86557172bc 100644
--- a/worker/src/constants/langfuse-dashboards.json
+++ b/worker/src/constants/langfuse-dashboards.json
@@ -475,6 +475,149 @@
"chartType": "BAR_TIME_SERIES",
"chartConfig": {"type": "BAR_TIME_SERIES"},
"owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000001ad076tqc9kr4",
+ "createdAt": "2026-08-26T00:00:00.000Z",
+ "updatedAt": "2026-08-26T00:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "Total Tool Calls",
+ "description": "Total number of tool calls across observations, including parallel tool calls within a single observation",
+ "view": "OBSERVATIONS",
+ "dimensions": [],
+ "metrics": [{"agg": "sum", "measure": "toolCalls"}],
+ "filters": [],
+ "chartType": "NUMBER",
+ "chartConfig": {"type": "NUMBER", "row_limit": 100},
+ "owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000002ad076rurfiln",
+ "createdAt": "2026-08-26T00:00:00.000Z",
+ "updatedAt": "2026-08-26T00:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "Total Tool Calls (over time)",
+ "description": "Tool call volume over time, including parallel tool calls within a single observation",
+ "view": "OBSERVATIONS",
+ "dimensions": [],
+ "metrics": [{"agg": "sum", "measure": "toolCalls"}],
+ "filters": [],
+ "chartType": "BAR_TIME_SERIES",
+ "chartConfig": {"type": "BAR_TIME_SERIES"},
+ "owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000003ad07afj8ag4o",
+ "createdAt": "2026-08-26T00:00:00.000Z",
+ "updatedAt": "2026-08-27T12:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "Top 20 Called Tools",
+ "description": "Invocations per tool, including repeated calls within one observation, limited to the top 20",
+ "view": "OBSERVATIONS",
+ "dimensions": [{"field": "calledToolNames"}],
+ "metrics": [{"agg": "sum", "measure": "toolCallInvocations"}],
+ "filters": [],
+ "chartType": "HORIZONTAL_BAR",
+ "chartConfig": {"type": "HORIZONTAL_BAR", "row_limit": 20},
+ "owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000007ad07terrbytl",
+ "createdAt": "2026-08-26T12:00:00.000Z",
+ "updatedAt": "2026-08-26T12:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "Tool Errors by Tool",
+ "description": "Failed tool executions (ERROR level) per tool, limited to the top 20",
+ "view": "OBSERVATIONS",
+ "dimensions": [{"field": "name"}],
+ "metrics": [{"agg": "count", "measure": "count"}],
+ "filters": [
+ {"column": "type", "type": "stringOptions", "operator": "any of", "value": ["TOOL"]},
+ {"column": "level", "type": "stringOptions", "operator": "any of", "value": ["ERROR"]}
+ ],
+ "chartType": "HORIZONTAL_BAR",
+ "chartConfig": {"type": "HORIZONTAL_BAR", "row_limit": 20},
+ "owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000008ad07p95bytl0",
+ "createdAt": "2026-08-26T12:00:00.000Z",
+ "updatedAt": "2026-08-26T12:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "P 95 Tool Latency by Tool",
+ "description": "P95 execution latency of TOOL observations per tool, limited to the top 20",
+ "view": "OBSERVATIONS",
+ "dimensions": [{"field": "name"}],
+ "metrics": [{"agg": "p95", "measure": "latency"}],
+ "filters": [
+ {"column": "type", "type": "stringOptions", "operator": "any of", "value": ["TOOL"]}
+ ],
+ "chartType": "HORIZONTAL_BAR",
+ "chartConfig": {"type": "HORIZONTAL_BAR", "row_limit": 20},
+ "owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000009ad07p95ts0tl",
+ "createdAt": "2026-08-26T12:30:00.000Z",
+ "updatedAt": "2026-08-26T12:30:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "P 95 Tool Latency by Tool (over time)",
+ "description": "P95 execution latency of TOOL observations over time, one series per tool",
+ "view": "OBSERVATIONS",
+ "dimensions": [{"field": "name"}],
+ "metrics": [{"agg": "p95", "measure": "latency"}],
+ "filters": [
+ {"column": "type", "type": "stringOptions", "operator": "any of", "value": ["TOOL"]}
+ ],
+ "chartType": "LINE_TIME_SERIES",
+ "chartConfig": {"type": "LINE_TIME_SERIES"},
+ "owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000004ad07qwlf51rv",
+ "createdAt": "2026-08-26T00:00:00.000Z",
+ "updatedAt": "2026-08-26T00:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "Observations by Type",
+ "description": "Distribution of observations by type, e.g. agents, tools, generations, and spans",
+ "view": "OBSERVATIONS",
+ "dimensions": [{"field": "type"}],
+ "metrics": [{"agg": "count", "measure": "count"}],
+ "filters": [],
+ "chartType": "PIE",
+ "chartConfig": {"type": "PIE", "row_limit": 100},
+ "owner": "LANGFUSE"
+ },
+ {
+ "id": "cmtdm68000005ad07q12mamip",
+ "createdAt": "2026-08-26T00:00:00.000Z",
+ "updatedAt": "2026-08-26T00:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "P 95 Latency by Observation Type",
+ "description": "P95 latency segmented by observation type",
+ "view": "OBSERVATIONS",
+ "dimensions": [{"field": "type"}],
+ "metrics": [{"agg": "p95", "measure": "latency"}],
+ "filters": [],
+ "chartType": "LINE_TIME_SERIES",
+ "chartConfig": {"type": "LINE_TIME_SERIES"},
+ "owner": "LANGFUSE"
}
],
"dashboards": [
@@ -550,6 +693,29 @@
{"x": 4, "y": 0, "id": "f7d74d8d-09a5-4a5e-a8de-f48dc6ba6a4b", "type": "widget", "x_size": 4, "y_size": 5, "widgetId": "cmawk5sik00igad07kjetg17j"}
]
}
+ },
+ {
+ "id": "cmtdm68000006ad07dzdb73zw",
+ "createdAt": "2026-08-26T00:00:00.000Z",
+ "updatedAt": "2026-08-26T13:00:00.000Z",
+ "createdBy": null,
+ "updatedBy": null,
+ "projectId": null,
+ "name": "Langfuse Agent Dashboard",
+ "description": "Monitor agent tool usage: total tool calls, most-called tools, tool errors and latency, and observation type breakdowns.",
+ "owner": "LANGFUSE",
+ "definition": {
+ "widgets": [
+ {"x": 0, "y": 0, "id": "20d6ebcc-0b92-4cf1-92ff-934e13dbdc90", "type": "widget", "x_size": 3, "y_size": 4, "widgetId": "cmtdm68000001ad076tqc9kr4"},
+ {"x": 3, "y": 0, "id": "11c3f92f-76f8-46c9-a89e-8211812dd6d9", "type": "widget", "x_size": 5, "y_size": 4, "widgetId": "cmtdm68000003ad07afj8ag4o"},
+ {"x": 8, "y": 0, "id": "8c5d70ff-9ca1-4879-893f-187c5a62ea1d", "type": "widget", "x_size": 4, "y_size": 4, "widgetId": "cmtdm68000004ad07qwlf51rv"},
+ {"x": 0, "y": 4, "id": "7c8ecb81-c586-4819-8c5f-0cd2155fda2d", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000002ad076rurfiln"},
+ {"x": 6, "y": 4, "id": "b82d4e57-3a19-4f06-9c3d-1a7e6f25c842", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000008ad07p95bytl0"},
+ {"x": 0, "y": 8, "id": "3f6a1c2e-91b4-4c8a-a7d2-5e0b8c94d711", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000007ad07terrbytl"},
+ {"x": 6, "y": 8, "id": "df9d14b2-0133-4d1c-a83a-f1437470fc0e", "type": "widget", "x_size": 6, "y_size": 4, "widgetId": "cmtdm68000005ad07q12mamip"},
+ {"x": 0, "y": 12, "id": "5d1f8a03-6c2e-47b9-8e54-90ab37c1d6f2", "type": "widget", "x_size": 12, "y_size": 4, "widgetId": "cmtdm68000009ad07p95ts0tl"}
+ ]
+ }
}
]
-}
\ No newline at end of file
+}
diff --git a/worker/src/scripts/langfuseDashboards.test.ts b/worker/src/scripts/langfuseDashboards.test.ts
new file mode 100644
index 0000000000..7ccc361b9c
--- /dev/null
+++ b/worker/src/scripts/langfuseDashboards.test.ts
@@ -0,0 +1,17 @@
+import { describe, expect, it } from "vitest";
+import langfuseDashboards from "../constants/langfuse-dashboards.json";
+
+describe("langfuse template dashboards", () => {
+ it("ranks Top 20 Called Tools by per-tool invocations, not observation count", () => {
+ const widget = langfuseDashboards.widgets.find(
+ (entry) => entry.name === "Top 20 Called Tools",
+ );
+
+ expect(widget).toMatchObject({
+ view: "OBSERVATIONS",
+ dimensions: [{ field: "calledToolNames" }],
+ metrics: [{ agg: "sum", measure: "toolCallInvocations" }],
+ });
+ expect(widget?.description).not.toMatch(/observations that called/i);
+ });
+});