🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { isoDateTime, InvalidRequestError } from "@langfuse/shared";
import { executeQuery } from "@langfuse/shared/query/server";
import {
dimension,
metric,
validateQuery,
viewDeclarations,
viewsV2,
} from "@langfuse/shared/query";
import {
MetricsQueryObjectV2,
publicGranularities,
} from "@/src/features/public-api/types/metrics";
import { defineTool } from "../../../core/define-tool";
import { McpAdvancedFilterBaseSchema } from "../../../core/filter-schema";
import { runMcpTool } from "../../../core/run-mcp-tool";
import { MCP_CLICKHOUSE_SETTINGS } from "../../../core/clickhouse-settings";
import { clampToDataAccessDays } from "@/src/features/entitlements/server/hasEntitlementLimit";
import { z } from "zod";
const DEFAULT_ROW_LIMIT = 100;
const normalizeMetricFilters = (
input: z.infer<typeof MetricsQueryObjectV2>,
): z.infer<typeof MetricsQueryObjectV2> => {
const tagsDimension = viewDeclarations.v2[input.view].dimensions.tags;
const tagsAreArray =
tagsDimension?.type === "string[]" || tagsDimension?.type === "arrayString";
if (!tagsAreArray) {
return input;
}
return {
...input,
filters: input.filters.map((filter) => {
if (filter.column !== "tags") {
return filter;
}
if (filter.type === "string" && filter.operator === "=") {
return {
type: "arrayOptions",
column: filter.column,
operator: "any of",
value: [filter.value],
};
}
if (filter.type === "stringOptions") {
return {
...filter,
type: "arrayOptions",
};
}
return filter;
}),
};
};
const normalizeMetricOrderByFields = (
input: z.infer<typeof MetricsQueryObjectV2>,
) => {
if (!input.orderBy) {
return input;
}
const metricAliases = input.metrics.map(
(metric) => `${metric.aggregation}_${metric.measure}`,
);
const reversedMetricAliases = new Map(
input.metrics.map((metric) => [
`${metric.measure}_${metric.aggregation}`,
`${metric.aggregation}_${metric.measure}`,
]),
);
const allowedOrderByFields = new Set([
...input.dimensions.map((dimension) => dimension.field),
...metricAliases,
...(input.timeDimension ? ["time_dimension"] : []),
]);
return {
...input,
orderBy: input.orderBy.map((orderBy) => {
const isDimensionField = input.dimensions.some(
(dimension) => dimension.field === orderBy.field,
);
const matchingMetrics = isDimensionField
? []
: input.metrics.filter((metric) => metric.measure === orderBy.field);
const normalizedField = (() => {
if (isDimensionField) {
return orderBy.field;
}
if (matchingMetrics.length === 1) {
return `${matchingMetrics[0].aggregation}_${matchingMetrics[0].measure}`;
}
return reversedMetricAliases.get(orderBy.field) ?? orderBy.field;
})();
if (!allowedOrderByFields.has(normalizedField)) {
throw new InvalidRequestError(
`Invalid orderBy field: ${orderBy.field}. Use returned metric aliases like 'sum_totalCost' or fields returned by getMetricsSchema.`,
);
}
return {
...orderBy,
field: normalizedField,
};
}),
};
};
const MetricsQueryObjectV2BaseSchema = z.object({
view: viewsV2,
dimensions: z.array(dimension).optional().default([]),
metrics: z.array(metric),
filters: z.array(McpAdvancedFilterBaseSchema).optional().default([]),
timeDimension: z
.object({
granularity: publicGranularities,
})
.optional(),
fromTimestamp: isoDateTime({ offset: true }).describe(
"Required start of the analytics time range, as an ISO 8601 timestamp with timezone (Z or an explicit offset). Must be before toTimestamp.",
),
toTimestamp: isoDateTime({ offset: true }).describe(
"Required end of the analytics time range, as an ISO 8601 timestamp with timezone. Must be after fromTimestamp. timeDimension.granularity controls aggregation buckets; it does not replace these explicit time bounds.",
),
orderBy: z
.array(
z.object({
field: z.string(),
direction: z.enum(["asc", "desc"]),
}),
)
.optional(),
config: z
.object({
bins: z.number().int().min(1).max(100).optional(),
row_limit: z.number().int().positive().lte(1000).optional(),
})
.optional(),
});
export const [queryMetricsTool, handleQueryMetrics] = defineTool({
name: "queryMetrics",
action: "metrics:read",
description:
"Answer analytics questions about the current Langfuse project, such as usage over time, model costs, latency, errors, scores, or grouped breakdowns by environment, trace, observation, model, user, session, tag, or score name.",
baseSchema: MetricsQueryObjectV2BaseSchema,
inputSchema: MetricsQueryObjectV2,
handler: async (input, context) => {
return await runMcpTool({
spanName: "mcp.metrics.query",
context,
attributes: {
"mcp.metrics_view": input.view,
},
fn: async () => {
const normalizedInputUnclamped = normalizeMetricOrderByFields(
normalizeMetricFilters(input),
);
const dataAccessWindow = clampToDataAccessDays({
plan: context.plan,
fromTimestamp: normalizedInputUnclamped.fromTimestamp,
});
const normalizedInput = {
...normalizedInputUnclamped,
fromTimestamp:
dataAccessWindow.effectiveFromTimestamp?.toISOString() ??
normalizedInputUnclamped.fromTimestamp,
};
const validation = validateQuery(normalizedInput, "v2");
if (!validation.valid) {
throw new InvalidRequestError(validation.reason);
}
const { config, ...query } = normalizedInput;
const queryParams = {
...query,
chartConfig: {
type: "TABLE",
...config,
row_limit: config?.row_limit ?? DEFAULT_ROW_LIMIT,
},
};
const result = await executeQuery(
context.projectId,
queryParams,
"v2",
true,
{ clickhouseSettings: MCP_CLICKHOUSE_SETTINGS },
);
return { data: result };
},
});
},
readOnlyHint: true,
});