🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-use-before-define */
import { type FilterState, getGenerationLikeTypes } from "@langfuse/shared";
import type { MissingBucketValue } from "@/src/features/widgets/chart-library/chart-props";
export type TimeSeriesChartDataPoint = {
ts: number;
values: { label: string; value?: number }[];
};
import { type DatabaseRow } from "@/src/server/api/services/sqlInterface";
import { type ViewVersion } from "@langfuse/shared/query";
import { mapLegacyUiTableFilterToView } from "@/src/features/dashboard/lib/dashboardUiTableToViewMapping";
import { useScheduledDashboardExecuteQuery } from "@/src/features/dashboard/hooks/DashboardQuerySchedulerProvider";
type UseAllModelsOptions = {
enabled?: boolean;
queryId: string;
};
export const useAllModels = (
projectId: string,
globalFilterState: FilterState,
fromTimestamp: Date,
toTimestamp: Date,
metricsVersion: ViewVersion,
options?: UseAllModelsOptions,
) => {
const allModels = useScheduledDashboardExecuteQuery(
{
projectId,
version: metricsVersion,
query: {
view: "observations",
dimensions: [{ field: "providedModelName" }],
metrics: [],
filters: [
...mapLegacyUiTableFilterToView("observations", globalFilterState),
{
column: "type",
operator: "any of",
value: getGenerationLikeTypes(),
type: "stringOptions",
},
],
timeDimension: null,
fromTimestamp: fromTimestamp.toISOString(),
toTimestamp: toTimestamp.toISOString(),
orderBy: null,
},
},
{
trpc: {
context: {
skipBatch: true,
},
},
enabled: options?.enabled ?? true,
queryId: `${options?.queryId ?? "dashboard:all-models"}:models`,
},
);
return allModels.data ? extractAllModels(allModels.data) : [];
};
const extractAllModels = (
data: Record<string, unknown>[],
): { model: string; count: number }[] => {
return data
.filter((item) => item.providedModelName !== null)
.map((item) => ({
model: item.providedModelName as string,
count: item.count as number,
}));
};
type ChartData = {
label: string;
value?: number;
};
type FieldMappingItem = {
uniqueIdentifierColumns: {
accessor: string;
formatFct?: (value: string) => string;
}[];
valueColumn: string;
};
function generateChartLabelFromColumns(
uniqueIdentifierColumns: FieldMappingItem["uniqueIdentifierColumns"],
row: DatabaseRow,
): string {
return uniqueIdentifierColumns
.map(({ accessor, formatFct }) => {
if (row[accessor] === null || row[accessor] === undefined) return null;
return formatFct
? formatFct(row[accessor] as string)
: (row[accessor] as string);
})
.filter((value) => value !== null)
.join(" ");
}
// we get data for time series in the following format:
// ts: 123, label1: 1, label2: 2
// ts: 456, label1: 5, label2: 9
// This needs to be mapped to the following format:
// [{ts: 123, values: [{label1: 1, label2: 2}]}, {ts: 456, values: [{label1: 5, label2: 9}]]
export function extractTimeSeriesData(
data: DatabaseRow[],
timeColumn: string,
mapping: FieldMappingItem[],
): Map<number, ChartData[]> {
return data.reduce((acc: Map<number, ChartData[]>, curr: DatabaseRow) => {
const date = new Date(curr[timeColumn] as Date).getTime();
const reducedData: ChartData[] = [];
// Map the desired fields from the DatabaseRow to the ChartData based on the mapping provided
mapping.forEach((mapItem) => {
const chartLabel = generateChartLabelFromColumns(
mapItem.uniqueIdentifierColumns,
curr,
);
const columnValue = curr[mapItem.valueColumn];
if (
chartLabel &&
columnValue !== undefined &&
typeof chartLabel === "string"
) {
reducedData.push({
label: chartLabel,
value: columnValue ? (columnValue as number) : 0,
});
}
});
const existingData = acc.get(date);
if (existingData) {
existingData.push(...reducedData);
} else {
acc.set(date, reducedData);
}
return acc;
}, new Map<number, ChartData[]>());
}
export function fillMissingValuesAndTransform(
inputMap: Map<number, ChartData[]>,
labelsToAdd: string[] = [],
missingValue: MissingBucketValue = "zero",
): TimeSeriesChartDataPoint[] {
const result: TimeSeriesChartDataPoint[] = [];
inputMap.forEach((chartDataArray, timestamp) => {
const existingLabels = chartDataArray.map((value) => value.label);
// For each missing label, add the metric's honest no-data value: a real 0
// for additive metrics ("zero"), no value at all for non-additive ones
// ("gap") — an average over zero events has no honest value, and a
// fabricated 0 draws a fake drop to zero. (LFE-10694)
labelsToAdd.forEach((label) => {
if (!existingLabels.includes(label)) {
chartDataArray.push(
missingValue === "zero" ? { label, value: 0 } : { label },
);
}
});
result.push({
ts: timestamp,
values: chartDataArray,
});
});
return result;
}
export const isEmptyTimeSeries = ({
data,
isNullValueAllowed = false,
}: {
data: TimeSeriesChartDataPoint[];
isNullValueAllowed?: boolean;
}) => {
return (
data.length === 0 ||
data.every(
(item) =>
item.values.length === 0 ||
(isNullValueAllowed
? false
: // A gap-filled entry (no value) is as empty as a zero-filled one.
item.values.every(
(value) => value.value === 0 || value.value == null,
)),
)
);
};