🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/**
* v4 transition orchestration: Postgres summaries, sidebar migration actions,
* and re-exports for the tRPC router. SDK / events_core and system.query_log
* logic live in sibling modules.
*/
import {
AnalyticsIntegrationExportSource,
type Prisma,
type PrismaClient,
} from "@langfuse/shared/src/db";
import type { Session } from "next-auth";
import { isForceV3ExperienceProject } from "@langfuse/shared/src/server";
import {
readExperimentPostUsageCache,
readLegacyApiUsageCache,
} from "@/src/features/v4/server/v4TransitionCache";
import {
getLegacyApiUsageSummaries,
trimLegacyApiUsageRows,
} from "@/src/features/v4/server/v4TransitionQueryLogUsage";
import {
deriveExperimentInstrumentationMigration,
getSdkUsageSummaries,
getSdkUsageSeriesByProject,
} from "@/src/features/v4/server/v4TransitionSdkUsage";
import { isActionableLegacyApiUsage } from "@/src/features/v4/utils";
export { getSdkUsageSummaries, getLegacyApiUsageSummaries };
const legacyIntegrationExportSources =
new Set<AnalyticsIntegrationExportSource>([
AnalyticsIntegrationExportSource.TRACES_OBSERVATIONS,
AnalyticsIntegrationExportSource.TRACES_OBSERVATIONS_EVENTS,
]);
const TRACE_EVAL_TARGET = "trace";
const DATASET_EVAL_TARGET = "dataset";
const isLegacyIntegrationExportSource = (
exportSource: AnalyticsIntegrationExportSource | null | undefined,
) => exportSource != null && legacyIntegrationExportSources.has(exportSource);
const isEnabledLegacyIntegration = (
integration:
| { enabled: boolean; exportSource: AnalyticsIntegrationExportSource }
| null
| undefined,
) =>
Boolean(
integration?.enabled &&
isLegacyIntegrationExportSource(integration.exportSource),
);
type TraceLevelEvalSummaryResultRow = {
projectId: string;
traceLevelEvalCount: number;
};
type LegacyIntegrations = {
posthog: boolean;
mixpanel: boolean;
blobStorage: boolean;
};
const getAccessibleOrganizationProjectWhere = ({
orgId,
session,
}: {
orgId: string;
session: Session;
}): Prisma.ProjectWhereInput => {
const projectIds = session.user?.organizations
.find((organization) => organization.id === orgId)
?.projects.map((project) => project.id);
return {
orgId,
deletedAt: null,
...(session.user?.admin ? {} : { id: { in: projectIds ?? [] } }),
};
};
const getLegacyIntegrations = ({
posthogIntegration,
mixpanelIntegration,
blobStorageIntegration,
}: {
posthogIntegration:
| { enabled: boolean; exportSource: AnalyticsIntegrationExportSource }
| null
| undefined;
mixpanelIntegration:
| { enabled: boolean; exportSource: AnalyticsIntegrationExportSource }
| null
| undefined;
blobStorageIntegration:
| { enabled: boolean; exportSource: AnalyticsIntegrationExportSource }
| null
| undefined;
}): LegacyIntegrations => ({
posthog: isEnabledLegacyIntegration(posthogIntegration),
mixpanel: isEnabledLegacyIntegration(mixpanelIntegration),
blobStorage: isEnabledLegacyIntegration(blobStorageIntegration),
});
type V4TransitionPrisma = Pick<
PrismaClient,
| "project"
| "posthogIntegration"
| "mixpanelIntegration"
| "blobStorageIntegration"
| "evaluationRule"
>;
export const getAccessibleOrganizationProjects = async ({
prisma,
orgId,
session,
}: {
prisma: V4TransitionPrisma;
orgId: string;
session: Session;
}) =>
prisma.project.findMany({
where: getAccessibleOrganizationProjectWhere({ orgId, session }),
select: { id: true, name: true },
orderBy: { createdAt: "desc" },
});
export const getLegacyIntegrationSummaries = async ({
prisma,
projectIds,
}: {
prisma: V4TransitionPrisma;
projectIds: string[];
}) => {
if (projectIds.length === 0) return [];
const [posthogIntegrations, mixpanelIntegrations, blobStorageIntegrations] =
await Promise.all([
prisma.posthogIntegration.findMany({
where: { projectId: { in: projectIds } },
select: { projectId: true, enabled: true, exportSource: true },
}),
prisma.mixpanelIntegration.findMany({
where: { projectId: { in: projectIds } },
select: { projectId: true, enabled: true, exportSource: true },
}),
prisma.blobStorageIntegration.findMany({
where: { projectId: { in: projectIds } },
select: { projectId: true, enabled: true, exportSource: true },
}),
]);
const posthogByProjectId = new Map(
posthogIntegrations.map((integration) => [
integration.projectId,
integration,
]),
);
const mixpanelByProjectId = new Map(
mixpanelIntegrations.map((integration) => [
integration.projectId,
integration,
]),
);
const blobStorageByProjectId = new Map(
blobStorageIntegrations.map((integration) => [
integration.projectId,
integration,
]),
);
return projectIds.map((projectId) => {
const legacyIntegrations = getLegacyIntegrations({
posthogIntegration: posthogByProjectId.get(projectId),
mixpanelIntegration: mixpanelByProjectId.get(projectId),
blobStorageIntegration: blobStorageByProjectId.get(projectId),
});
return {
projectId,
legacyIntegrationCount:
Object.values(legacyIntegrations).filter(Boolean).length,
legacyIntegrations,
};
});
};
export const getTraceLevelEvalSummaries = async ({
prisma,
projectIds,
}: {
prisma: V4TransitionPrisma;
projectIds: string[];
}): Promise<TraceLevelEvalSummaryResultRow[]> => {
if (projectIds.length === 0) return [];
const counts = await prisma.evaluationRule.groupBy({
by: ["projectId"],
where: {
projectId: { in: projectIds },
targetObject: { in: [TRACE_EVAL_TARGET, DATASET_EVAL_TARGET] },
status: "ACTIVE",
timeScope: { has: "NEW" },
},
_count: { _all: true },
});
const countByProjectId = new Map(
counts.map((row) => [row.projectId, row._count._all]),
);
return projectIds.map((projectId) => ({
projectId,
traceLevelEvalCount: countByProjectId.get(projectId) ?? 0,
}));
};
/**
* Project-scoped migration evidence exposed by the v4 migration API.
* Keep MCP and UI consumers on the same data sources and detection windows.
*/
export const getProjectV4MigrationData = async ({
prisma,
projectId,
}: {
prisma: V4TransitionPrisma;
projectId: string;
}) => {
const projectIds = [projectId];
const [sdkUsage, legacyIntegrations, legacyApiUsage, traceLevelEvals] =
await Promise.all([
getSdkUsageSummaries({ projectIds }),
getLegacyIntegrationSummaries({ prisma, projectIds }),
getLegacyApiUsageSummaries({ projectIds }),
getTraceLevelEvalSummaries({ prisma, projectIds }),
]);
return {
projectId,
forceV3Experience: isForceV3ExperienceProject(projectId),
sdkUsage: sdkUsage[0]!,
legacyIntegrations: legacyIntegrations[0]!,
legacyApiUsage,
traceLevelEvals: traceLevelEvals[0]!,
};
};
/**
* Migration signals for the always-mounted sidebar "Action required" pill.
* SDK uses the same Redis + live gap-fill path as the panel summaries so the
* pill cannot go stale relative to them. Deprecated API / experiment signals
* read worker-maintained Redis caches when available (`null` = unknown).
*/
export const getMigrationActions = async ({
prisma,
projectId,
}: {
prisma: PrismaClient;
projectId: string;
}) => {
// Forced-v3 projects are partner-managed: the client suppresses every
// migration signal, so skip all I/O on this always-mounted hot path.
if (isForceV3ExperienceProject(projectId)) {
return {
forceV3Experience: true,
sdkActionNeeded: null,
experimentsActionNeeded: null,
apisActionNeeded: null,
evalsActionNeeded: false,
exportsActionNeeded: false,
};
}
const nowMs = Date.now();
const projectIds = [projectId];
const [
evalSummaries,
integrationSummaries,
seriesByProject,
postBlobs,
apiBlobs,
] = await Promise.all([
getTraceLevelEvalSummaries({ prisma, projectIds }),
getLegacyIntegrationSummaries({ prisma, projectIds }),
getSdkUsageSeriesByProject({ projectIds, nowMs }),
readExperimentPostUsageCache(projectIds),
readLegacyApiUsageCache(projectIds),
]);
const sdkUsageSeries = seriesByProject.get(projectId) ?? [];
const postBlob = postBlobs[0] ?? null;
const apiBlob = apiBlobs[0] ?? null;
const experimentsActionNeeded = (() => {
if (postBlob === null) {
return null;
}
if (postBlob.used === false) {
return false;
}
return ["required", "sdk_usage_inconclusive"].includes(
deriveExperimentInstrumentationMigration({
sdkUsageSeries,
postUsage: true,
}).status,
);
})();
return {
forceV3Experience: isForceV3ExperienceProject(projectId),
sdkActionNeeded: sdkUsageSeries.some(
(series) => series.actionLevel === "required",
),
experimentsActionNeeded,
apisActionNeeded:
apiBlob === null
? null
: trimLegacyApiUsageRows(apiBlob.rows, nowMs).some(
isActionableLegacyApiUsage,
),
evalsActionNeeded: (evalSummaries[0]?.traceLevelEvalCount ?? 0) > 0,
exportsActionNeeded:
(integrationSummaries[0]?.legacyIntegrationCount ?? 0) > 0,
};
};