🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { createHash } from "node:crypto";
import { prisma } from "@langfuse/shared/src/db";
import { InvalidRequestError } from "@langfuse/shared";
import {
logger,
QueueJobs,
recordIncrement,
redis,
type QueueName,
type TQueueJobTypes,
} from "@langfuse/shared/src/server";
import {
createTopicExecution,
enqueueTopicExecution,
getTopicProcessingMapIds,
getTopicRun,
getTopicClusteringSummaryStats,
getTopicExecutionQueueState,
getEnabledTopicsModels,
listTopicSummaries,
readTopicAssignments,
readTopicExecutionSummary,
TOPICS_ACTION,
TOPICS_SYSTEM_USER_ID,
TOPICS_TRACE_BATCH_SIZE,
TOPIC_CLUSTERING_CAPACITY_ERROR,
TopicsAutomationQueue,
writeTopicAssignments,
} from "@langfuse/shared/topics/server";
import type {
TopicEmbeddingConfig,
TopicExecutionSummary,
TopicFacetRef,
TopicRun,
TopicSummary,
TopicTimeRange,
} from "@langfuse/shared/topics";
import { topicAssignmentRows } from "./assignments";
import { TopicMetrics } from "./metrics";
import { topicClusterSettings } from "./numeric";
import { env } from "../../env";
type AutomationBatch = TQueueJobTypes[QueueName.TopicsAutomation]["payload"];
const DISCOVERY_WINDOW_MS = 7 * 24 * 60 * 60_000;
const hash = (value: unknown) =>
createHash("sha256").update(JSON.stringify(value)).digest("hex");
/**
* Queues assignment for the complete summaries a trace batch stored, one job per project,
* vector space and 100 traces. Job IDs repeat for a retried batch, so it queues nothing twice.
*/
export async function enqueueTopicsAutomation(
batchId: string,
rows: TopicSummary[],
): Promise<void> {
const queue = TopicsAutomationQueue.getInstance();
if (!queue) return;
for (const batch of groupStoredSummaries(rows).flatMap(splitTraces)) {
// Facets belong in the ID: a retry can store a facet the first attempt failed.
const id = hash([
batchId,
batch.projectId,
batch.embeddingConfig,
batch.facets,
batch.traceIds,
]);
await queue.add(
QueueJobs.TopicsAutomation,
{
id,
timestamp: new Date(),
name: QueueJobs.TopicsAutomation,
payload: batch,
},
{ jobId: id },
);
}
}
/**
* Assigns the batch's current complete summaries to each facet's compatible serving map,
* skipping ones already assigned, or starts the facet's first discovery once enough exist.
*/
export async function processTopicsAutomation(
batch: AutomationBatch,
): Promise<void> {
const { projectId, embeddingConfig } = batch;
for (const facet of batch.facets) {
const [map] = await getTopicProcessingMapIds(
projectId,
[facet],
embeddingConfig,
);
const run = map?.runId ? await getTopicRun(projectId, map.runId) : null;
if (run?.status === "completed") {
const summaries = await currentSummaries(batch, facet);
if (summaries.length > 0) await assign(batch, facet, summaries, run);
} else await startDiscovery(batch, facet);
}
}
/** One batch per project and vector space, spanning its summaries' unit start times. */
function groupStoredSummaries(rows: TopicSummary[]): AutomationBatch[] {
const groups = new Map<string, AutomationBatch>();
for (const row of rows) {
if (row.state !== "complete" || row.traceId === null) continue;
const embeddingConfig = {
embeddingModel: row.embeddingModel,
embeddingDimensions: row.embedding.length,
};
const key = JSON.stringify([row.projectId, embeddingConfig]);
const start = new Date(row.unitStartTime);
const end = new Date(start.getTime() + 1);
const group = groups.get(key) ?? {
projectId: row.projectId,
embeddingConfig,
timeRange: { from: start, to: end },
traceIds: [],
facets: [],
};
if (start < group.timeRange.from) group.timeRange.from = start;
if (end > group.timeRange.to) group.timeRange.to = end;
if (!group.traceIds.includes(row.traceId)) group.traceIds.push(row.traceId);
if (
!group.facets.some(
(facet) =>
facet.facetId === row.facetId && facet.version === row.facetVersion,
)
)
group.facets.push({ facetId: row.facetId, version: row.facetVersion });
groups.set(key, group);
}
return [...groups.values()].map((group) => ({
...group,
facets: group.facets.sort(
(a, b) => a.facetId.localeCompare(b.facetId) || a.version - b.version,
),
}));
}
function splitTraces(batch: AutomationBatch): AutomationBatch[] {
const batches: AutomationBatch[] = [];
for (
let index = 0;
index < batch.traceIds.length;
index += TOPICS_TRACE_BATCH_SIZE
)
batches.push({
...batch,
traceIds: batch.traceIds.slice(index, index + TOPICS_TRACE_BATCH_SIZE),
});
return batches;
}
async function currentSummaries(batch: AutomationBatch, facet: TopicFacetRef) {
const { embeddingConfig } = batch;
const rows = await listTopicSummaries(
batch.projectId,
{
traceIds: batch.traceIds,
facetId: facet.facetId,
facetVersion: facet.version,
},
batch.timeRange,
);
return rows.filter(
(row) =>
row.state === "complete" &&
row.traceId !== null &&
row.embeddingModel === embeddingConfig.embeddingModel &&
row.embedding.length === embeddingConfig.embeddingDimensions,
);
}
async function assign(
batch: AutomationBatch,
facet: TopicFacetRef,
summaries: TopicSummary[],
run: TopicRun,
) {
const metrics = new TopicMetrics();
await metrics.measure("assignment", async () => {
const assigned = new Set(
(
await readTopicAssignments(
batch.projectId,
facet,
summaries,
run.id,
batch.timeRange,
)
).map((row) => JSON.stringify([row.traceId, row.summaryProcessedAt])),
);
const pending = summaries.filter(
(row) => !assigned.has(JSON.stringify([row.traceId, row.processedAt])),
);
if (pending.length === 0) return;
const rows = topicAssignmentRows({
projectId: batch.projectId,
facet,
embeddingConfig: batch.embeddingConfig,
summaries: pending,
run,
origin: "online",
assignedAt: new Date().toISOString(),
});
await writeTopicAssignments(rows);
const topics = rows.filter((row) => row.topicId !== null).length;
metrics.result("assignment", "assigned", topics);
metrics.result("assignment", "outlier", rows.length - topics);
});
}
/** Each capacity failure gets one deterministic successor with a smaller window. */
async function startDiscovery(batch: AutomationBatch, facet: TopicFacetRef) {
const { projectId, embeddingConfig } = batch;
if (!(await usesCurrentEmbedding(projectId, embeddingConfig))) return;
let existing = await latestDiscovery(batch, facet);
if (await continueDiscovery(batch, facet, existing)) return;
if (await hasMapOrFit(projectId, facet)) return;
// Bound concurrent searches and repeated probes when no admissible window exists.
const discoveryKey = hash(["topics-discovery", facet, embeddingConfig]);
const gateKey = `langfuse:topics:discovery:${projectId}:${discoveryKey}`;
if (redis && (await redis.set(gateKey, "1", "EX", 60, "NX")) !== "OK") {
await deferDiscovery(batch, facet);
return;
}
// Another callback may have advanced discovery while this one waited for Redis.
existing = await latestDiscovery(batch, facet);
if (await continueDiscovery(batch, facet, existing)) return;
if (await hasMapOrFit(projectId, facet)) return;
const timeRange = await selectDiscoveryWindow(
projectId,
facet,
embeddingConfig,
existing?.input.operation === "update"
? existing.input.timeRange
: undefined,
).catch(async (error: unknown) => {
await redis?.del(gateKey);
throw error;
});
if (!timeRange) return;
// Probes can outlive the cooldown; never replace a newer attempt's window.
const latest = await latestDiscovery(batch, facet);
if (latest?.id !== existing?.id || latest?.status !== existing?.status) {
await deferDiscovery(batch, facet);
return;
}
if (await hasMapOrFit(projectId, facet)) return;
const requestId = existing ? hash([discoveryKey, existing.id]) : discoveryKey;
// Monitor before dispatch so the last incoming batch can recover a later failure.
await deferDiscovery(batch, facet);
let execution;
try {
execution = await createTopicExecution(
{
projectId,
requestId,
operation: "update",
facets: [facet],
embeddingConfig,
timeRange,
exploratory: false,
},
requestId,
TOPICS_SYSTEM_USER_ID,
);
} catch (error) {
// A concurrent job created it first with its own time range.
if (error instanceof InvalidRequestError) return;
throw error;
}
await enqueueTopicExecution(projectId, execution.id);
recordIncrement("langfuse.topics.automatic_discoveries", 1);
logger.info("Started automatic Topics discovery", {
projectId,
facetId: facet.facetId,
facetVersion: facet.version,
});
}
/** Read only the latest matching request, including earlier discovery request formats. */
async function latestDiscovery(batch: AutomationBatch, facet: TopicFacetRef) {
const row = await prisma.batchAction.findFirst({
where: {
projectId: batch.projectId,
actionType: TOPICS_ACTION,
userId: TOPICS_SYSTEM_USER_ID,
AND: [
{ config: { path: ["inputSettings", "operation"], equals: "update" } },
{ config: { path: ["inputSettings", "facets"], equals: [facet] } },
{
config: {
path: ["inputSettings", "embeddingConfig"],
equals: batch.embeddingConfig,
},
},
],
},
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
select: { id: true },
});
return row ? readTopicExecutionSummary(batch.projectId, row.id) : null;
}
/** Active work is monitored; only a capacity failure permits another discovery. */
async function continueDiscovery(
batch: AutomationBatch,
facet: TopicFacetRef,
existing: TopicExecutionSummary | null,
): Promise<boolean> {
if (!existing) return false;
if (existing.status === "queued") {
await deferDiscovery(batch, facet);
await enqueueTopicExecution(batch.projectId, existing.id);
return true;
}
if (existing.status === "running") {
const state = await getTopicExecutionQueueState(existing);
if (["failed", "completed", "unknown", "missing"].includes(state)) {
// Queue failure can outlive persisted running status. Recheck progress in
// case admission recorded a capacity failure while the queue was read.
const current = await readTopicExecutionSummary(
batch.projectId,
existing.id,
);
if (current && discoveryCapacityFailure(current, facet))
await deferDiscovery(batch, facet);
return true;
}
await deferDiscovery(batch, facet);
return true;
}
const capacityFailure = discoveryCapacityFailure(existing, facet);
if (!capacityFailure) return true;
// Admission can fail before a precreated run enters the fitting lifecycle.
if (capacityFailure.runId)
await prisma.topicClusteringRun.updateMany({
where: {
projectId: batch.projectId,
id: capacityFailure.runId,
facetId: facet.facetId,
facetVersion: facet.version,
status: "pending",
},
data: {
status: "failed",
error: TOPIC_CLUSTERING_CAPACITY_ERROR,
finishedAt: new Date(),
},
});
return false;
}
function discoveryCapacityFailure(
execution: TopicExecutionSummary,
facet: TopicFacetRef,
) {
return ["failed", "completed_with_errors"].includes(execution.status)
? execution.facets.find(
(progress) =>
progress.facetId === facet.facetId &&
progress.facetVersion === facet.version &&
progress.outcome === "failed" &&
progress.error === TOPIC_CLUSTERING_CAPACITY_ERROR,
)
: undefined;
}
/** One delayed check per facet/model, including a successor to an active check. */
async function deferDiscovery(batch: AutomationBatch, facet: TopicFacetRef) {
const queue = TopicsAutomationQueue.getInstance();
if (!queue)
throw new Error("Topics discovery requires its automation queue.");
const timestamp = new Date();
const delay = 60_000;
const key = hash([batch.projectId, facet, batch.embeddingConfig]);
const id = hash([key, Math.floor((timestamp.getTime() + delay) / delay)]);
await queue.add(
QueueJobs.TopicsAutomation,
{
id,
timestamp,
name: QueueJobs.TopicsAutomation,
payload: { ...batch, facets: [facet] },
},
{ jobId: id, delay, deduplication: { id: key, keepLastIfActive: true } },
);
}
/** Keep a complete recent cohort; only aggregate counts and byte estimates are loaded. */
async function selectDiscoveryWindow(
projectId: string,
facet: TopicFacetRef,
embeddingConfig: TopicEmbeddingConfig,
failedWindow?: TopicTimeRange,
): Promise<TopicTimeRange | null> {
const to = new Date();
let earliest = to.getTime() - DISCOVERY_WINDOW_MS;
let latest = to.getTime();
const minimumCount = topicClusterSettings(false).minimumCount;
if (env.LANGFUSE_TOPICS_MAX_CLUSTERING_ROWS < minimumCount) return null;
const limits = {
maxRows: Math.max(
minimumCount,
Math.floor(env.LANGFUSE_TOPICS_MAX_CLUSTERING_ROWS * 0.8),
),
maxBytes: Math.max(
1,
Math.floor(env.LANGFUSE_TOPICS_MAX_CLUSTERING_BYTES * 0.8),
),
};
const firstDuration = failedWindow
? Math.min(
DISCOVERY_WINDOW_MS,
Math.max(
1,
Math.floor(
(failedWindow.to.getTime() - failedWindow.from.getTime()) / 2,
),
),
)
: DISCOVERY_WINDOW_MS;
// A seven-day range has fewer than 2^30 millisecond boundaries.
for (let check = 0; check < 32 && earliest < latest; check++) {
const from =
check === 0
? to.getTime() - firstDuration
: Math.floor((earliest + latest) / 2);
const timeRange = { from: new Date(from), to };
const { count, bytes } = await getTopicClusteringSummaryStats(
projectId,
facet.facetId,
facet.version,
embeddingConfig,
timeRange,
limits,
);
if (count < minimumCount) {
if (from === to.getTime() - DISCOVERY_WINDOW_MS) return null;
latest = from;
} else if (count > limits.maxRows || bytes > limits.maxBytes) {
earliest = from + 1;
} else {
return timeRange;
}
}
return null;
}
/** Automatic processing is on with the embedding model these summaries used. */
async function usesCurrentEmbedding(
projectId: string,
embeddingConfig: TopicEmbeddingConfig,
) {
const models = (await getEnabledTopicsModels([projectId])).get(projectId);
return (
models !== undefined &&
models.embedding.model === embeddingConfig.embeddingModel &&
models.embedding.dimensions === embeddingConfig.embeddingDimensions
);
}
/** A map or fit, finished or in progress, exists for this or a newer facet version. */
async function hasMapOrFit(projectId: string, facet: TopicFacetRef) {
const run = await prisma.topicClusteringRun.findFirst({
where: {
projectId,
facetId: facet.facetId,
facetVersion: { gte: facet.version },
status: { in: ["pending", "running", "completed"] },
},
select: { id: true },
});
return run !== null;
}