🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): per-project trace idle time in tracing_readiness_settings (#18595)
Lysander Kiesel3a2551dd35be
diff --git a/packages/shared/prisma/migrations/20261012060000_add_tracing_readiness_settings/migration.sql b/packages/shared/prisma/migrations/20261012060000_add_tracing_readiness_settings/migration.sqlnew file mode 100644index 0000000000..9f0201806f--- /dev/null+++ b/packages/shared/prisma/migrations/20261012060000_add_tracing_readiness_settings/migration.sql@@ -0,0 +1,20 @@+-- CreateTable+CREATE TABLE "tracing_readiness_settings" (+ "project_id" TEXT NOT NULL,+ "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "created_by" TEXT,+ "updated_by" TEXT,+ "trace_idle_ms" INTEGER,++ CONSTRAINT "tracing_readiness_settings_pkey" PRIMARY KEY ("project_id")+);++-- AddForeignKey+ALTER TABLE "tracing_readiness_settings" ADD CONSTRAINT "tracing_readiness_settings_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;++-- AddForeignKey+ALTER TABLE "tracing_readiness_settings" ADD CONSTRAINT "tracing_readiness_settings_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;++-- AddForeignKey+ALTER TABLE "tracing_readiness_settings" ADD CONSTRAINT "tracing_readiness_settings_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex 0b72c47cbe..21ecab1c93 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -77,6 +77,8 @@ model User {tableViewPresetUpdated TableViewPreset[] @relation("UpdatedByUser")monitorsCreated Monitor[] @relation("CreatedByUser")monitorsUpdated Monitor[] @relation("UpdatedByUser")+ tracingReadinessCreated TracingReadinessSettings[] @relation("CreatedByUser")+ tracingReadinessUpdated TracingReadinessSettings[] @relation("UpdatedByUser")annotationQueueAssignment AnnotationQueueAssignment[]surveys Survey[]commentReactions CommentReaction[]@@ -171,6 +173,7 @@ model Project {Evaluator Evaluator[]EvaluationRule EvaluationRule[]EvaluationRuleEvaluatorAssignment EvaluationRuleEvaluatorAssignment[]+ tracingReadinessSettings TracingReadinessSettings?LlmApiKeys LlmApiKeys[]PosthogIntegration PosthogIntegration[]MixpanelIntegration MixpanelIntegration[]@@ -1244,6 +1247,7 @@ model EvaluationRule {filter Jsonsampling Decimaldelay Int+ // Unused; kept until no deployed version selects it, then dropped.idleTime Int? @map("idle_time") // millisecondstimeScope String[] @default(["NEW"]) @map("time_scope")@@ -1253,6 +1257,27 @@ model EvaluationRule {@@map("evaluation_rules")}+// When tracing data is ready for downstream work (Topics, trace-level+// evaluations). One row per project; null columns fall back to the deployment+// default. Session settings join as their own nullable columns.+model TracingReadinessSettings {+ projectId String @id @map("project_id")+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)+ createdAt DateTime @default(now()) @map("created_at")+ updatedAt DateTime @default(now()) @updatedAt @map("updated_at")++ createdBy String? @map("created_by")+ createdByUser User? @relation("CreatedByUser", fields: [createdBy], references: [id], onDelete: SetNull)+ updatedBy String? @map("updated_by")+ updatedByUser User? @relation("UpdatedByUser", fields: [updatedBy], references: [id], onDelete: SetNull)++ // A trace is due this long after its latest event. Null uses+ // LANGFUSE_TRACE_BATCH_IDLE_MS. Bounds are enforced on write, not in the DB.+ traceIdleMs Int? @map("trace_idle_ms")++ @@map("tracing_readiness_settings")+}+model EvaluationRuleEvaluatorAssignment {id String @id @default(cuid())createdAt DateTime @default(now()) @map("created_at")diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex 9ed589de76..d7a3bffdf4 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -31,6 +31,7 @@ export * from "./services/commentFilterService";export * from "./datasets/schemaValidation";export * from "./datasets/schemaTypes";export * from "./evalJobConfigCache";+export * from "./tracingReadinessSettings";export * from "./evals/codeEvalDispatchers";export * from "./evals/codeEvalExecution";export * from "./evals/evalExecutionMetadata";diff --git a/packages/shared/src/server/topics/postgres.ts b/packages/shared/src/server/topics/postgres.tsindex 2facb4006d..63fd9d8443 100644--- a/packages/shared/src/server/topics/postgres.ts+++ b/packages/shared/src/server/topics/postgres.ts@@ -342,7 +342,6 @@ export async function saveTopicRule() as Prisma.InputJsonValue,sampling: 1,delay: 0,- idleTime: null,targetObject: EvalTargetObject.TRACE,timeScope: ["NEW"],status: "ACTIVE" as const,diff --git a/packages/shared/src/server/tracingReadinessSettings.ts b/packages/shared/src/server/tracingReadinessSettings.tsnew file mode 100644index 0000000000..23a04dd1f5--- /dev/null+++ b/packages/shared/src/server/tracingReadinessSettings.ts@@ -0,0 +1,118 @@+import { z } from "zod";+import { prisma } from "../db";+import { logger } from "./logger";+import { redis } from "./redis/redis";++// The minimum stays at least 2x the 30 s dispatch interval so event batches of+// one trace are not dispatched separately. The maximum stays below the 2 h+// pending TTL, after which idle traces are dropped before dispatch. The DB does+// not enforce these, so every write goes through this schema.+export const TRACE_IDLE_MS_MIN = 60_000;+export const TRACE_IDLE_MS_MAX = 3_600_000;++// Cache fills do not replace saved values. A failed cache write can leave a+// previous value until it expires.+const CACHE_TTL_SECONDS = 300;+const CACHE_KEY_PREFIX = "langfuse:tracing-readiness-settings";++export const tracingReadinessSettingsSchema = z.object({+ traceIdleMs: z+ .number()+ .int()+ .min(TRACE_IDLE_MS_MIN)+ .max(TRACE_IDLE_MS_MAX)+ .nullable(),+});++export type TracingReadinessSettings = z.infer<+ typeof tracingReadinessSettingsSchema+>;++const cacheKey = (projectId: string) => `${CACHE_KEY_PREFIX}:${projectId}`;++/**+ * Null fields mean the deployment default; a project without a row has all+ * nulls. Served from Redis when cached; Postgres errors propagate.+ */+export async function getTracingReadinessSettings(+ projectId: string,+): Promise<TracingReadinessSettings> {+ const cached = await readCache(projectId);+ if (cached) return cached;++ const row = await prisma.tracingReadinessSettings.findUnique({+ where: { projectId },+ select: { traceIdleMs: true },+ });+ const parsed = tracingReadinessSettingsSchema.safeParse({+ traceIdleMs: row?.traceIdleMs ?? null,+ });+ if (!parsed.success) {+ logger.warn("Invalid stored trace idle time, using the default", {+ projectId,+ traceIdleMs: row?.traceIdleMs,+ });+ }+ const settings = parsed.success ? parsed.data : { traceIdleMs: null };+ try {+ await redis?.set(+ cacheKey(projectId),+ JSON.stringify(settings),+ "EX",+ CACHE_TTL_SECONDS,+ "NX",+ );+ } catch (error) {+ logger.warn("Failed to cache tracing readiness settings", {+ projectId,+ error,+ });+ }+ return settings;+}++export async function saveTracingReadinessSettings(+ projectId: string,+ settings: TracingReadinessSettings,+ { userId = null }: { userId?: string | null } = {},+): Promise<TracingReadinessSettings> {+ const data = tracingReadinessSettingsSchema.parse(settings);+ const row = await prisma.tracingReadinessSettings.upsert({+ where: { projectId },+ create: { projectId, ...data, createdBy: userId, updatedBy: userId },+ update: { ...data, updatedBy: userId },+ select: { traceIdleMs: true },+ });+ const saved = { traceIdleMs: row.traceIdleMs };+ try {+ await redis?.set(+ cacheKey(projectId),+ JSON.stringify(saved),+ "EX",+ CACHE_TTL_SECONDS,+ );+ } catch (error) {+ logger.warn("Failed to cache saved tracing readiness settings", {+ projectId,+ error,+ });+ }+ return saved;+}++async function readCache(+ projectId: string,+): Promise<TracingReadinessSettings | null> {+ try {+ const cached = await redis?.get(cacheKey(projectId));+ if (!cached) return null;+ const parsed = tracingReadinessSettingsSchema.safeParse(JSON.parse(cached));+ return parsed.success ? parsed.data : null;+ } catch (error) {+ logger.warn("Failed to read cached tracing readiness settings", {+ projectId,+ error,+ });+ return null;+ }+}diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex cb0165f835..092a962689 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -26,6 +26,7 @@ import {type TQueueJobTypes,} from "@langfuse/shared/src/server";import * as shared from "@langfuse/shared/src/server";+import { prisma } from "@langfuse/shared/src/db";import { env } from "../env";import { TraceBatchMetricsRunner } from "../features/traceBatching/TraceBatchMetricsRunner";import {@@ -51,6 +52,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {recordGauge: vi.fn(),recordIncrement: vi.fn(),getS3EventStorageClient: vi.fn(),+ getTracingReadinessSettings: vi.fn(original.getTracingReadinessSettings),};});@@ -870,6 +872,182 @@ describe("trace micro-batch scheduling with Redis", () => {);});+ it("sets the due time from the project's trace idle time, capped below the pending TTL", async () => {+ const orgId = `org-${randomUUID()}`;+ const projectId = `project-${randomUUID()}`;+ await prisma.organization.create({+ data: { id: orgId, name: "Idle time org" },+ });+ await prisma.project.create({+ data: { id: projectId, orgId, name: "Idle time project" },+ });+ try {+ await shared.saveTracingReadinessSettings(projectId, {+ traceIdleMs: 3_600_000,+ });+ const now = Date.now();+ // The Lua script reads Redis time, so allow for clock skew.+ const dueOffset = async (traceId: string) =>+ Number(await client().zscore(dueKey, member(projectId, traceId))) - now;++ await trackTraceBatchActivity(projectId, [event("custom")]);+ await trackTraceBatchActivity("project", [event("default")]);+ expect(await dueOffset("custom")).toBeGreaterThan(3_590_000);+ expect(await dueOffset("custom")).toBeLessThan(3_610_000);+ expect(+ Number(await client().zscore(dueKey, member("project", "default"))) -+ now,+ ).toBeLessThan(610_000);++ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 1_800_000;+ await trackTraceBatchActivity(projectId, [event("capped")]);+ expect(await dueOffset("capped")).toBeLessThan(1_780_000);+ expect(await dueOffset("capped")).toBeGreaterThan(1_760_000);+ } finally {+ await prisma.organization.delete({ where: { id: orgId } });+ }+ });++ it("caches the project's settings in Redis with an expiry and replaces them on save", async () => {+ const orgId = `org-${randomUUID()}`;+ const projectId = `project-${randomUUID()}`;+ const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;+ await prisma.organization.create({+ data: { id: orgId, name: "Idle time cache org" },+ });+ await prisma.project.create({+ data: { id: projectId, orgId, name: "Idle time cache project" },+ });+ try {+ await trackTraceBatchActivity(projectId, [event("uncached")]);+ expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({+ traceIdleMs: null,+ });+ const ttl = await connection.ttl(cacheKey);+ expect(ttl).toBeGreaterThan(0);+ expect(ttl).toBeLessThanOrEqual(300);+ await shared.saveTracingReadinessSettings(projectId, {+ traceIdleMs: 300_000,+ });+ expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({+ traceIdleMs: 300_000,+ });+ await trackTraceBatchActivity(projectId, [event("saved")]);+ const offset =+ Number(await client().zscore(dueKey, member(projectId, "saved"))) -+ Date.now();+ expect(offset).toBeGreaterThan(290_000);+ expect(offset).toBeLessThan(310_000);+ } finally {+ await prisma.organization.delete({ where: { id: orgId } });+ await connection.del(cacheKey);+ }+ });++ it("does not let an earlier database read overwrite newly saved settings", async () => {+ const orgId = `org-${randomUUID()}`;+ const projectId = `project-${randomUUID()}`;+ const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;+ await prisma.organization.create({+ data: { id: orgId, name: "Idle time race org" },+ });+ await prisma.project.create({+ data: { id: projectId, orgId, name: "Idle time race project" },+ });+ try {+ await shared.saveTracingReadinessSettings(projectId, {+ traceIdleMs: 120_000,+ });+ await connection.del(cacheKey);++ let releaseFill!: () => void;+ const fillHeld = new Promise<void>((resolve) => {+ releaseFill = resolve;+ });+ let signalFill!: () => void;+ const fillStarted = new Promise<void>((resolve) => {+ signalFill = resolve;+ });+ const actualShared = await vi.importActual<+ typeof import("@langfuse/shared/src/server")+ >("@langfuse/shared/src/server");+ if (!actualShared.redis)+ throw new Error("Redis is required for this test");+ const originalSet = actualShared.redis.set.bind(actualShared.redis);+ vi.spyOn(actualShared.redis, "set").mockImplementationOnce(+ async (...args) => {+ signalFill();+ await fillHeld;+ return originalSet(...args);+ },+ );++ const earlierRead = shared.getTracingReadinessSettings(projectId);+ await fillStarted;+ try {+ await shared.saveTracingReadinessSettings(projectId, {+ traceIdleMs: 300_000,+ });+ } finally {+ releaseFill();+ }+ await earlierRead;++ expect(await shared.getTracingReadinessSettings(projectId)).toEqual({+ traceIdleMs: 300_000,+ });+ } finally {+ await prisma.organization.delete({ where: { id: orgId } });+ await connection.del(cacheKey);+ }+ });++ it("uses and caches the default for an out-of-range stored idle time", async () => {+ const orgId = `org-${randomUUID()}`;+ const projectId = `project-${randomUUID()}`;+ const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;+ await prisma.organization.create({+ data: { id: orgId, name: "Invalid idle time org" },+ });+ await prisma.project.create({+ data: { id: projectId, orgId, name: "Invalid idle time project" },+ });+ try {+ await prisma.tracingReadinessSettings.create({+ data: { projectId, traceIdleMs: 1 },+ });++ expect(await shared.getTracingReadinessSettings(projectId)).toEqual({+ traceIdleMs: null,+ });+ expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({+ traceIdleMs: null,+ });+ } finally {+ await prisma.organization.delete({ where: { id: orgId } });+ await connection.del(cacheKey);+ }+ });++ it("uses the default when the idle time read fails", async () => {+ const projectId = `project-${randomUUID()}`;+ vi.mocked(shared.getTracingReadinessSettings).mockRejectedValueOnce(+ new Error("Postgres unavailable"),+ );++ await trackTraceBatchActivity(projectId, [event("failed")]);++ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.trace_batch.idle_time_read_failures",+ 1,+ );+ const offset =+ Number(await client().zscore(dueKey, member(projectId, "failed"))) -+ Date.now();+ expect(offset).toBeGreaterThan(590_000);+ expect(offset).toBeLessThan(610_000);+ });+it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;const evaluate = vi.spyOn(client(), "eval");diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 37bff1ba7f..cb31080d15 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -22,8 +22,8 @@ Topics reuses the evaluator and evaluation-rule tables:original request, including its selection criteria. Retries never re-evaluate a rule.Postgres uses normal Prisma migrations; `topic_clustering_runs` is the only new-Topics table. `EvaluationRule.idleTime` stores an optional duration in milliseconds;-it does not change automatic scheduling or trace-batch admission.+Topics table. Idle time is per project, not per rule: see+`tracing_readiness_settings` in `worker/src/features/traceBatching/README.md`.## Setupdiff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.tsindex 2651be8400..bda474bea3 100644--- a/worker/src/features/topics/storage.integration.test.ts+++ b/worker/src/features/topics/storage.integration.test.ts@@ -155,7 +155,6 @@ describe("Topics eval-backed configuration", () => {).toMatchObject({targetObject: "trace",delay: 0,- idleTime: null,timeScope: ["NEW"],status: "ACTIVE",});diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex 599de25406..cb56fef35f 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -29,7 +29,7 @@ templates. The local dev template sets the idle time below.| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. || `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. || `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. |+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 s to 60 min) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, so the next activity reads the new value. A failed read uses the default and is not cached. || `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). || `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.tsindex fc9b71a3de..afce38c302 100644--- a/worker/src/features/traceBatching/traceBatching.ts+++ b/worker/src/features/traceBatching/traceBatching.ts@@ -1,6 +1,7 @@import { createHash, randomUUID } from "node:crypto";import { xxh32 } from "@node-rs/xxhash";import {+ getTracingReadinessSettings,logger,QueueJobs,redis,@@ -758,6 +759,7 @@ export async function trackTraceBatchActivity();if (entries.length === 0) return;if (!redis) throw new Error("Trace batching requires Redis");+ const idleMs = await resolveTraceIdleMs(projectId);// Bounded scripts keep ingestion from monopolizing the global Redis slot.for (let offset = 0; offset < entries.length; offset += CHUNK_SIZE) {const chunk = entries.slice(offset, offset + CHUNK_SIZE);@@ -767,7 +769,7 @@ export async function trackTraceBatchActivity(2,DUE_KEY,STATE_KEY,- env.LANGFUSE_TRACE_BATCH_IDLE_MS,+ idleMs,env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,...chunk.flatMap(([traceId, state]) => [JSON.stringify([projectId, traceId]),@@ -794,6 +796,25 @@ export async function trackTraceBatchActivity(}}+async function resolveTraceIdleMs(projectId: string): Promise<number> {+ let traceIdleMs: number | null = null;+ try {+ ({ traceIdleMs } = await getTracingReadinessSettings(projectId));+ } catch (error) {+ recordIncrement("langfuse.trace_batch.idle_time_read_failures", 1);+ logger.warn("Failed to read trace idle time, using the default", {+ projectId,+ error,+ });+ }+ if (traceIdleMs === null) return env.LANGFUSE_TRACE_BATCH_IDLE_MS;+ // A due time past the pending TTL would expire the trace before dispatch.+ const cap =+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS -+ env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;+ return cap > 0 ? Math.min(traceIdleMs, cap) : traceIdleMs;+}+export class TraceBatchDispatcher extends PeriodicExclusiveRunner {private stopping = false;private activeDispatch: Promise<void> | null = null;