Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): per-project trace idle time in tracing_readiness_settings (#18595)

Lysander Kiesel3a2551dd35be
diff --git a/packages/shared/prisma/migrations/20261012060000_add_tracing_readiness_settings/migration.sql b/packages/shared/prisma/migrations/20261012060000_add_tracing_readiness_settings/migration.sql
new file mode 100644
index 0000000000..9f0201806f
--- /dev/null
+++ b/packages/shared/prisma/migrations/20261012060000_add_tracing_readiness_settings/migration.sql
@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE "tracing_readiness_settings" (
+ "project_id" TEXT NOT NULL,
+ "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "created_by" TEXT,
+ "updated_by" TEXT,
+ "trace_idle_ms" INTEGER,
+
+ CONSTRAINT "tracing_readiness_settings_pkey" PRIMARY KEY ("project_id")
+);
+
+-- AddForeignKey
+ALTER TABLE "tracing_readiness_settings" ADD CONSTRAINT "tracing_readiness_settings_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "tracing_readiness_settings" ADD CONSTRAINT "tracing_readiness_settings_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "tracing_readiness_settings" ADD CONSTRAINT "tracing_readiness_settings_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prisma
index 0b72c47cbe..21ecab1c93 100644
--- a/packages/shared/prisma/schema.prisma
+++ b/packages/shared/prisma/schema.prisma
@@ -77,6 +77,8 @@ model User {
tableViewPresetUpdated TableViewPreset[] @relation("UpdatedByUser")
monitorsCreated Monitor[] @relation("CreatedByUser")
monitorsUpdated Monitor[] @relation("UpdatedByUser")
+ tracingReadinessCreated TracingReadinessSettings[] @relation("CreatedByUser")
+ tracingReadinessUpdated TracingReadinessSettings[] @relation("UpdatedByUser")
annotationQueueAssignment AnnotationQueueAssignment[]
surveys Survey[]
commentReactions CommentReaction[]
@@ -171,6 +173,7 @@ model Project {
Evaluator Evaluator[]
EvaluationRule EvaluationRule[]
EvaluationRuleEvaluatorAssignment EvaluationRuleEvaluatorAssignment[]
+ tracingReadinessSettings TracingReadinessSettings?
LlmApiKeys LlmApiKeys[]
PosthogIntegration PosthogIntegration[]
MixpanelIntegration MixpanelIntegration[]
@@ -1244,6 +1247,7 @@ model EvaluationRule {
filter Json
sampling Decimal
delay Int
+ // Unused; kept until no deployed version selects it, then dropped.
idleTime Int? @map("idle_time") // milliseconds
timeScope String[] @default(["NEW"]) @map("time_scope")
@@ -1253,6 +1257,27 @@ model EvaluationRule {
@@map("evaluation_rules")
}
+// When tracing data is ready for downstream work (Topics, trace-level
+// evaluations). One row per project; null columns fall back to the deployment
+// default. Session settings join as their own nullable columns.
+model TracingReadinessSettings {
+ projectId String @id @map("project_id")
+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+ createdAt DateTime @default(now()) @map("created_at")
+ updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+ createdBy String? @map("created_by")
+ createdByUser User? @relation("CreatedByUser", fields: [createdBy], references: [id], onDelete: SetNull)
+ updatedBy String? @map("updated_by")
+ updatedByUser User? @relation("UpdatedByUser", fields: [updatedBy], references: [id], onDelete: SetNull)
+
+ // A trace is due this long after its latest event. Null uses
+ // LANGFUSE_TRACE_BATCH_IDLE_MS. Bounds are enforced on write, not in the DB.
+ traceIdleMs Int? @map("trace_idle_ms")
+
+ @@map("tracing_readiness_settings")
+}
+
model EvaluationRuleEvaluatorAssignment {
id String @id @default(cuid())
createdAt DateTime @default(now()) @map("created_at")
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index 9ed589de76..d7a3bffdf4 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -31,6 +31,7 @@ export * from "./services/commentFilterService";
export * from "./datasets/schemaValidation";
export * from "./datasets/schemaTypes";
export * from "./evalJobConfigCache";
+export * from "./tracingReadinessSettings";
export * from "./evals/codeEvalDispatchers";
export * from "./evals/codeEvalExecution";
export * from "./evals/evalExecutionMetadata";
diff --git a/packages/shared/src/server/topics/postgres.ts b/packages/shared/src/server/topics/postgres.ts
index 2facb4006d..63fd9d8443 100644
--- a/packages/shared/src/server/topics/postgres.ts
+++ b/packages/shared/src/server/topics/postgres.ts
@@ -342,7 +342,6 @@ export async function saveTopicRule(
) as Prisma.InputJsonValue,
sampling: 1,
delay: 0,
- idleTime: null,
targetObject: EvalTargetObject.TRACE,
timeScope: ["NEW"],
status: "ACTIVE" as const,
diff --git a/packages/shared/src/server/tracingReadinessSettings.ts b/packages/shared/src/server/tracingReadinessSettings.ts
new file mode 100644
index 0000000000..23a04dd1f5
--- /dev/null
+++ b/packages/shared/src/server/tracingReadinessSettings.ts
@@ -0,0 +1,118 @@
+import { z } from "zod";
+import { prisma } from "../db";
+import { logger } from "./logger";
+import { redis } from "./redis/redis";
+
+// The minimum stays at least 2x the 30 s dispatch interval so event batches of
+// one trace are not dispatched separately. The maximum stays below the 2 h
+// pending TTL, after which idle traces are dropped before dispatch. The DB does
+// not enforce these, so every write goes through this schema.
+export const TRACE_IDLE_MS_MIN = 60_000;
+export const TRACE_IDLE_MS_MAX = 3_600_000;
+
+// Cache fills do not replace saved values. A failed cache write can leave a
+// previous value until it expires.
+const CACHE_TTL_SECONDS = 300;
+const CACHE_KEY_PREFIX = "langfuse:tracing-readiness-settings";
+
+export const tracingReadinessSettingsSchema = z.object({
+ traceIdleMs: z
+ .number()
+ .int()
+ .min(TRACE_IDLE_MS_MIN)
+ .max(TRACE_IDLE_MS_MAX)
+ .nullable(),
+});
+
+export type TracingReadinessSettings = z.infer<
+ typeof tracingReadinessSettingsSchema
+>;
+
+const cacheKey = (projectId: string) => `${CACHE_KEY_PREFIX}:${projectId}`;
+
+/**
+ * Null fields mean the deployment default; a project without a row has all
+ * nulls. Served from Redis when cached; Postgres errors propagate.
+ */
+export async function getTracingReadinessSettings(
+ projectId: string,
+): Promise<TracingReadinessSettings> {
+ const cached = await readCache(projectId);
+ if (cached) return cached;
+
+ const row = await prisma.tracingReadinessSettings.findUnique({
+ where: { projectId },
+ select: { traceIdleMs: true },
+ });
+ const parsed = tracingReadinessSettingsSchema.safeParse({
+ traceIdleMs: row?.traceIdleMs ?? null,
+ });
+ if (!parsed.success) {
+ logger.warn("Invalid stored trace idle time, using the default", {
+ projectId,
+ traceIdleMs: row?.traceIdleMs,
+ });
+ }
+ const settings = parsed.success ? parsed.data : { traceIdleMs: null };
+ try {
+ await redis?.set(
+ cacheKey(projectId),
+ JSON.stringify(settings),
+ "EX",
+ CACHE_TTL_SECONDS,
+ "NX",
+ );
+ } catch (error) {
+ logger.warn("Failed to cache tracing readiness settings", {
+ projectId,
+ error,
+ });
+ }
+ return settings;
+}
+
+export async function saveTracingReadinessSettings(
+ projectId: string,
+ settings: TracingReadinessSettings,
+ { userId = null }: { userId?: string | null } = {},
+): Promise<TracingReadinessSettings> {
+ const data = tracingReadinessSettingsSchema.parse(settings);
+ const row = await prisma.tracingReadinessSettings.upsert({
+ where: { projectId },
+ create: { projectId, ...data, createdBy: userId, updatedBy: userId },
+ update: { ...data, updatedBy: userId },
+ select: { traceIdleMs: true },
+ });
+ const saved = { traceIdleMs: row.traceIdleMs };
+ try {
+ await redis?.set(
+ cacheKey(projectId),
+ JSON.stringify(saved),
+ "EX",
+ CACHE_TTL_SECONDS,
+ );
+ } catch (error) {
+ logger.warn("Failed to cache saved tracing readiness settings", {
+ projectId,
+ error,
+ });
+ }
+ return saved;
+}
+
+async function readCache(
+ projectId: string,
+): Promise<TracingReadinessSettings | null> {
+ try {
+ const cached = await redis?.get(cacheKey(projectId));
+ if (!cached) return null;
+ const parsed = tracingReadinessSettingsSchema.safeParse(JSON.parse(cached));
+ return parsed.success ? parsed.data : null;
+ } catch (error) {
+ logger.warn("Failed to read cached tracing readiness settings", {
+ projectId,
+ error,
+ });
+ return null;
+ }
+}
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index cb0165f835..092a962689 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -26,6 +26,7 @@ import {
type TQueueJobTypes,
} from "@langfuse/shared/src/server";
import * as shared from "@langfuse/shared/src/server";
+import { prisma } from "@langfuse/shared/src/db";
import { env } from "../env";
import { TraceBatchMetricsRunner } from "../features/traceBatching/TraceBatchMetricsRunner";
import {
@@ -51,6 +52,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
recordGauge: vi.fn(),
recordIncrement: vi.fn(),
getS3EventStorageClient: vi.fn(),
+ getTracingReadinessSettings: vi.fn(original.getTracingReadinessSettings),
};
});
@@ -870,6 +872,182 @@ describe("trace micro-batch scheduling with Redis", () => {
);
});
+ it("sets the due time from the project's trace idle time, capped below the pending TTL", async () => {
+ const orgId = `org-${randomUUID()}`;
+ const projectId = `project-${randomUUID()}`;
+ await prisma.organization.create({
+ data: { id: orgId, name: "Idle time org" },
+ });
+ await prisma.project.create({
+ data: { id: projectId, orgId, name: "Idle time project" },
+ });
+ try {
+ await shared.saveTracingReadinessSettings(projectId, {
+ traceIdleMs: 3_600_000,
+ });
+ const now = Date.now();
+ // The Lua script reads Redis time, so allow for clock skew.
+ const dueOffset = async (traceId: string) =>
+ Number(await client().zscore(dueKey, member(projectId, traceId))) - now;
+
+ await trackTraceBatchActivity(projectId, [event("custom")]);
+ await trackTraceBatchActivity("project", [event("default")]);
+ expect(await dueOffset("custom")).toBeGreaterThan(3_590_000);
+ expect(await dueOffset("custom")).toBeLessThan(3_610_000);
+ expect(
+ Number(await client().zscore(dueKey, member("project", "default"))) -
+ now,
+ ).toBeLessThan(610_000);
+
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 1_800_000;
+ await trackTraceBatchActivity(projectId, [event("capped")]);
+ expect(await dueOffset("capped")).toBeLessThan(1_780_000);
+ expect(await dueOffset("capped")).toBeGreaterThan(1_760_000);
+ } finally {
+ await prisma.organization.delete({ where: { id: orgId } });
+ }
+ });
+
+ it("caches the project's settings in Redis with an expiry and replaces them on save", async () => {
+ const orgId = `org-${randomUUID()}`;
+ const projectId = `project-${randomUUID()}`;
+ const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;
+ await prisma.organization.create({
+ data: { id: orgId, name: "Idle time cache org" },
+ });
+ await prisma.project.create({
+ data: { id: projectId, orgId, name: "Idle time cache project" },
+ });
+ try {
+ await trackTraceBatchActivity(projectId, [event("uncached")]);
+ expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({
+ traceIdleMs: null,
+ });
+ const ttl = await connection.ttl(cacheKey);
+ expect(ttl).toBeGreaterThan(0);
+ expect(ttl).toBeLessThanOrEqual(300);
+ await shared.saveTracingReadinessSettings(projectId, {
+ traceIdleMs: 300_000,
+ });
+ expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({
+ traceIdleMs: 300_000,
+ });
+ await trackTraceBatchActivity(projectId, [event("saved")]);
+ const offset =
+ Number(await client().zscore(dueKey, member(projectId, "saved"))) -
+ Date.now();
+ expect(offset).toBeGreaterThan(290_000);
+ expect(offset).toBeLessThan(310_000);
+ } finally {
+ await prisma.organization.delete({ where: { id: orgId } });
+ await connection.del(cacheKey);
+ }
+ });
+
+ it("does not let an earlier database read overwrite newly saved settings", async () => {
+ const orgId = `org-${randomUUID()}`;
+ const projectId = `project-${randomUUID()}`;
+ const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;
+ await prisma.organization.create({
+ data: { id: orgId, name: "Idle time race org" },
+ });
+ await prisma.project.create({
+ data: { id: projectId, orgId, name: "Idle time race project" },
+ });
+ try {
+ await shared.saveTracingReadinessSettings(projectId, {
+ traceIdleMs: 120_000,
+ });
+ await connection.del(cacheKey);
+
+ let releaseFill!: () => void;
+ const fillHeld = new Promise<void>((resolve) => {
+ releaseFill = resolve;
+ });
+ let signalFill!: () => void;
+ const fillStarted = new Promise<void>((resolve) => {
+ signalFill = resolve;
+ });
+ const actualShared = await vi.importActual<
+ typeof import("@langfuse/shared/src/server")
+ >("@langfuse/shared/src/server");
+ if (!actualShared.redis)
+ throw new Error("Redis is required for this test");
+ const originalSet = actualShared.redis.set.bind(actualShared.redis);
+ vi.spyOn(actualShared.redis, "set").mockImplementationOnce(
+ async (...args) => {
+ signalFill();
+ await fillHeld;
+ return originalSet(...args);
+ },
+ );
+
+ const earlierRead = shared.getTracingReadinessSettings(projectId);
+ await fillStarted;
+ try {
+ await shared.saveTracingReadinessSettings(projectId, {
+ traceIdleMs: 300_000,
+ });
+ } finally {
+ releaseFill();
+ }
+ await earlierRead;
+
+ expect(await shared.getTracingReadinessSettings(projectId)).toEqual({
+ traceIdleMs: 300_000,
+ });
+ } finally {
+ await prisma.organization.delete({ where: { id: orgId } });
+ await connection.del(cacheKey);
+ }
+ });
+
+ it("uses and caches the default for an out-of-range stored idle time", async () => {
+ const orgId = `org-${randomUUID()}`;
+ const projectId = `project-${randomUUID()}`;
+ const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;
+ await prisma.organization.create({
+ data: { id: orgId, name: "Invalid idle time org" },
+ });
+ await prisma.project.create({
+ data: { id: projectId, orgId, name: "Invalid idle time project" },
+ });
+ try {
+ await prisma.tracingReadinessSettings.create({
+ data: { projectId, traceIdleMs: 1 },
+ });
+
+ expect(await shared.getTracingReadinessSettings(projectId)).toEqual({
+ traceIdleMs: null,
+ });
+ expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({
+ traceIdleMs: null,
+ });
+ } finally {
+ await prisma.organization.delete({ where: { id: orgId } });
+ await connection.del(cacheKey);
+ }
+ });
+
+ it("uses the default when the idle time read fails", async () => {
+ const projectId = `project-${randomUUID()}`;
+ vi.mocked(shared.getTracingReadinessSettings).mockRejectedValueOnce(
+ new Error("Postgres unavailable"),
+ );
+
+ await trackTraceBatchActivity(projectId, [event("failed")]);
+
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.trace_batch.idle_time_read_failures",
+ 1,
+ );
+ const offset =
+ Number(await client().zscore(dueKey, member(projectId, "failed"))) -
+ Date.now();
+ expect(offset).toBeGreaterThan(590_000);
+ expect(offset).toBeLessThan(610_000);
+ });
+
it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
const evaluate = vi.spyOn(client(), "eval");
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index 37bff1ba7f..cb31080d15 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -22,8 +22,8 @@ Topics reuses the evaluator and evaluation-rule tables:
original request, including its selection criteria. Retries never re-evaluate a rule.
Postgres uses normal Prisma migrations; `topic_clustering_runs` is the only new
-Topics table. `EvaluationRule.idleTime` stores an optional duration in milliseconds;
-it does not change automatic scheduling or trace-batch admission.
+Topics table. Idle time is per project, not per rule: see
+`tracing_readiness_settings` in `worker/src/features/traceBatching/README.md`.
## Setup
diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.ts
index 2651be8400..bda474bea3 100644
--- a/worker/src/features/topics/storage.integration.test.ts
+++ b/worker/src/features/topics/storage.integration.test.ts
@@ -155,7 +155,6 @@ describe("Topics eval-backed configuration", () => {
).toMatchObject({
targetObject: "trace",
delay: 0,
- idleTime: null,
timeScope: ["NEW"],
status: "ACTIVE",
});
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 599de25406..cb56fef35f 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -29,7 +29,7 @@ templates. The local dev template sets the idle time below.
| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
-| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. |
+| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 s to 60 min) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, so the next activity reads the new value. A failed read uses the default and is not cached. |
| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Pending retention and inactivity expiry (2 hours). |
| `LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS` | `30000` | Dispatcher cadence (30 seconds). |
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
index fc9b71a3de..afce38c302 100644
--- a/worker/src/features/traceBatching/traceBatching.ts
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -1,6 +1,7 @@
import { createHash, randomUUID } from "node:crypto";
import { xxh32 } from "@node-rs/xxhash";
import {
+ getTracingReadinessSettings,
logger,
QueueJobs,
redis,
@@ -758,6 +759,7 @@ export async function trackTraceBatchActivity(
);
if (entries.length === 0) return;
if (!redis) throw new Error("Trace batching requires Redis");
+ const idleMs = await resolveTraceIdleMs(projectId);
// Bounded scripts keep ingestion from monopolizing the global Redis slot.
for (let offset = 0; offset < entries.length; offset += CHUNK_SIZE) {
const chunk = entries.slice(offset, offset + CHUNK_SIZE);
@@ -767,7 +769,7 @@ export async function trackTraceBatchActivity(
2,
DUE_KEY,
STATE_KEY,
- env.LANGFUSE_TRACE_BATCH_IDLE_MS,
+ idleMs,
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS,
...chunk.flatMap(([traceId, state]) => [
JSON.stringify([projectId, traceId]),
@@ -794,6 +796,25 @@ export async function trackTraceBatchActivity(
}
}
+async function resolveTraceIdleMs(projectId: string): Promise<number> {
+ let traceIdleMs: number | null = null;
+ try {
+ ({ traceIdleMs } = await getTracingReadinessSettings(projectId));
+ } catch (error) {
+ recordIncrement("langfuse.trace_batch.idle_time_read_failures", 1);
+ logger.warn("Failed to read trace idle time, using the default", {
+ projectId,
+ error,
+ });
+ }
+ if (traceIdleMs === null) return env.LANGFUSE_TRACE_BATCH_IDLE_MS;
+ // A due time past the pending TTL would expire the trace before dispatch.
+ const cap =
+ env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS -
+ env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;
+ return cap > 0 ? Math.min(traceIdleMs, cap) : traceIdleMs;
+}
+
export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
private stopping = false;
private activeDispatch: Promise<void> | null = null;