🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): stop applying data retention without an enterprise license (#18283)
Der Gerät971706265f36
diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex ac3d5b0466..eadaf6a89c 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -67,6 +67,10 @@"import": "./dist/src/server/ee/ingestionMasking/index.js","require": "./dist/src/server/ee/ingestionMasking/index.js"},+ "./src/server/ee/licenseCheck": {+ "import": "./dist/src/server/ee/licenseCheck/index.js",+ "require": "./dist/src/server/ee/licenseCheck/index.js"+ },"./query": {"import": "./dist/src/features/query/index.js","require": "./dist/src/features/query/index.js"diff --git a/worker/src/__tests__/batchDataRetentionCleaner.test.ts b/worker/src/__tests__/batchDataRetentionCleaner.test.tsindex 4f45cfa98c..feeb9f98b2 100644--- a/worker/src/__tests__/batchDataRetentionCleaner.test.ts+++ b/worker/src/__tests__/batchDataRetentionCleaner.test.ts@@ -19,9 +19,14 @@ import {toClickhouseDateTime,traceException,} from "@langfuse/shared/src/server";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { prisma } from "@langfuse/shared/src/db";import { env } from "../env";+vi.mock("@langfuse/shared/src/server/ee/licenseCheck", () => ({+ isEnterpriseLicenseAvailable: vi.fn(() => true),+}));+const integrationHooks = vi.hoisted(() => ({failExactEnrichmentForProjectId: null as string | null,failExactEnrichmentOnReadOnly: true,@@ -207,6 +212,13 @@ function getLastGaugeValue(stat: string): number {}describe("BatchDataRetentionCleaner", () => {+ beforeEach(() => {+ // Retention only runs under an enterprise license. Pin that here rather+ // than inheriting it from the ambient env, so the suite does not depend on+ // which .env example the run happens to have loaded.+ vi.mocked(isEnterpriseLicenseAvailable).mockReturnValue(true);+ });+describe("processBatch - traces", () => {const TABLE = "traces" as const;@@ -241,6 +253,36 @@ describe("BatchDataRetentionCleaner", () => {expect(await getClickhouseCount(TABLE, projectId)).toBe(0);});+ it("should NOT delete traces when no enterprise license is available", async () => {+ const now = Date.now();+ const tenDaysAgo = now - 10 * 24 * 60 * 60 * 1000;++ const { projectId } = await createOrgProjectAndApiKey();+ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays: 7 },+ });++ await createTracesCh([+ createTrace({+ id: randomUUID(),+ project_id: projectId,+ timestamp: tenDaysAgo,+ }),+ ]);++ expect(await getClickhouseCount(TABLE, projectId)).toBe(1);++ // Simulate a self-hosted deployment whose license key was removed.+ vi.mocked(isEnterpriseLicenseAvailable).mockReturnValue(false);+ const cleaner = new BatchDataRetentionCleaner(TABLE);+ await cleaner.processBatch();++ // Retention is an enterprise entitlement, so the stored policy must stop+ // being applied once the instance can no longer configure it.+ expect(await getClickhouseCount(TABLE, projectId)).toBe(1);+ });+it("should NOT delete traces within retention period", async () => {const now = Date.now();const fiveDaysAgo = now - 5 * 24 * 60 * 60 * 1000;diff --git a/worker/src/__tests__/dataRetentionProcessing.test.ts b/worker/src/__tests__/dataRetentionProcessing.test.tsindex 76b2a97715..a0961b26b3 100644--- a/worker/src/__tests__/dataRetentionProcessing.test.ts+++ b/worker/src/__tests__/dataRetentionProcessing.test.ts@@ -1,4 +1,12 @@-import { expect, it, describe, beforeAll, beforeEach, afterEach } from "vitest";+import {+ expect,+ it,+ describe,+ beforeAll,+ beforeEach,+ afterEach,+ vi,+} from "vitest";import { env } from "../env";import { randomUUID } from "crypto";import {@@ -18,11 +26,36 @@ import {toClickhouseDateTime,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";+import { env as sharedEnv } from "@langfuse/shared/src/env";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { handleDataRetentionProcessingJob } from "../ee/dataRetention/handleDataRetentionProcessingJob";import { Job } from "bullmq";import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";import { InAppAgentRunStatus } from "@langfuse/shared/in-app-agent";+type LicenseCheckModule =+ typeof import("@langfuse/shared/src/server/ee/licenseCheck");++// Overriding the shared env object is not observed by the license check at+// runtime here, so cases cannot simulate a deployment that way. Wrap the real+// implementation instead: it keeps the actual key-prefix rules under test+// while letting a case choose the environment they are evaluated against.+vi.mock(+ "@langfuse/shared/src/server/ee/licenseCheck",+ async (importOriginal) => {+ const actual = await importOriginal<LicenseCheckModule>();+ return {+ ...actual,+ isEnterpriseLicenseAvailable: vi.fn(actual.isEnterpriseLicenseAvailable),+ };+ },+);++const { isEnterpriseLicenseAvailable: actualIsEnterpriseLicenseAvailable } =+ await vi.importActual<LicenseCheckModule>(+ "@langfuse/shared/src/server/ee/licenseCheck",+ );+describe("DataRetentionProcessingJob", () => {let storageService: StorageService;let s3Prefix: string | null = null;@@ -41,6 +74,10 @@ describe("DataRetentionProcessingJob", () => {beforeEach(() => {s3Prefix = `${randomUUID()}/`;+ // Retention only runs under an enterprise license. Pin that here rather+ // than inheriting it from the ambient env, so the suite does not depend on+ // which .env example the run happens to have loaded.+ vi.mocked(isEnterpriseLicenseAvailable).mockReturnValue(true);});afterEach(async () => {@@ -784,4 +821,98 @@ describe("DataRetentionProcessingJob", () => {data: { retentionDays: null },});});++ describe("enterprise license enforcement", () => {+ /**+ * Point the license check at a simulated self-hosted deployment holding+ * the given license key. The real implementation is kept, so these cases+ * still exercise the actual key-prefix rules; only the environment it+ * reads is substituted. The cloud region has to be cleared too, because+ * any cloud region counts as licensed.+ */+ const withSelfHostedLicense = async (+ licenseKey: string | undefined,+ fn: () => Promise<void>,+ ) => {+ vi.mocked(isEnterpriseLicenseAvailable).mockImplementation(() =>+ actualIsEnterpriseLicenseAvailable({+ ...sharedEnv,+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,+ LANGFUSE_EE_LICENSE_KEY: licenseKey,+ }),+ );+ try {+ await fn();+ } finally {+ vi.mocked(isEnterpriseLicenseAvailable).mockReturnValue(true);+ }+ };++ const seedExpiredTrace = async (traceId: string) => {+ await createTracesCh([+ createTrace({+ id: traceId,+ project_id: projectId,+ timestamp: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30).getTime(), // 30 days in the past+ }),+ ]);+ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays: 7 },+ });+ };++ afterEach(async () => {+ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays: null },+ });+ });++ it("should not delete expired data when the self-hosted license key is removed", async () => {+ const traceId = `${randomUUID()}-trace-unlicensed`;+ await seedExpiredTrace(traceId);++ await withSelfHostedLicense(undefined, async () => {+ await handleDataRetentionProcessingJob({+ data: { payload: { projectId, retention: 7 } },+ } as Job);+ });++ // Retention is an enterprise entitlement, so a stored policy must stop+ // being applied once the instance can no longer configure it.+ const trace = await getTraceById({ traceId, projectId });+ expect(trace).toBeDefined();+ });++ it("should not delete expired data on a self-hosted pro license", async () => {+ const traceId = `${randomUUID()}-trace-pro`;+ await seedExpiredTrace(traceId);++ await withSelfHostedLicense("langfuse_pro_some-key", async () => {+ await handleDataRetentionProcessingJob({+ data: { payload: { projectId, retention: 7 } },+ } as Job);+ });++ // Pro does not carry the data-retention entitlement.+ const trace = await getTraceById({ traceId, projectId });+ expect(trace).toBeDefined();+ });++ it("should delete expired data on a self-hosted enterprise license", async () => {+ const traceId = `${randomUUID()}-trace-ee`;+ await seedExpiredTrace(traceId);++ await withSelfHostedLicense("langfuse_ee_some-key", async () => {+ await handleDataRetentionProcessingJob({+ data: { payload: { projectId, retention: 7 } },+ } as Job);+ });++ // Pins that the guard is the license and not an unrelated skip.+ const trace = await getTraceById({ traceId, projectId });+ expect(trace).toBeUndefined();+ });+ });});diff --git a/worker/src/__tests__/mediaRetentionCleaner.test.ts b/worker/src/__tests__/mediaRetentionCleaner.test.tsindex 78499df1d6..d09409fba9 100644--- a/worker/src/__tests__/mediaRetentionCleaner.test.ts+++ b/worker/src/__tests__/mediaRetentionCleaner.test.ts@@ -8,9 +8,14 @@ import {removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { MediaRetentionCleaner } from "../features/media-retention-cleaner";import { env } from "../env";+vi.mock("@langfuse/shared/src/server/ee/licenseCheck", () => ({+ isEnterpriseLicenseAvailable: vi.fn(),+}));+// Mock S3 and blob storage functionsvi.mock("@langfuse/shared/src/server", async () => {const actual = await vi.importActual("@langfuse/shared/src/server");@@ -106,6 +111,11 @@ describe("MediaRetentionCleaner", () => {beforeEach(() => {vi.clearAllMocks();vi.mocked(getS3MediaStorageClient).mockReturnValue(mockS3Client as never);+ // Retention only runs under an enterprise license. Pin that here rather+ // than inheriting it from the ambient env, so the suite does not depend on+ // which .env example the run happens to have loaded. Must be set after+ // clearAllMocks, which would otherwise leave it returning undefined.+ vi.mocked(isEnterpriseLicenseAvailable).mockReturnValue(true);});afterEach(() => {@@ -222,6 +232,34 @@ describe("MediaRetentionCleaner", () => {expect(allDeletedPaths).toContain(media.bucketPath);});+ it("should NOT delete media when no enterprise license is available", async () => {+ const now = Date.now();+ const tenDaysAgo = new Date(now - 10 * 24 * 60 * 60 * 1000);++ const { projectId } = await createOrgProjectAndApiKey();+ await prisma.project.update({+ where: { id: projectId },+ data: { retentionDays: 7 },+ });++ await createTestMedia(projectId, tenDaysAgo);+ expect(await getMediaCount(projectId)).toBe(1);++ // Simulate a self-hosted deployment whose license key was removed.+ vi.mocked(isEnterpriseLicenseAvailable).mockReturnValue(false);+ const cleaner = new MediaRetentionCleaner();++ // Backs off a full interval instead of doing work.+ await expect(cleaner.processBatch()).resolves.toBe(+ env.LANGFUSE_MEDIA_RETENTION_CLEANER_INTERVAL_MS,+ );++ // Retention is an enterprise entitlement, so the stored policy must stop+ // being applied once the instance can no longer configure it.+ expect(await getMediaCount(projectId)).toBe(1);+ expect(mockDeleteFiles).not.toHaveBeenCalled();+ });+it("cleans expired links, preserves recent links, and drains the project", async () => {await drainExpiredMedia();mockDeleteFiles.mockClear();diff --git a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.tsindex aa6784f284..8a97b190eb 100644--- a/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts+++ b/worker/src/ee/dataRetention/handleDataRetentionProcessingJob.ts@@ -16,6 +16,7 @@ import { prisma } from "@langfuse/shared/src/db";import { Prisma } from "@prisma/client";import { isMissingInAppAgentMcpApiKeyError } from "@langfuse/shared/in-app-agent/server/runLifecycle";import { deleteInAppAgentMcpApiKeyFromDb } from "@langfuse/shared/src/server/auth/apiKeys";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { env, v4WritesToEventsTable } from "../../env";export const handleDataRetentionProcessingJob = async (job: Job) => {@@ -27,6 +28,14 @@ export const handleDataRetentionProcessingJob = async (job: Job) => {span.setAttribute("messaging.bullmq.job.input.projectId", projectId);}+ // Data retention is an enterprise feature, and the license is read from the+ // environment at startup. Jobs enqueued while licensed survive in Redis+ // across the restart that removes the key, so the license has to be checked+ // here as well as at scheduling time.+ if (!isEnterpriseLicenseAvailable()) {+ return;+ }+// CRITICAL FIX: Re-fetch current retention setting from database// This prevents stale queued jobs from deleting data after retention is disabledconst project = await prisma.project.findUnique({diff --git a/worker/src/ee/dataRetention/handleDataRetentionSchedule.ts b/worker/src/ee/dataRetention/handleDataRetentionSchedule.tsindex f13c76394d..7f54eba726 100644--- a/worker/src/ee/dataRetention/handleDataRetentionSchedule.ts+++ b/worker/src/ee/dataRetention/handleDataRetentionSchedule.ts@@ -3,9 +3,18 @@ import {DataRetentionProcessingQueue,QueueJobs,} from "@langfuse/shared/src/server";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { randomUUID } from "crypto";export const handleDataRetentionSchedule = async () => {+ // Data retention is an enterprise feature. Without a license the setting+ // cannot be viewed or changed, so stored policies must not be enforced+ // either: a deployment that loses its license would otherwise keep deleting+ // data under a policy its operators can no longer reach.+ if (!isEnterpriseLicenseAvailable()) {+ return;+ }+const projectsWithRetention = await prisma.project.findMany({select: {id: true,diff --git a/worker/src/features/batch-data-retention-cleaner/index.ts b/worker/src/features/batch-data-retention-cleaner/index.tsindex 89c15222db..0b86a96254 100644--- a/worker/src/features/batch-data-retention-cleaner/index.ts+++ b/worker/src/features/batch-data-retention-cleaner/index.ts@@ -13,6 +13,7 @@ import {recordIncrement,traceException,} from "@langfuse/shared/src/server";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { env } from "../../env";import { getRetentionCutoffDate } from "../utils";import {@@ -207,6 +208,14 @@ export class BatchDataRetentionCleaner extends PeriodicExclusiveRunner {* Preflight and deletion are both under lock to avoid redundant expensive queries.*/protected async execute(): Promise<void> {+ // Data retention is an enterprise feature. Without a license the setting+ // cannot be viewed or changed, so stored policies must not be enforced+ // either: a deployment that loses its license would otherwise keep+ // deleting data under a policy its operators can no longer reach.+ if (!isEnterpriseLicenseAvailable()) {+ return;+ }+const timestampColumn = TIMESTAMP_COLUMN_MAP[this.tableName];let observedBacklog: ReturnType<typeof summarizeBacklog> | undefined;let observedLagMeasurementComplete = false;diff --git a/worker/src/features/media-retention-cleaner/index.ts b/worker/src/features/media-retention-cleaner/index.tsindex 1cb60f732b..aece864f4f 100644--- a/worker/src/features/media-retention-cleaner/index.ts+++ b/worker/src/features/media-retention-cleaner/index.ts@@ -9,6 +9,7 @@ import {recordIncrement,removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject,} from "@langfuse/shared/src/server";+import { isEnterpriseLicenseAvailable } from "@langfuse/shared/src/server/ee/licenseCheck";import { env } from "../../env";import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";@@ -58,6 +59,14 @@ export class MediaRetentionCleaner extends PeriodicExclusiveRunner {* Preflight and deletion are both under lock to avoid redundant expensive queries.*/protected async execute(): Promise<number> {+ // Data retention is an enterprise feature. Without a license the setting+ // cannot be viewed or changed, so stored policies must not be enforced+ // either: a deployment that loses its license would otherwise keep+ // deleting data under a policy its operators can no longer reach.+ if (!isEnterpriseLicenseAvailable()) {+ return this.defaultIntervalMs;+ }+// Reset gauge before attempting lock - ensures it doesn't appear stuck// if another worker holds the lockrecordGauge(`${METRIC_PREFIX}.seconds_past_cutoff`, 0);