🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): maintain v4 legacy API usage in Redis via incremental worker job (#16164)
Max Deichmannae901b330f3e
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex ebca27edee..b3308bb988 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -104,6 +104,7 @@ export * from "./redis/batchExport";export * from "./redis/batchActionQueue";export * from "./redis/ingestionQueue";export * from "./redis/postHogIntegrationQueue";+export * from "./redis/v4LegacyApiUsageQueue";export * from "./redis/postHogIntegrationProcessingQueue";export * from "./redis/mixpanelIntegrationQueue";export * from "./redis/mixpanelIntegrationProcessingQueue";@@ -175,6 +176,7 @@ export * from "./utils/formatAuthProvider";export * from "./traceDeletionProcessor";export * from "./deletionGuard";export * from "./forceV3Experience";+export * from "./v4/legacyApiUsage";export * from "./analytics-integrations/types";// Re-annotate these deprecated routing wrappers at the public server barrel.diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.tsindex f30d34377e..9ace762f48 100644--- a/packages/shared/src/server/queues.ts+++ b/packages/shared/src/server/queues.ts@@ -415,6 +415,7 @@ export enum QueueName {NotificationQueue = "notification-queue",MonitorQueue = "monitor-queue",InAppAgentRunQueue = "in-app-agent-run-queue",+ V4LegacyApiUsageQueue = "v4-legacy-api-usage-queue",}export enum QueueJobs {@@ -453,6 +454,7 @@ export enum QueueJobs {NotificationJob = "notification-job",MonitorJob = "monitor-job",InAppAgentRunJob = "in-app-agent-run-job",+ V4LegacyApiUsageJob = "v4-legacy-api-usage-job",}export type TQueueJobTypes = {diff --git a/packages/shared/src/server/redis/getQueue.ts b/packages/shared/src/server/redis/getQueue.tsindex c964be2eb7..4e65c0bfa0 100644--- a/packages/shared/src/server/redis/getQueue.ts+++ b/packages/shared/src/server/redis/getQueue.ts@@ -29,6 +29,7 @@ import { EventPropagationQueue } from "./eventPropagationQueue";import { NotificationQueue } from "./notificationQueue";import { MonitorQueue } from "./monitorQueue";import { InAppAgentRunQueue } from "./inAppAgentRunQueue";+import { V4LegacyApiUsageQueue } from "./v4LegacyApiUsageQueue";// Sharded queues require a sharding key.// Use the queue class directly, for example IngestionQueue.getInstance({ shardingKey }).@@ -105,6 +106,8 @@ export function getQueue(return MonitorQueue.getInstance();case QueueName.InAppAgentRunQueue:return InAppAgentRunQueue.getInstance();+ case QueueName.V4LegacyApiUsageQueue:+ return V4LegacyApiUsageQueue.getInstance();default: {const _exhaustiveCheckDefault: never = queueName;throw new Error(`Queue ${queueName} not found`);diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.tsnew file mode 100644index 0000000000..de43592d6d--- /dev/null+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts@@ -0,0 +1,58 @@+import { Queue } from "bullmq";+import { QueueName, QueueJobs } from "../queues";+import { createBullMQQueueOptionsWithRedis } from "./redis";+import { logger } from "../logger";++// Hourly, offset from other scheduled jobs (:5 metering, :30 posthog, :35+// free-tier). The worker re-scans a trailing margin each run, so the exact+// minute only affects freshness, not correctness.+export const V4_LEGACY_API_USAGE_CRON_PATTERN = "25 * * * *";++export class V4LegacyApiUsageQueue {+ private static instance: Queue | null = null;++ public static getInstance(): Queue | null {+ if (V4LegacyApiUsageQueue.instance) {+ return V4LegacyApiUsageQueue.instance;+ }++ const queueOptionsWithRedis = createBullMQQueueOptionsWithRedis(+ QueueName.V4LegacyApiUsageQueue,+ );+ V4LegacyApiUsageQueue.instance = queueOptionsWithRedis+ ? new Queue(QueueName.V4LegacyApiUsageQueue, {+ ...queueOptionsWithRedis,+ defaultJobOptions: {+ removeOnComplete: true,+ removeOnFail: 100,+ attempts: 5,+ backoff: {+ type: "exponential",+ delay: 30_000,+ },+ },+ })+ : null;++ V4LegacyApiUsageQueue.instance?.on("error", (err) => {+ logger.error("V4LegacyApiUsageQueue error", err);+ });++ if (V4LegacyApiUsageQueue.instance) {+ logger.debug("Scheduling jobs for V4LegacyApiUsageQueue");+ V4LegacyApiUsageQueue.instance+ .add(+ QueueJobs.V4LegacyApiUsageJob,+ {},+ {+ repeat: { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },+ },+ )+ .catch((err) => {+ logger.error("Error adding V4LegacyApiUsageJob schedule", err);+ });+ }++ return V4LegacyApiUsageQueue.instance;+ }+}diff --git a/packages/shared/src/server/v4/legacyApiUsage.test.ts b/packages/shared/src/server/v4/legacyApiUsage.test.tsnew file mode 100644index 0000000000..7a5d40a879--- /dev/null+++ b/packages/shared/src/server/v4/legacyApiUsage.test.ts@@ -0,0 +1,33 @@+import { describe, expect, it } from "vitest";+import {+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ v4LegacyApiHourBucketTtlSeconds,+} from "./legacyApiUsage";++describe("v4LegacyApiHourBucketTtlSeconds", () => {+ const HOUR_MS = 60 * 60 * 1000;+ const nowMs = Date.parse("2026-06-25T00:30:00.000Z");++ it("gives nearly the full GC horizon for the current hour", () => {+ const hourStartMs = Date.parse("2026-06-25T00:00:00.000Z");+ const ttl = v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs);+ expect(ttl).toBe(+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS -+ Math.floor((nowMs - hourStartMs) / 1000),+ );+ });++ it("shortens TTL for hole-repaired older hours so they expire with the window", () => {+ const tenDaysAgo = nowMs - 10 * 24 * HOUR_MS;+ const hourStartMs = Math.floor(tenDaysAgo / HOUR_MS) * HOUR_MS;+ const ttl = v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs);+ // ~5 days remaining of the 15d horizon+ expect(ttl).toBeGreaterThan(4 * 24 * 60 * 60);+ expect(ttl).toBeLessThan(6 * 24 * 60 * 60);+ });++ it("clamps to at least 1 second when the hour is already past the horizon", () => {+ const hourStartMs = nowMs - 20 * 24 * HOUR_MS;+ expect(v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs)).toBe(1);+ });+});diff --git a/packages/shared/src/server/v4/legacyApiUsage.ts b/packages/shared/src/server/v4/legacyApiUsage.tsnew file mode 100644index 0000000000..767e6ffc99--- /dev/null+++ b/packages/shared/src/server/v4/legacyApiUsage.ts@@ -0,0 +1,286 @@+import { z } from "zod/v4";++/**+ * Contracts for the v4 legacy API usage pipeline.+ *+ * Deprecated public API usage (and experiment `POST /api/public/dataset-run-items`+ * usage) is derived from `system.query_log`, which is expensive to scan:+ * filters live in unindexed JSON `log_comment` fields and the log is local to+ * every ClickHouse service, so each check fans out across services and+ * replicas. Instead of running that scan from web requests, a worker job+ * maintains the data incrementally in Redis:+ *+ * - Hour buckets (`hourBucketKey`): environment-wide usage rows per hour,+ * written by full overwrite so re-scans and backfills are idempotent. TTL is+ * remaining life until `hourStart + GC horizon` (not a flat clock from write+ * time), so hole-repaired older hours expire with the sliding window. The+ * worker never deletes buckets; Redis expiry is the GC.+ * - Cursor (`V4_LEGACY_API_USAGE_CURSOR_KEY`): the last hour the worker+ * materialized. Missed runs backfill from here, so gaps self-heal.+ * - Per-project usage entries (`legacyApiUsageProjectKey`,+ * `experimentPostUsageProjectKey`, 12h TTL): the aggregated trailing-window+ * answer web requests read. Refreshed every worker run, so they only expire+ * when the pipeline is down. Projects without usage get no entry.+ * - Heartbeat (`V4_LEGACY_API_USAGE_HEARTBEAT_KEY`): proof the pipeline ran+ * recently. While fresh, a missing per-project entry authoritatively means+ * "no usage" and web must not fall back to querying ClickHouse.+ *+ * Worker Datadog freshness gauge+ * `langfuse.v4.legacy_api_usage.heartbeat_age_seconds` exposes staleness for+ * alerting (alert when it exceeds `V4_LEGACY_API_HEARTBEAT_FRESHNESS_MS`, 3h).+ *+ * This module is pure (keys, schemas, constants, aggregation); Redis and+ * ClickHouse I/O live with the producer (worker) and consumer (web).+ */++const HOUR_MS = 60 * 60 * 1000;++/** Detection window served to the UI; must match the web-side window. */+export const V4_LEGACY_API_USAGE_WINDOW_MS = 14 * 24 * HOUR_MS;++/** GC horizon for hour buckets: window plus one day of slack. */+export const V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS = 15 * 24 * 60 * 60;++/**+ * Redis TTL for an hour bucket keyed by that hour's start, not by write time.+ * A hole-repaired bucket from 10 days ago must expire in ~5 days (horizon+ * minus age), not live another full 15 days from the repair write.+ */+export const v4LegacyApiHourBucketTtlSeconds = (+ hourStartMs: number,+ nowMs: number,+): number => {+ const expireAtMs = hourStartMs + V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS * 1000;+ return Math.max(1, Math.floor((expireAtMs - nowMs) / 1000));+};++/**+ * Per-project entries outlive several missed worker runs; the worker+ * refreshes them hourly, so they only expire when the pipeline is down and+ * web falls back to its request-path cache behavior.+ */+export const V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS = 12 * 60 * 60;++/**+ * Heartbeats older than this stop being authoritative: absence of a+ * per-project entry then no longer means "no usage".+ */+export const V4_LEGACY_API_HEARTBEAT_FRESHNESS_MS = 3 * HOUR_MS;++/** Trailing hours re-scanned every run to absorb late query_log flushes. */+export const V4_LEGACY_API_RESCAN_HOURS = 3;+/** Deeper re-scan for stragglers beyond the hourly margin. */+export const V4_LEGACY_API_DEEP_RESCAN_HOURS = 24;+/**+ * Minimum spacing between deep re-scans. Tracked via a persisted timestamp+ * (not a wall-clock hour) so a missed run only delays the deep sweep instead+ * of skipping it for a day.+ */+export const V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS = 24 * HOUR_MS;++export const V4_LEGACY_API_USAGE_CURSOR_KEY =+ "langfuse:v4:legacy-api-usage:cursor:v1";+export const V4_LEGACY_API_USAGE_HEARTBEAT_KEY =+ "langfuse:v4:legacy-api-usage:heartbeat:v1";+export const V4_LEGACY_API_USAGE_LOCK_KEY =+ "langfuse:v4:legacy-api-usage:lock:v1";+export const V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY =+ "langfuse:v4:legacy-api-usage:deep-rescan-at:v1";+/**+ * Project IDs the worker wrote per-project entries for in its last run; the+ * next run deletes entries for projects that dropped out of the window so+ * stale usage does not linger for the entry TTL.+ */+export const V4_LEGACY_API_ROLLUP_PROJECTS_KEY =+ "langfuse:v4:legacy-api-usage:rollup-projects:v1";++/** Hour-aligned ISO without milliseconds, e.g. "2026-06-25T13:00:00Z". */+export const v4LegacyApiHourStartIso = (hourStartMs: number): string =>+ new Date(hourStartMs).toISOString().replace(".000Z", "Z");++// Hour buckets stay v1 across window changes: they hold per-hour raw facts+// that are independent of the window length. Widening the window simply+// leaves older hours missing, which the worker's hole repair backfills on+// its next run.+export const v4LegacyApiHourBucketKey = (hourStartMs: number): string =>+ `langfuse:v4:legacy-api-usage:hour:v1:${v4LegacyApiHourStartIso(hourStartMs)}`;++// v1: per-project entries. Age trimming and worker rollup deletion handle+// semantic invalidation when the detection window or aggregation changes.+export const v4LegacyApiUsageProjectKey = (projectId: string): string =>+ `langfuse:v4:legacy-api-usage:v1:${projectId}`;++export const v4ExperimentPostUsageProjectKey = (projectId: string): string =>+ `langfuse:v4:experiment-post-usage:v1:${projectId}`;++const legacyApiUsageRowSchema = z.object({+ entrypoint: z.string(),+ count: z.number(),+ lastSeen: z.string(),+});++export type V4LegacyApiUsageRow = z.infer<typeof legacyApiUsageRowSchema>;++export const v4LegacyApiHourBucketSchema = z.object({+ version: z.literal(1),+ computedAt: z.string(),+ apiRows: z.array(+ legacyApiUsageRowSchema.extend({+ projectId: z.string(),+ }),+ ),+ experimentPostRows: z.array(+ z.object({+ projectId: z.string(),+ count: z.number(),+ // Optional so pre-lastSeen hour buckets keep parsing; new writes always set it.+ lastSeen: z.string().optional(),+ }),+ ),+});++export type V4LegacyApiHourBucket = z.infer<typeof v4LegacyApiHourBucketSchema>;++/** Per-project entry consumed by web (`legacyApiUsageSummary`). */+export const v4LegacyApiUsageBlobSchema = z.object({+ version: z.literal(1),+ computedAt: z.string(),+ rows: z.array(legacyApiUsageRowSchema),+});++export type V4LegacyApiUsageBlob = z.infer<typeof v4LegacyApiUsageBlobSchema>;++/** Per-project entry consumed by web (experiment instrumentation check). */+export const v4ExperimentPostUsageBlobSchema = z.object({+ version: z.literal(1),+ computedAt: z.string(),+ used: z.boolean(),+ /**+ * Latest POST /api/public/dataset-run-items in the window; null when unused.+ * Default keeps older worker blobs readable until the next refresh.+ */+ lastSeen: z.string().nullable().default(null),+});++export type V4ExperimentPostUsageBlob = z.infer<+ typeof v4ExperimentPostUsageBlobSchema+>;++export const v4LegacyApiRollupProjectsSchema = z.object({+ version: z.literal(1),+ api: z.array(z.string()),+ experimentPost: z.array(z.string()),+});++export type V4LegacyApiRollupProjects = z.infer<+ typeof v4LegacyApiRollupProjectsSchema+>;++export const isV4LegacyApiHeartbeatFresh = (+ heartbeatIso: string | null | undefined,+ nowMs: number,+): boolean => {+ if (!heartbeatIso) return false;+ const heartbeatMs = Date.parse(heartbeatIso);+ return (+ Number.isFinite(heartbeatMs) &&+ nowMs - heartbeatMs <= V4_LEGACY_API_HEARTBEAT_FRESHNESS_MS+ );+};++/** Hour starts in `[fromMs, toMs]`, inclusive, aligned to full hours. */+export const listV4LegacyApiHourStarts = (+ fromMs: number,+ toMs: number,+): number[] => {+ const hourStarts: number[] = [];+ for (+ let hourStartMs = Math.floor(fromMs / HOUR_MS) * HOUR_MS;+ hourStartMs <= toMs;+ hourStartMs += HOUR_MS+ ) {+ hourStarts.push(hourStartMs);+ }+ return hourStarts;+};++export type V4LegacyApiUsageRollup = {+ apiRowsByProjectId: Map<string, V4LegacyApiUsageRow[]>;+ /**+ * projectId → latest experiment POST lastSeen within the aggregated buckets.+ * `null` means usage was observed in a legacy hour bucket that lacked lastSeen.+ */+ experimentPostLastSeenByProjectId: Map<string, string | null>;+};++/**+ * Aggregates hour buckets into the per-project trailing-window answer:+ * counts add per entrypoint, `lastSeen` takes the maximum. Buckets carry raw+ * facts only, so aggregation is associative and order-independent.+ */+export const aggregateV4LegacyApiHourBuckets = (+ buckets: V4LegacyApiHourBucket[],+): V4LegacyApiUsageRollup => {+ const rowsByProjectAndEntrypoint = new Map<+ string,+ Map<string, V4LegacyApiUsageRow>+ >();+ const experimentPostLastSeenByProjectId = new Map<string, string | null>();++ for (const bucket of buckets) {+ for (const row of bucket.apiRows) {+ const rowsByEntrypoint =+ rowsByProjectAndEntrypoint.get(row.projectId) ??+ new Map<string, V4LegacyApiUsageRow>();+ const existing = rowsByEntrypoint.get(row.entrypoint);+ rowsByEntrypoint.set(+ row.entrypoint,+ existing+ ? {+ entrypoint: row.entrypoint,+ count: existing.count + row.count,+ lastSeen:+ existing.lastSeen > row.lastSeen+ ? existing.lastSeen+ : row.lastSeen,+ }+ : {+ entrypoint: row.entrypoint,+ count: row.count,+ lastSeen: row.lastSeen,+ },+ );+ rowsByProjectAndEntrypoint.set(row.projectId, rowsByEntrypoint);+ }+ for (const row of bucket.experimentPostRows) {+ if (row.count <= 0) continue;+ const existingLastSeen = experimentPostLastSeenByProjectId.get(+ row.projectId,+ );+ if (!row.lastSeen) {+ if (existingLastSeen === undefined) {+ experimentPostLastSeenByProjectId.set(row.projectId, null);+ }+ continue;+ }+ experimentPostLastSeenByProjectId.set(+ row.projectId,+ existingLastSeen && existingLastSeen > row.lastSeen+ ? existingLastSeen+ : row.lastSeen,+ );+ }+ }++ const apiRowsByProjectId = new Map<string, V4LegacyApiUsageRow[]>();+ for (const [projectId, rowsByEntrypoint] of rowsByProjectAndEntrypoint) {+ apiRowsByProjectId.set(+ projectId,+ Array.from(rowsByEntrypoint.values()).sort((left, right) =>+ left.entrypoint.localeCompare(right.entrypoint),+ ),+ );+ }++ return { apiRowsByProjectId, experimentPostLastSeenByProjectId };+};diff --git a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.tsindex 83a3b80195..c5ba599a01 100644--- a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts+++ b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts@@ -9,17 +9,88 @@ vi.mock("@/src/server/auth", () => ({}));/**- * Mutable Redis stub shared with the mocked "@langfuse/shared/src/server"- * barrel. Defaults to status "end" (cache unavailable, direct-query mode);- * cache tests flip status to "ready" and stub mget/setex.+ * In-memory Redis stand-in for cache tests. Avoids a real localhost:6379+ * dependency so the redis-cluster CI matrix (no standalone Redis) stays green.+ * Most tests keep `available=false` to exercise the direct-query path.*/-const redisMock = vi.hoisted(() => ({- status: "end" as string,- disconnect: vi.fn(),- mget: vi.fn(),- setex: vi.fn(),+const redisAvailability = vi.hoisted(() => ({+ available: false,}));+const memoryRedis = vi.hoisted(() => {+ type Entry = { value: string; expiresAtMs: number | null };+ const store = new Map<string, Entry>();+ const nowMs = () => Date.now();+ const isLive = (entry: Entry | undefined) =>+ entry != null && (entry.expiresAtMs == null || entry.expiresAtMs > nowMs());++ const client = {+ get status() {+ return redisAvailability.available ? "ready" : "end";+ },+ disconnect: vi.fn(),+ ping: async () => "PONG",+ get: async (key: string) => {+ const entry = store.get(key);+ if (!isLive(entry)) {+ store.delete(key);+ return null;+ }+ return entry!.value;+ },+ mget: async (...keysOrArray: string[] | [string[]]) => {+ const keys =+ keysOrArray.length === 1 && Array.isArray(keysOrArray[0])+ ? keysOrArray[0]+ : (keysOrArray as string[]);+ return Promise.all(keys.map((key) => client.get(key)));+ },+ setex: async (key: string, ttlSeconds: number, value: string) => {+ store.set(key, {+ value,+ expiresAtMs: nowMs() + ttlSeconds * 1000,+ });+ return "OK";+ },+ del: async (...keys: string[]) => {+ let removed = 0;+ for (const key of keys) {+ if (store.delete(key)) removed += 1;+ }+ return removed;+ },+ ttl: async (key: string) => {+ const entry = store.get(key);+ if (!isLive(entry)) {+ store.delete(key);+ return -2;+ }+ if (entry!.expiresAtMs == null) return -1;+ return Math.max(0, Math.ceil((entry!.expiresAtMs - nowMs()) / 1000));+ },+ scan: async (+ cursor: string,+ _matchToken: string,+ pattern: string,+ _countToken: string,+ _count: number,+ ) => {+ const prefix = pattern.endsWith("*") ? pattern.slice(0, -1) : pattern;+ const keys = [...store.keys()].filter((key) =>+ pattern.endsWith("*") ? key.startsWith(prefix) : key === pattern,+ );+ // Single-page scan: first call returns all keys, next cursor is "0".+ if (cursor !== "0") return ["0", [] as string[]];+ return ["0", keys];+ },+ __store: store,+ __reset() {+ store.clear();+ },+ };+ return client;+});+const sharedServerMock = vi.hoisted(() => ({queryClickhouse: vi.fn(),isForceV3ExperienceProject: vi.fn(() => false),@@ -119,10 +190,12 @@ const sharedEnvMock = vi.hoisted(() => ({vi.mock("@langfuse/shared/src/env", () => ({ env: sharedEnvMock }));-vi.mock("@langfuse/shared/src/server", async () => {+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const actual = await importOriginal();const { ROOT_CONTEXT } = await import("@opentelemetry/api");return {+ ...(actual as object),...sharedServerMock,getTraceById: vi.fn(),logger: {@@ -131,7 +204,7 @@ vi.mock("@langfuse/shared/src/server", async () => {warn: vi.fn(),error: vi.fn(),},- redis: redisMock,+ redis: memoryRedis,ClickHouseClientManager: {getInstance: () => ({closeAllConnections: vi.fn(),@@ -390,7 +463,6 @@ const accessibleProjectsFindManyArgs = {const TEST_NOW = new Date("2026-06-25T00:30:00Z");const HOT_START_ISO = "2026-06-25T00:00:00.000Z";const HOT_START_CLICKHOUSE = "2026-06-25 00:00:00.000";-const WINDOW_END_CLICKHOUSE = "2026-06-25 01:00:00.000";const WINDOW_START_CLICKHOUSE = "2026-06-11 01:00:00.000";// Recent SDK gap cutoff: TEST_NOW floored to the minute (zero// seconds/millis) so repeated calls within the minute share one ClickHouse@@ -398,28 +470,65 @@ const WINDOW_START_CLICKHOUSE = "2026-06-11 01:00:00.000";const RECENT_CUTOFF_CLICKHOUSE = "2026-06-25 00:30:00.000";const sdkUsageCacheKey = `langfuse:v4:sdk-usage:v1:${projectId}`;+const legacyApiUsageCacheKey = `langfuse:v4:legacy-api-usage:v1:${projectId}`;+const experimentPostUsageCacheKey = `langfuse:v4:experiment-post-usage:v1:${projectId}`;-/** In-memory Redis fake: mget/setex against a Map, status "ready". */-const enableRedisCache = (initialEntries: Record<string, string> = {}) => {- const store = new Map<string, string>(Object.entries(initialEntries));- redisMock.status = "ready";- redisMock.mget.mockImplementation(async (keys: string[]) =>- keys.map((key) => store.get(key) ?? null),- );- redisMock.setex.mockImplementation(- async (key: string, _ttlSeconds: number, value: string) => {- store.set(key, value);- return "OK";- },+const experimentPostBlob = (+ used: boolean,+ lastSeen: string | null = used ? "2026-06-24T15:00:00.000000Z" : null,+) =>+ JSON.stringify({+ version: 1,+ computedAt: "2026-06-25T00:00:00.000Z",+ used,+ lastSeen,+ });++const legacyApiBlob = (+ rows: { entrypoint: string; count: number; lastSeen: string }[],+) =>+ JSON.stringify({+ version: 1,+ computedAt: "2026-06-25T00:00:00.000Z",+ rows,+ });++const legacyApiUsageHeartbeatKey = "langfuse:v4:legacy-api-usage:heartbeat:v1";++const DEFAULT_SEED_TTL_SECONDS = 60 * 60;++const clearV4CacheKeys = async () => {+ memoryRedis.__reset();+};++/** Seed in-memory Redis keys and enable the cache for the code under test. */+const seedRedisCache = async (+ initialEntries: Record<string, string> = {},+ ttlSeconds = DEFAULT_SEED_TTL_SECONDS,+) => {+ await clearV4CacheKeys();+ redisAvailability.available = true;+ if (Object.keys(initialEntries).length === 0) return;+ await Promise.all(+ Object.entries(initialEntries).map(([key, value]) =>+ memoryRedis.setex(key, ttlSeconds, value),+ ),);- return store;};+const readRedisJson = async (key: string) => {+ const raw = await memoryRedis.get(key);+ return raw === null ? null : JSON.parse(raw);+};++const readRedisTtl = async (key: string) => memoryRedis.ttl(key);+describe("v4TransitionRouter", () => {- beforeEach(() => {+ beforeEach(async () => {vi.useFakeTimers({ toFake: ["Date"] });vi.setSystemTime(TEST_NOW);- redisMock.status = "end";+ redisAvailability.available = false;+ await clearV4CacheKeys();sharedEnvMock.CLICKHOUSE_READ_ONLY_URL ="https://clickhouse-read.example.com";sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =@@ -434,35 +543,31 @@ describe("v4TransitionRouter", () => {]);});+ afterEach(async () => {+ redisAvailability.available = false;+ await clearV4CacheKeys();+ });+afterEach(() => {vi.useRealTimers();vi.clearAllMocks();});- it("summarizes legacy public API usage for a project with route classification", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([+ it("serves legacy public API usage from Redis without querying ClickHouse", async () => {+ await seedRedisCache({+ [legacyApiUsageCacheKey]: legacyApiBlob([{- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "0.6666666666666666",- lastSeen: "2026-06-24T12:34:56.789123Z",- },- {- projectId,entrypoint: "publicapi: GET /api/public/datasets/{datasetName}/runs",- count: "1",+ count: 1,lastSeen: "2026-06-24T14:00:00.000000Z",},- ])- .mockResolvedValueOnce([{- projectId,entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "1.3333333333333335",+ count: 2,lastSeen: "2026-06-24T15:00:00.000000Z",},- ]);+ ]),+ });const caller = createCaller();const rows = await caller.legacyApiUsageSummary({@@ -483,187 +588,34 @@ describe("v4TransitionRouter", () => {lastSeen: "2026-06-24T15:00:00.000000Z",},]);-- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);- const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];- expect(clickhouseQuery?.query).toContain(- "FROM clusterAllReplicas('test-cluster', 'system.query_log')",- );- expect(clickhouseQuery?.query).not.toContain("toStartOfInterval");- expect(clickhouseQuery?.query).not.toContain("bucket_time");- expect(clickhouseQuery?.query).toContain(- "splitByChar('?', JSONExtractString(log_comment, 'route'))[1]",- );- expect(clickhouseQuery?.query).toContain(- "sum(1.0 / clickhouse_queries_per_api_call) AS count",- );- expect(clickhouseQuery?.query).toContain(- "formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen",- );- expect(clickhouseQuery?.query).toContain(- "SETTINGS skip_unavailable_shards = 1",- );- expect(clickhouseQuery?.query).toContain("AND type = 'QueryFinish'");- expect(clickhouseQuery?.query).toContain(- "JSONExtractString(log_comment, 'tag_schema_version') = '1'",- );- expect(clickhouseQuery?.query).toContain(- "JSONExtractString(log_comment, 'surface') = 'publicapi'",- );- expect(clickhouseQuery?.query).toContain(- "JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}",- );- expect(clickhouseQuery?.query).toContain(- "GROUP BY project_id, legacy_route",- );- expect(clickhouseQuery?.query).not.toContain("hostName()");- expect(clickhouseQuery?.params).toMatchObject({- projectIds: [projectId],- });- expect(clickhouseQuery?.tags).toEqual({- route: "v4-legacy-api-usage-summary",- });- expect(clickhouseQuery?.clickhouseSettings).toEqual({- skip_unavailable_shards: 1,- });- expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({- preferredClickhouseService: "ReadOnly",- });- expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({- preferredClickhouseService: "ReadWrite",- });-- [- "GET /api/public/spans",- "GET /api/public/generations",- "GET /api/public/traces",- "GET /api/public/sessions",- "GET /api/public/observations",- "GET /api/public/scores",- "GET /api/public/v2/scores",- "GET /api/public/metrics",- "GET /api/public/metrics/daily",- "GET /api/public/dataset-run-items",- ].forEach((route) => expect(clickhouseQuery?.query).toContain(route));-- [- "GET /api/public/traces/{id}",- "GET /api/public/sessions/{id}",- "GET /api/public/observations/{id}",- "GET /api/public/scores/{id}",- "GET /api/public/v2/scores/{id}",- "GET /api/public/datasets/{datasetName}/runs",- "GET /api/public/datasets/{datasetName}/runs/{runName}",- ].forEach((route) => expect(clickhouseQuery?.query).toContain(route));- expect(clickhouseQuery?.query).toContain(- "match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs'",- );- expect(clickhouseQuery?.query).toContain(- "match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1",- );- expect(clickhouseQuery?.query).not.toContain("DELETE /api/public/datasets");- expect(clickhouseQuery?.query).toContain(- "'GET /api/public/traces',\n 'GET /api/public/observations',\n 'GET /api/public/scores',\n 'GET /api/public/v2/scores',\n 'GET /api/public/metrics/daily',\n 'GET /api/public/dataset-run-items'\n ), 2",- );- expect(clickhouseQuery?.query).toContain(- "'GET /api/public/sessions',\n 'GET /api/public/metrics'\n ), 1",- );- expect(clickhouseQuery?.query).toContain(- "match(route_path, '^GET /api/public/traces/[^/?#]+$'), 3",- );- });-- it("queries the main service once when no separate read replica is configured", async () => {- sharedEnvMock.CLICKHOUSE_READ_ONLY_URL = sharedEnvMock.CLICKHOUSE_URL;- const caller = createCaller();-- await caller.legacyApiUsageSummary({- projectId,- });-- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);- expect(mockedQueryClickhouse).toHaveBeenCalledWith(- expect.objectContaining({ preferredClickhouseService: "ReadWrite" }),- );+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();});- it("returns legacy API usage when one ClickHouse service is unavailable", async () => {- mockedQueryClickhouse- .mockRejectedValueOnce(new Error("read replica unavailable"))- .mockResolvedValueOnce([- {- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "1",- lastSeen: "2026-06-24T15:00:00.000000Z",- },- ]);+ it("treats missing legacy API Redis entries as no usage", async () => {+ await seedRedisCache();const caller = createCaller();await expect(caller.legacyApiUsageSummary({projectId,}),- ).resolves.toEqual([- {- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: 1,- lastSeen: "2026-06-24T15:00:00.000000Z",- },- ]);-- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);+ ).resolves.toEqual([]);+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();});- it("does not double-count identical results from two ClickHouse services", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- {- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "2",- lastSeen: "2026-06-24T14:00:00.000000Z",- },- ])- .mockResolvedValueOnce([- {- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "2",- lastSeen: "2026-06-24T14:00:00.000000Z",- },- ]);- const caller = createCaller();-- await expect(- caller.legacyApiUsageSummary({+ it("queries SDK usage for only the authorized project", async () => {+ mockedQueryClickhouse.mockResolvedValueOnce([+ mockSdkUsageRow({projectId,+ sdkName: "python",+ sdkVersion: "4.7.0",+ publicKey: "pk-lf-python",+ eventCount: "2",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T02:00:00Z",}),- ).resolves.toEqual([- {- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: 2,- lastSeen: "2026-06-24T14:00:00.000000Z",- },]);- });-- it("queries SDK usage for only the authorized project", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "4.7.0",- publicKey: "pk-lf-python",- eventCount: "2",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T02:00:00Z",- }),- ])- .mockResolvedValueOnce([]);const caller = createCaller();const summary = await caller.sdkUsageSummary({@@ -686,24 +638,23 @@ describe("v4TransitionRouter", () => {},],});- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);expect(mockedQueryClickhouse.mock.calls[0]?.[0].params).toMatchObject({projectIds: [projectId],});- expect(mockedQueryClickhouse.mock.calls[1]?.[0].params).toMatchObject({- projectIds: [projectId],- });});it("rejects project summaries outside the caller session", async () => {const caller = createCaller();+ const range = {};await expect(- caller.sdkUsageSummary({ projectId: outsideProjectId }),+ caller.sdkUsageSummary({ projectId: outsideProjectId, ...range }),).rejects.toThrow("User is not a member of this project");await expect(caller.legacyApiUsageSummary({projectId: outsideProjectId,+ ...range,}),).rejects.toThrow("User is not a member of this project");expect(mockedQueryClickhouse).not.toHaveBeenCalled();@@ -1118,75 +1069,73 @@ describe("v4TransitionRouter", () => {);it("summarizes outdated SDK usage series by organization project", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "3.9.0",- publicKey: "pk-lf-python",- eventCount: "8",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T02:00:00Z",- }),- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "4.14.1",- publicKey: "pk-lf-python",- eventCount: "13",- firstSeen: "2026-06-24T03:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- }),- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "4.6.9",- publicKey: "pk-lf-pre-v4-python",- eventCount: "5",- firstSeen: "2026-06-24T12:00:00Z",- lastSeen: "2026-06-24T13:00:00Z",- }),- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "4.7.0",- publicKey: "pk-lf-current-python",- eventCount: "6",- firstSeen: "2026-06-24T13:30:00Z",- lastSeen: "2026-06-24T14:00:00Z",- }),- mockSdkUsageRow({- projectId: secondProjectId,- sdkName: "@langfuse/tracing",- sdkVersion: "5.3.9",- publicKey: "pk-lf-old-js",- eventCount: "5",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- }),- mockSdkUsageRow({- projectId: secondProjectId,- source: "otel",- sdkName: "unknown",- sdkVersion: "unknown",- publicKey: "pk-lf-otel",- eventCount: "3",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- }),- mockSdkUsageRow({- projectId: secondProjectId,- source: "otel-dual-write",- sdkName: "custom-otel-writer",- sdkVersion: "1.2.3",- publicKey: "pk-lf-custom-otel",- eventCount: "2",- firstSeen: "2026-06-24T02:00:00Z",- lastSeen: "2026-06-24T03:00:00Z",- }),- ])- .mockResolvedValueOnce([{ projectId, count: "1" }]);+ mockedQueryClickhouse.mockResolvedValueOnce([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "3.9.0",+ publicKey: "pk-lf-python",+ eventCount: "8",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T02:00:00Z",+ }),+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "4.14.1",+ publicKey: "pk-lf-python",+ eventCount: "13",+ firstSeen: "2026-06-24T03:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ }),+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "4.6.9",+ publicKey: "pk-lf-pre-v4-python",+ eventCount: "5",+ firstSeen: "2026-06-24T12:00:00Z",+ lastSeen: "2026-06-24T13:00:00Z",+ }),+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "4.7.0",+ publicKey: "pk-lf-current-python",+ eventCount: "6",+ firstSeen: "2026-06-24T13:30:00Z",+ lastSeen: "2026-06-24T14:00:00Z",+ }),+ mockSdkUsageRow({+ projectId: secondProjectId,+ sdkName: "@langfuse/tracing",+ sdkVersion: "5.3.9",+ publicKey: "pk-lf-old-js",+ eventCount: "5",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ }),+ mockSdkUsageRow({+ projectId: secondProjectId,+ source: "otel",+ sdkName: "unknown",+ sdkVersion: "unknown",+ publicKey: "pk-lf-otel",+ eventCount: "3",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ }),+ mockSdkUsageRow({+ projectId: secondProjectId,+ source: "otel-dual-write",+ sdkName: "custom-otel-writer",+ sdkVersion: "1.2.3",+ publicKey: "pk-lf-custom-otel",+ eventCount: "2",+ firstSeen: "2026-06-24T02:00:00Z",+ lastSeen: "2026-06-24T03:00:00Z",+ }),+ ]);const mockPrisma = {project: {findMany: vi.fn().mockResolvedValue([@@ -1205,8 +1154,8 @@ describe("v4TransitionRouter", () => {{projectId,experimentInstrumentationMigration: {- status: "sdk_usage_inconclusive",- upgradePath: "sdk",+ status: "not_required",+ upgradePath: null,},// Most recently seen first: the server re-sorts after merging cached// and live series.@@ -1361,7 +1310,7 @@ describe("v4TransitionRouter", () => {expect(mockPrisma.project.findMany).toHaveBeenCalledWith(accessibleProjectsFindManyArgs,);- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);const usageQuery = mockedQueryClickhouse.mock.calls[0]?.[0];expect(usageQuery?.query).toContain("FROM events_core");expect(usageQuery?.query).not.toContain("WITH selected");@@ -1409,46 +1358,21 @@ describe("v4TransitionRouter", () => {use_query_cache: 1,query_cache_ttl: 300,});-- const experimentUsageQuery = mockedQueryClickhouse.mock.calls[1]?.[0];- expect(experimentUsageQuery?.query).toContain(- "splitByChar('?', JSONExtractString(log_comment, 'route'))[1] = 'POST /api/public/dataset-run-items'",- );- expect(experimentUsageQuery?.query).toContain(- "JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}",- );- expect(experimentUsageQuery?.query).toContain(- "event_date >= toDate({fromTimestamp: DateTime64(3)})",- );- expect(experimentUsageQuery?.query).not.toContain("hostName()");- expect(experimentUsageQuery?.params).toMatchObject({- projectIds: [projectId, secondProjectId],- fromTimestamp: WINDOW_START_CLICKHOUSE,- toTimestamp: WINDOW_END_CLICKHOUSE,- });- expect(experimentUsageQuery?.tags).toEqual({- route: "v4-experiment-instrumentation-summary",- });- // The query_log-backed experiment check must not enable the query cache.- expect(- experimentUsageQuery?.clickhouseSettings?.use_query_cache,- ).toBeUndefined();});- it("finds dataset-run-items POST usage when only the main service has it", async () => {- sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =- "https://clickhouse-events-read.example.com";- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- source: "otel",- sdkName: "unknown",- sdkVersion: "unknown",- }),- ])- .mockRejectedValueOnce(new Error("events read replica unavailable"))- .mockResolvedValueOnce([{ projectId, count: "1" }]);+ it("requires an API upgrade when Redis shows dataset-run-items POST usage", async () => {+ await seedRedisCache({+ [experimentPostUsageCacheKey]: experimentPostBlob(true),+ });+ // Redis is available but the SDK blob is missing: historical + live gap.+ mockedQueryClickhouse.mockResolvedValue([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "3.3.5",+ publicKey: "pk-lf-python",+ }),+ ]);const caller = createCaller({project: {findMany: vi.fn().mockResolvedValue([{ id: projectId }]),@@ -1463,29 +1387,29 @@ describe("v4TransitionRouter", () => {status: "required",upgradePath: "api",});- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(3);- expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({- preferredClickhouseService: "EventsReadOnly",- });- expect(mockedQueryClickhouse.mock.calls[2]?.[0]).toMatchObject({- preferredClickhouseService: "ReadWrite",- });+ const eventsCoreCalls = mockedQueryClickhouse.mock.calls.filter(([args]) =>+ args.query.includes("FROM events_core"),+ );+ expect(eventsCoreCalls).toHaveLength(2);+ expect(+ mockedQueryClickhouse.mock.calls.some(([args]) =>+ args.query.includes("system.query_log"),+ ),+ ).toBe(false);});it("summarizes SDK usage for a single project with exactly that projectId", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "4.7.0",- publicKey: "pk-lf-python",- eventCount: "6",- firstSeen: "2026-06-24T13:30:00Z",- lastSeen: "2026-06-24T14:00:00Z",- }),- ])- .mockResolvedValueOnce([]);+ mockedQueryClickhouse.mockResolvedValueOnce([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "4.7.0",+ publicKey: "pk-lf-python",+ eventCount: "6",+ firstSeen: "2026-06-24T13:30:00Z",+ lastSeen: "2026-06-24T14:00:00Z",+ }),+ ]);const caller = createCaller();@@ -1511,64 +1435,56 @@ describe("v4TransitionRouter", () => {],});- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);expect(mockedQueryClickhouse.mock.calls[0]?.[0]?.params).toMatchObject({projectIds: [projectId],});- expect(mockedQueryClickhouse.mock.calls[1]?.[0]?.params).toMatchObject({- projectIds: [projectId],- });expect(mockedQueryClickhouse.mock.calls[0]?.[0]?.tags).toEqual({route: "v4-sdk-usage-summary",});- expect(mockedQueryClickhouse.mock.calls[1]?.[0]?.tags).toEqual({- route: "v4-experiment-instrumentation-summary",- });});it("keeps source-specific series and consumes SQL-owned classification", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- {- projectId,- source: "otel-dual-write",- ingestionPath: "otel",- deliveryMode: "delayed",- sdkName: "unknown",- sdkVersion: "unknown",- canonicalSdkName: null,- sdkVersionMajor: null,- isValidSdkVersion: false,- attributionStatus: "missing_name_and_version",- publicKey: "pk-lf-shared",- v4MigrationStatus: "unknown",- remediationType: "update_otel_instrumentation",- actionLevel: "required",- eventCount: "3",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T02:00:00Z",- },- {- projectId,- source: "ingestion-api-dual-write",- ingestionPath: "ingestion_api",- deliveryMode: "delayed",- sdkName: "unknown",- sdkVersion: "unknown",- canonicalSdkName: null,- sdkVersionMajor: null,- isValidSdkVersion: false,- attributionStatus: "missing_name_and_version",- publicKey: "pk-lf-shared",- v4MigrationStatus: "unknown",- remediationType: "upgrade_instrumentation",- actionLevel: "required",- eventCount: "2",- firstSeen: "2026-06-24T03:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- },- ])- .mockResolvedValueOnce([]);+ mockedQueryClickhouse.mockResolvedValueOnce([+ {+ projectId,+ source: "otel-dual-write",+ ingestionPath: "otel",+ deliveryMode: "delayed",+ sdkName: "unknown",+ sdkVersion: "unknown",+ canonicalSdkName: null,+ sdkVersionMajor: null,+ isValidSdkVersion: false,+ attributionStatus: "missing_name_and_version",+ publicKey: "pk-lf-shared",+ v4MigrationStatus: "unknown",+ remediationType: "update_otel_instrumentation",+ actionLevel: "required",+ eventCount: "3",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T02:00:00Z",+ },+ {+ projectId,+ source: "ingestion-api-dual-write",+ ingestionPath: "ingestion_api",+ deliveryMode: "delayed",+ sdkName: "unknown",+ sdkVersion: "unknown",+ canonicalSdkName: null,+ sdkVersionMajor: null,+ isValidSdkVersion: false,+ attributionStatus: "missing_name_and_version",+ publicKey: "pk-lf-shared",+ v4MigrationStatus: "unknown",+ remediationType: "upgrade_instrumentation",+ actionLevel: "required",+ eventCount: "2",+ firstSeen: "2026-06-24T03:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ },+ ]);const caller = createCaller();const summary = await caller.sdkUsageSummary({@@ -1606,20 +1522,21 @@ describe("v4TransitionRouter", () => {});it("requires removing dataset-run-items POST usage for native OTel instrumentation", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- source: "otel",- sdkName: "unknown",- sdkVersion: "unknown",- publicKey: "pk-lf-otel",- eventCount: "3",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- }),- ])- .mockResolvedValueOnce([{ projectId, count: "1" }]);+ await seedRedisCache({+ [experimentPostUsageCacheKey]: experimentPostBlob(true),+ });+ mockedQueryClickhouse.mockResolvedValue([+ mockSdkUsageRow({+ projectId,+ source: "otel",+ sdkName: "unknown",+ sdkVersion: "unknown",+ publicKey: "pk-lf-otel",+ eventCount: "3",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ }),+ ]);const caller = createCaller({project: {findMany: vi.fn().mockResolvedValue([{ id: projectId }]),@@ -1639,20 +1556,19 @@ describe("v4TransitionRouter", () => {});});- it("preserves SDK usage when the experiment instrumentation check fails", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "3.9.0",- publicKey: "pk-lf-python",- eventCount: "3",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- }),- ])- .mockRejectedValueOnce(new Error("query_log unavailable"));+ it("keeps SDK usage when experiment POST Redis data is missing", async () => {+ // No worker entry and no query_log fallback: experiment stays not_required.+ mockedQueryClickhouse.mockResolvedValueOnce([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "3.9.0",+ publicKey: "pk-lf-python",+ eventCount: "3",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ }),+ ]);const caller = createCaller({project: {findMany: vi.fn().mockResolvedValue([{ id: projectId }]),@@ -1666,7 +1582,7 @@ describe("v4TransitionRouter", () => {expect(summary).toMatchObject({projectId,experimentInstrumentationMigration: {- status: "check_failed",+ status: "not_required",upgradePath: null,},sdkUsageSeries: [@@ -1682,19 +1598,20 @@ describe("v4TransitionRouter", () => {});it("requires an API upgrade when POST usage predates the experiment runner", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "3.3.5",- publicKey: "pk-lf-python",- eventCount: "3",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- }),- ])- .mockResolvedValueOnce([{ projectId, count: "1" }]);+ await seedRedisCache({+ [experimentPostUsageCacheKey]: experimentPostBlob(true),+ });+ mockedQueryClickhouse.mockResolvedValue([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "3.3.5",+ publicKey: "pk-lf-python",+ eventCount: "3",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ }),+ ]);const caller = createCaller({project: {findMany: vi.fn().mockResolvedValue([{ id: projectId }]),@@ -1712,19 +1629,20 @@ describe("v4TransitionRouter", () => {});it("does not require an upgrade for current experiment instrumentation", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "4.0.0",- publicKey: "pk-lf-python",- eventCount: "3",- firstSeen: "2026-06-24T01:00:00Z",- lastSeen: "2026-06-24T04:00:00Z",- }),- ])- .mockResolvedValueOnce([{ projectId, count: "1" }]);+ await seedRedisCache({+ [experimentPostUsageCacheKey]: experimentPostBlob(true),+ });+ mockedQueryClickhouse.mockResolvedValue([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "4.0.0",+ publicKey: "pk-lf-python",+ eventCount: "3",+ firstSeen: "2026-06-24T01:00:00Z",+ lastSeen: "2026-06-24T04:00:00Z",+ }),+ ]);const caller = createCaller({project: {findMany: vi.fn().mockResolvedValue([{ id: projectId }]),@@ -1741,36 +1659,23 @@ describe("v4TransitionRouter", () => {});});- it("summarizes legacy public API usage by organization project", async () => {- mockedQueryClickhouse- .mockResolvedValueOnce([+ it("summarizes legacy public API usage by organization project from Redis", async () => {+ await seedRedisCache({+ [legacyApiUsageCacheKey]: legacyApiBlob([{- projectId,entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "0.6666666666666666",- lastSeen: "2026-06-24T12:34:56.789123Z",- },- {- projectId: secondProjectId,- entrypoint: "publicapi: GET /api/public/metrics",- count: 3,- lastSeen: "2026-06-24T15:00:00.000000Z",- },- ])- .mockResolvedValueOnce([- {- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "0.3333333333333334",+ count: 1,lastSeen: "2026-06-24T14:00:00.000000Z",},+ ]),+ [`langfuse:v4:legacy-api-usage:v1:${secondProjectId}`]: legacyApiBlob([{- projectId: secondProjectId,entrypoint: "publicapi: GET /api/public/metrics",- count: 2,+ count: 5,lastSeen: "2026-06-24T16:00:00.000000Z",},- ]);+ ]),+ });const mockPrisma = {project: {findMany: vi.fn().mockResolvedValue([@@ -1799,39 +1704,10 @@ describe("v4TransitionRouter", () => {lastSeen: "2026-06-24T16:00:00.000000Z",},]);-expect(mockPrisma.project.findMany).toHaveBeenCalledWith(accessibleProjectsFindManyArgs,);- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);- const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];- expect(clickhouseQuery?.query).not.toContain("toStartOfInterval");- expect(clickhouseQuery?.query).not.toContain("bucket_time");- expect(clickhouseQuery?.query).toContain(- "JSONExtractString(log_comment, 'projectId') AS project_id",- );- expect(clickhouseQuery?.query).toContain(- "JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}",- );- expect(clickhouseQuery?.query).toContain("project_id AS projectId");- expect(clickhouseQuery?.query).toContain(- "formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen",- );- expect(clickhouseQuery?.query).toContain(- "GROUP BY project_id, legacy_route",- );- expect(clickhouseQuery?.params).toMatchObject({- projectIds: [projectId, secondProjectId],- });- expect(clickhouseQuery?.tags).toEqual({- route: "v4-legacy-api-usage-summary",- });- expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({- preferredClickhouseService: "ReadOnly",- });- expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({- preferredClickhouseService: "ReadWrite",- });+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();});describe("redis caching", () => {@@ -1876,7 +1752,7 @@ describe("v4TransitionRouter", () => {});it("splits SDK usage into a cached-historical and a live-gap query on cache miss", async () => {- enableRedisCache();+ await seedRedisCache();const historicalRow = mockSdkUsageRow({projectId,sdkName: "python",@@ -1896,8 +1772,6 @@ describe("v4TransitionRouter", () => {lastSeen: "2026-06-25T00:15:00Z",});mockedQueryClickhouse.mockImplementation(async (args) => {- // SDK path also fans out a query_log experiment-POST check; only- // events_core participates in the history/gap split under test.if (!args.query.includes("FROM events_core")) return [];return (args.params?.toTimestamp as string) === HOT_START_CLICKHOUSE? [historicalRow]@@ -1953,38 +1827,37 @@ describe("v4TransitionRouter", () => {});}- // Only the historical SDK slice is cached (1h TTL); the gap stays live.- // query_log experiment usage is not Redis-cached in this PR.- expect(redisMock.setex).toHaveBeenCalledTimes(1);- const sdkWrite = redisMock.setex.mock.calls.find(- ([key]) => key === sdkUsageCacheKey,- );- expect(sdkWrite?.[1]).toBe(60 * 60);- expect(JSON.parse(sdkWrite?.[2] as string)).toMatchObject({+ // Only the historical slice is cached (1h TTL); the gap stays live.+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({version: 1,hotStart: HOT_START_ISO,series: [expect.objectContaining({ eventCount: 5 })],});+ expect(await readRedisTtl(sdkUsageCacheKey)).toBeGreaterThan(55 * 60);+ expect(await readRedisTtl(sdkUsageCacheKey)).toBeLessThanOrEqual(60 * 60);+ // Experiment POST is Redis-only from the worker; the request path must+ // not shadow-write empty entries on miss.+ await expect(+ readRedisJson(experimentPostUsageCacheKey),+ ).resolves.toBeNull();});it("serves cached SDK history with a live gap query on cache hit", async () => {- enableRedisCache({+ await seedRedisCache({[sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries()]),+ [experimentPostUsageCacheKey]: experimentPostBlob(false),});- mockedQueryClickhouse.mockImplementation(async (args) => {- if (!args.query.includes("FROM events_core")) return [];- return [- mockSdkUsageRow({- projectId,- sdkName: "python",- sdkVersion: "3.9.0",- publicKey: "pk-lf-python",- eventCount: "2",- firstSeen: "2026-06-25T00:05:00Z",- lastSeen: "2026-06-25T00:15:00Z",- }),- ];- });+ mockedQueryClickhouse.mockResolvedValue([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "3.9.0",+ publicKey: "pk-lf-python",+ eventCount: "2",+ firstSeen: "2026-06-25T00:05:00Z",+ lastSeen: "2026-06-25T00:15:00Z",+ }),+ ]);const caller = createCaller();const summary = await caller.sdkUsageSummary({@@ -1999,25 +1872,30 @@ describe("v4TransitionRouter", () => {lastSeen: "2026-06-25T00:15:00Z",}),]);- // No dataset-run-items POST usage in the uncached query_log check.+ // Cached experiment usage: not_required without a query_log scan.expect(summary.experimentInstrumentationMigration).toEqual({status: "not_required",upgradePath: null,});- const eventsCoreCalls = mockedQueryClickhouse.mock.calls.filter(- ([args]) => args.query.includes("FROM events_core"),+ // Exactly one ClickHouse query: the live gap on events_core.+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);+ expect(mockedQueryClickhouse.mock.calls[0]?.[0].query).toContain(+ "FROM events_core",);- expect(eventsCoreCalls).toHaveLength(1);- expect(eventsCoreCalls[0]?.[0].params).toMatchObject({+ expect(mockedQueryClickhouse.mock.calls[0]?.[0].params).toMatchObject({fromTimestamp: HOT_START_CLICKHOUSE,toTimestamp: RECENT_CUTOFF_CLICKHOUSE,});- expect(redisMock.setex).not.toHaveBeenCalled();+ // Cache hit must not rewrite the historical blob.+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({+ hotStart: HOT_START_ISO,+ series: [expect.objectContaining({ eventCount: 5 })],+ });});it("drops cached SDK series that aged out of the 14-day window", async () => {- enableRedisCache({+ await seedRedisCache({[sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries(),cachedSdkSeries({@@ -2027,11 +1905,9 @@ describe("v4TransitionRouter", () => {lastSeen: "2026-06-10T00:00:00Z",}),]),+ [experimentPostUsageCacheKey]: experimentPostBlob(false),});- mockedQueryClickhouse.mockImplementation(async (args) => {- if (!args.query.includes("FROM events_core")) return [];- return [];- });+ mockedQueryClickhouse.mockResolvedValue([]);const caller = createCaller();const summary = await caller.sdkUsageSummary({@@ -2044,13 +1920,11 @@ describe("v4TransitionRouter", () => {});it("treats blobs with corrupt JSON as cache misses", async () => {- enableRedisCache({+ await seedRedisCache({[sdkUsageCacheKey]: "not-json",+ [experimentPostUsageCacheKey]: experimentPostBlob(false),});- mockedQueryClickhouse.mockImplementation(async (args) => {- if (!args.query.includes("FROM events_core")) return [];- return [];- });+ mockedQueryClickhouse.mockResolvedValue([]);const caller = createCaller();await caller.sdkUsageSummary({@@ -2062,9 +1936,155 @@ describe("v4TransitionRouter", () => {([args]) => args.query.includes("FROM events_core"),);expect(eventsCoreCalls).toHaveLength(2);- expect(- redisMock.setex.mock.calls.some(([key]) => key === sdkUsageCacheKey),- ).toBe(true);+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({+ version: 1,+ hotStart: HOT_START_ISO,+ });+ });++ it("serves legacy API usage from the cache without querying ClickHouse", async () => {+ await seedRedisCache({+ [legacyApiUsageCacheKey]: legacyApiBlob([+ {+ entrypoint: "publicapi: GET /api/public/traces",+ count: 4,+ lastSeen: "2026-06-24T12:00:00.000000Z",+ },+ {+ // Aged out of the window: trimmed at read time.+ entrypoint: "publicapi: GET /api/public/sessions",+ count: 2,+ lastSeen: "2026-06-10T00:00:00.000000Z",+ },+ ]),+ });+ const caller = createCaller();++ const rows = await caller.legacyApiUsageSummary({+ projectId,+ });++ expect(rows).toEqual([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces",+ count: 4,+ lastSeen: "2026-06-24T12:00:00.000000Z",+ },+ ]);+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();+ });++ it("does not query or shadow-write legacy API Redis entries on miss", async () => {+ await seedRedisCache();+ const mockPrisma = {+ project: {+ findMany: vi.fn().mockResolvedValue([+ { id: projectId, name: "V4 Transition Project" },+ { id: secondProjectId, name: "Second Project" },+ ]),+ },+ };+ const caller = createCaller(mockPrisma);++ await expect(+ caller.legacyApiUsageSummaryByProject({+ orgId,+ }),+ ).resolves.toEqual([]);+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();+ await expect(+ readRedisJson(`langfuse:v4:legacy-api-usage:v1:${secondProjectId}`),+ ).resolves.toBeNull();+ });++ describe("worker pipeline heartbeat", () => {+ it("serves experiment POST Redis data even when the heartbeat is stale", async () => {+ await seedRedisCache({+ [legacyApiUsageHeartbeatKey]: "2026-06-24T20:30:00.000Z",+ [sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries()]),+ [experimentPostUsageCacheKey]: experimentPostBlob(true),+ });+ mockedQueryClickhouse.mockResolvedValue([]);+ const caller = createCaller();++ const summary = await caller.sdkUsageSummary({+ projectId,+ });+ // python 3.9.0 + POST usage → inconclusive (needs current experiment SDK).+ expect(summary.experimentInstrumentationMigration).toEqual({+ status: "sdk_usage_inconclusive",+ upgradePath: "sdk",+ });+ expect(+ mockedQueryClickhouse.mock.calls.some(([args]) =>+ args.query.includes("system.query_log"),+ ),+ ).toBe(false);+ });++ it("treats missing entries as no usage while the heartbeat is fresh", async () => {+ await seedRedisCache({+ // Worker ran 30 minutes ago; it only writes entries for projects+ // WITH usage, so absence is authoritative.+ [legacyApiUsageHeartbeatKey]: "2026-06-25T00:00:00.000Z",+ [sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries()]),+ });+ mockedQueryClickhouse.mockResolvedValue([]);+ const caller = createCaller();++ await expect(+ caller.legacyApiUsageSummary({+ projectId,+ }),+ ).resolves.toEqual([]);+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();+ // The request path must not shadow the worker's entries either.+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();++ const summary = await caller.sdkUsageSummary({+ projectId,+ });+ expect(summary.experimentInstrumentationMigration).toEqual({+ status: "not_required",+ upgradePath: null,+ });+ // Only the SDK live-gap query on events_core; no query_log scan.+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);+ expect(mockedQueryClickhouse.mock.calls[0]?.[0].query).toContain(+ "FROM events_core",+ );+ });++ it("serves legacy API Redis data even when the heartbeat is stale", async () => {+ await seedRedisCache({+ // 4h old: past the 3h freshness horizon — still serve Redis, no CH.+ [legacyApiUsageHeartbeatKey]: "2026-06-24T20:30:00.000Z",+ [legacyApiUsageCacheKey]: legacyApiBlob([+ {+ entrypoint: "publicapi: GET /api/public/traces",+ count: 4,+ lastSeen: "2026-06-24T12:00:00.000000Z",+ },+ ]),+ });+ const caller = createCaller();++ const rows = await caller.legacyApiUsageSummary({+ projectId,+ });++ expect(rows).toEqual([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces",+ count: 4,+ lastSeen: "2026-06-24T12:00:00.000000Z",+ },+ ]);+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();+ });});describe("migrationActions", () => {@@ -2084,7 +2104,7 @@ describe("v4TransitionRouter", () => {});it("on cold cache, runs history+gap queries, writes back, and derives the SDK flag", async () => {- enableRedisCache();+ await seedRedisCache();mockedQueryClickhouse.mockImplementation(async (args) => {if (!args.query.includes("FROM events_core")) return [];return (args.params?.toTimestamp as string) === HOT_START_CLICKHOUSE@@ -2116,16 +2136,24 @@ describe("v4TransitionRouter", () => {([args]) => args.query.includes("FROM events_core"),);expect(eventsCoreCalls).toHaveLength(2);- expect(- redisMock.setex.mock.calls.some(([key]) => key === sdkUsageCacheKey),- ).toBe(true);+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({+ version: 1,+ hotStart: HOT_START_ISO,+ series: [expect.objectContaining({ eventCount: 5 })],+ });+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();+ await expect(+ readRedisJson(experimentPostUsageCacheKey),+ ).resolves.toBeNull();});it("on cache hit, runs the live gap query and merges for the SDK flag", async () => {- enableRedisCache({+ await seedRedisCache({[sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries({ actionLevel: "required" }),]),+ [experimentPostUsageCacheKey]: experimentPostBlob(false),+ [legacyApiUsageCacheKey]: legacyApiBlob([]),});mockedQueryClickhouse.mockImplementation(async (args) => {if (!args.query.includes("FROM events_core")) return [];@@ -2136,9 +2164,8 @@ describe("v4TransitionRouter", () => {await expect(caller.migrationActions({ projectId })).resolves.toEqual({forceV3Experience: false,sdkActionNeeded: true,- // query_log-backed categories stay unknown until the follow-up PR.- experimentsActionNeeded: null,- apisActionNeeded: null,+ experimentsActionNeeded: false,+ apisActionNeeded: false,evalsActionNeeded: true,exportsActionNeeded: false,});@@ -2151,11 +2178,15 @@ describe("v4TransitionRouter", () => {fromTimestamp: HOT_START_CLICKHOUSE,toTimestamp: RECENT_CUTOFF_CLICKHOUSE,});- expect(redisMock.setex).not.toHaveBeenCalled();+ // Cache hit must not rewrite the historical SDK blob.+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({+ hotStart: HOT_START_ISO,+ series: [expect.objectContaining({ eventCount: 5 })],+ });});it("short-circuits partner-managed (forced v3) projects without any I/O", async () => {- enableRedisCache();+ await seedRedisCache();sharedServerMock.isForceV3ExperienceProject.mockReturnValueOnce(true);const prismaMock = mockPrismaForActions();const caller = createCaller(prismaMock);@@ -2170,12 +2201,12 @@ describe("v4TransitionRouter", () => {});expect(prismaMock.jobConfiguration.groupBy).not.toHaveBeenCalled();- expect(redisMock.mget).not.toHaveBeenCalled();expect(mockedQueryClickhouse).not.toHaveBeenCalled();});it("when Redis is unavailable, queries the full window and derives the SDK flag", async () => {- redisMock.status = "end";+ // Default beforeEach leaves the cache gated off (status proxy → "end").+ redisAvailability.available = false;mockedQueryClickhouse.mockImplementation(async (args) => {if (!args.query.includes("FROM events_core")) return [];return [@@ -2204,7 +2235,6 @@ describe("v4TransitionRouter", () => {([args]) => args.query.includes("FROM events_core"),);expect(eventsCoreCalls).toHaveLength(1);- expect(redisMock.mget).not.toHaveBeenCalled();});});});diff --git a/web/src/features/v4/server/v4TransitionCache.ts b/web/src/features/v4/server/v4TransitionCache.tsindex 9751d67cbf..4ee56d056f 100644--- a/web/src/features/v4/server/v4TransitionCache.ts+++ b/web/src/features/v4/server/v4TransitionCache.ts@@ -1,8 +1,18 @@import { z } from "zod/v4";-import { logger, redis } from "@langfuse/shared/src/server";+import {+ logger,+ redis,+ v4ExperimentPostUsageBlobSchema,+ v4ExperimentPostUsageProjectKey,+ v4LegacyApiUsageBlobSchema,+ v4LegacyApiUsageProjectKey,+ type V4ExperimentPostUsageBlob,+ type V4LegacyApiUsageBlob,+ type V4LegacyApiUsageRow,+} from "@langfuse/shared/src/server";/**- * Redis caching for the v4 transition SDK usage check.+ * Redis caching for the v4 transition usage checks.** SDK ingestion is read from the indexed `events_core` table. A per-project* blob covers the detection window up to an hour-aligned `hotStart` boundary;@@ -11,9 +21,9 @@ import { logger, redis } from "@langfuse/shared/src/server";* left-edge overcount on the sliding 14-day window (and stale classification* fields embedded in the blob) stay bounded to ~1h.*- * Deprecated public API and experiment POST usage come from `system.query_log`- * and are intentionally not cached here — that path lives in a follow-up PR- * with a worker-maintained pipeline. Sidebar and panel SDK reads share this+ * Deprecated public API and experiment POST usage are maintained by the+ * worker pipeline into the shared Redis keys; the request path only reads+ * those entries (miss = no usage). Sidebar and panel SDK reads share this* blob + gap-fill path so both stay consistent.** Helpers degrade gracefully: no Redis, Redis errors, or unparsable entries@@ -29,7 +39,6 @@ export const MIGRATION_INGRESS_EVENT_SOURCES = [/** Historical blob TTL: 60 minutes (1 hour). */export const SDK_USAGE_CACHE_TTL_SECONDS = 60 * 60;-const sdkUsageSeriesSchema = z.object({source: z.enum(MIGRATION_INGRESS_EVENT_SOURCES),ingestionPath: z.enum(["otel", "ingestion_api"]),@@ -71,8 +80,14 @@ const sdkUsageBlobSchema = z.object({export type SdkUsageCacheBlob = z.infer<typeof sdkUsageBlobSchema>;+export type CachedLegacyApiUsageRow = V4LegacyApiUsageRow;+export type LegacyApiUsageCacheBlob = V4LegacyApiUsageBlob;+export type ExperimentPostUsageCacheBlob = V4ExperimentPostUsageBlob;+const sdkUsageKey = (projectId: string) =>`langfuse:v4:sdk-usage:v1:${projectId}`;+const legacyApiUsageKey = v4LegacyApiUsageProjectKey;+const experimentPostUsageKey = v4ExperimentPostUsageProjectKey;export const isV4TransitionCacheAvailable = (): boolean =>redis != null && redis.status !== "end" && redis.status !== "close";@@ -142,6 +157,19 @@ export const writeSdkUsageCache = (})),);+export const readLegacyApiUsageCache = (+ projectIds: string[],+): Promise<(LegacyApiUsageCacheBlob | null)[]> =>+ readBlobs(projectIds.map(legacyApiUsageKey), v4LegacyApiUsageBlobSchema);++export const readExperimentPostUsageCache = (+ projectIds: string[],+): Promise<(ExperimentPostUsageCacheBlob | null)[]> =>+ readBlobs(+ projectIds.map(experimentPostUsageKey),+ v4ExperimentPostUsageBlobSchema,+ );+const MINUTE_MS = 60 * 1000;const HOUR_MS = 60 * MINUTE_MS;diff --git a/web/src/features/v4/server/v4TransitionQueryLogUsage.ts b/web/src/features/v4/server/v4TransitionQueryLogUsage.tsindex e2b7f37923..03c3ece75d 100644--- a/web/src/features/v4/server/v4TransitionQueryLogUsage.ts+++ b/web/src/features/v4/server/v4TransitionQueryLogUsage.ts@@ -1,31 +1,18 @@/**- * ClickHouse system.query_log helpers for v4 transition checks- * (legacy public API usage and experiment POST /dataset-run-items).+ * Redis-backed v4 transition usage reads for deprecated public API and+ * experiment POST /dataset-run-items checks.*- * Legacy API stays on a direct query_log read (no Redis worker yet). The- * public summary uses the server detection window so behavior matches the- * pre-split service; internal helpers still accept from/to for reuse.+ * Both paths are worker-maintained Redis only (including when the pipeline+ * heartbeat is stale). A missing entry means no usage; the request path never+ * scans `system.query_log`.*/-import { env } from "@langfuse/shared/src/env";import {- convertDateToClickhouseDateTime,- logger,- queryClickhouse,- systemTableRef,- type PreferredClickhouseService,-} from "@langfuse/shared/src/server";-import {- getV4TransitionDetectionWindow,+ readExperimentPostUsageCache,+ readLegacyApiUsageCache,V4_TRANSITION_DETECTION_WINDOW_MS,+ type CachedLegacyApiUsageRow,} from "@/src/features/v4/server/v4TransitionCache";-type LegacyApiUsageSummaryByProjectRow = {- projectId: string;- entrypoint: string;- count: string | number;- lastSeen: string;-};-export type LegacyApiUsageSummaryByProjectResultRow = {projectId: string;entrypoint: string;@@ -33,64 +20,6 @@ export type LegacyApiUsageSummaryByProjectResultRow = {lastSeen: string;};-export type DatasetRunItemsPostUsageByProjectRow = {- projectId: string;- count: string | number;-};--type SystemQueryLogReadService = "ReadOnly" | "EventsReadOnly";--const getSystemQueryLogServices = ({- readService,- readUrl,-}: {- readService: SystemQueryLogReadService;- readUrl: string | undefined;-}): PreferredClickhouseService[] =>- readUrl &&- new URL(readUrl).toString() !== new URL(env.CLICKHOUSE_URL).toString()- ? [readService, "ReadWrite"]- : ["ReadWrite"];--const querySystemQueryLogAcrossServices = async <T>({- preferredClickhouseServices,- queryService,- failureMessage,-}: {- preferredClickhouseServices: PreferredClickhouseService[];- queryService: (service: PreferredClickhouseService) => Promise<T[]>;- failureMessage: string;-}): Promise<T[][]> => {- const settledServiceRows = await Promise.allSettled(- preferredClickhouseServices.map(queryService),- );- const firstSuccessfulResult = settledServiceRows.find(- (result) => result.status === "fulfilled",- );- if (!firstSuccessfulResult) {- const firstFailure = settledServiceRows.find(- (result) => result.status === "rejected",- );- throw firstFailure?.reason ?? new Error("ClickHouse services unavailable");- }-- settledServiceRows.forEach((result, index) => {- if (result.status === "rejected") {- logger.warn(failureMessage, {- preferredClickhouseService: preferredClickhouseServices[index],- error:- result.reason instanceof Error- ? result.reason.message- : "Unknown error",- });- }- });-- return settledServiceRows.flatMap((result) =>- result.status === "fulfilled" ? [result.value] : [],- );-};-/*** Read-time aging-out rule: drop entries whose last activity left the trailing* detection window (same rule as the SDK path).@@ -98,298 +27,71 @@ const querySystemQueryLogAcrossServices = async <T>({const isWithinDetectionWindow = (lastSeen: string, nowMs: number): boolean =>Date.parse(lastSeen) >= nowMs - V4_TRANSITION_DETECTION_WINDOW_MS;-export type DatasetRunItemsPostUsageResult =- | {- status: "success";- rows: DatasetRunItemsPostUsageByProjectRow[];- }- | {- status: "error";- rows: DatasetRunItemsPostUsageByProjectRow[];- };--/** POST /api/public/dataset-run-items usage from system.query_log. */-export const getDatasetRunItemsPostUsageByProject = async ({- projectIds,- fromTimestamp,- toTimestamp,-}: {- projectIds: string[];- fromTimestamp: Date;- toTimestamp: Date;-}): Promise<DatasetRunItemsPostUsageResult> => {- if (projectIds.length === 0) {- return { status: "success", rows: [] };- }-- const systemQueryLogServices = getSystemQueryLogServices({- readService: "EventsReadOnly",- readUrl:- env.CLICKHOUSE_EVENTS_READ_ONLY_URL ?? env.CLICKHOUSE_READ_ONLY_URL,- });-- try {- const serviceRows = await querySystemQueryLogAcrossServices({- preferredClickhouseServices: systemQueryLogServices,- queryService: (preferredClickhouseService) =>- queryClickhouse<DatasetRunItemsPostUsageByProjectRow>({- query: `-SELECT- JSONExtractString(log_comment, 'projectId') AS projectId,- count() AS count-FROM ${systemTableRef("system.query_log")}-WHERE- event_time >= {fromTimestamp: DateTime64(3)}- AND event_time <= {toTimestamp: DateTime64(3)}- AND event_date >= toDate({fromTimestamp: DateTime64(3)})- AND event_date <= toDate({toTimestamp: DateTime64(3)})- AND type = 'QueryFinish'- AND JSONExtractString(log_comment, 'tag_schema_version') = '1'- AND JSONExtractString(log_comment, 'surface') = 'publicapi'- AND JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}- AND splitByChar('?', JSONExtractString(log_comment, 'route'))[1] = 'POST /api/public/dataset-run-items'-GROUP BY projectId-SETTINGS skip_unavailable_shards = 1- `,- params: {- projectIds,- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),- },- tags: { route: "v4-experiment-instrumentation-summary" },- preferredClickhouseService,- clickhouseSettings: { skip_unavailable_shards: 1 },- }),- failureMessage:- "Failed to query dataset-run-items POST usage from ClickHouse service",- });- return {- status: "success",- rows: serviceRows.flat(),- };- } catch (error: unknown) {- logger.warn(- "Failed to query dataset-run-items POST usage for v4 migration",- error,- );- return {- status: "error",- rows: [],- };- }-};+export const trimLegacyApiUsageRows = (+ rows: CachedLegacyApiUsageRow[],+ nowMs: number,+): CachedLegacyApiUsageRow[] =>+ rows.filter((row) => isWithinDetectionWindow(row.lastSeen, nowMs));-const getLegacyApiUsageSummariesForService = async ({+/**+ * Whether each project called `POST /api/public/dataset-run-items` within the+ * detection window. Redis-only: serve worker-maintained entries (including+ * when the heartbeat is stale) and treat a miss as no usage. No `query_log`+ * fallback on the request path.+ */+export const getExperimentPostUsageByProject = async ({projectIds,- fromTimestamp,- toTimestamp,- preferredClickhouseService,+ nowMs,}: {projectIds: string[];- fromTimestamp: Date;- toTimestamp: Date;- preferredClickhouseService: PreferredClickhouseService;-}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {- if (projectIds.length === 0) return [];-- const rows = await queryClickhouse<LegacyApiUsageSummaryByProjectRow>({- query: `-WITH selected AS (- SELECT- JSONExtractString(log_comment, 'projectId') AS project_id,- event_time_microseconds,- splitByChar('?', JSONExtractString(log_comment, 'route'))[1] AS route_path- FROM ${systemTableRef("system.query_log")}- WHERE- event_time >= {fromTimestamp: DateTime64(3)}- AND event_time <= {toTimestamp: DateTime64(3)}- AND event_date >= toDate({fromTimestamp: DateTime64(3)})- AND event_date <= toDate({toTimestamp: DateTime64(3)})- AND type = 'QueryFinish'- AND JSONExtractString(log_comment, 'tag_schema_version') = '1'- AND JSONExtractString(log_comment, 'surface') = 'publicapi'- AND JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}-),-classified AS (- SELECT- project_id,- event_time_microseconds,- multiIf(- route_path IN (- 'GET /api/public/spans',- 'GET /api/public/generations',- 'GET /api/public/traces',- 'GET /api/public/sessions',- 'GET /api/public/observations',- 'GET /api/public/scores',- 'GET /api/public/v2/scores',- 'GET /api/public/metrics',- 'GET /api/public/metrics/daily',- 'GET /api/public/dataset-run-items'- ), route_path,- match(route_path, '^GET /api/public/traces/[^/?#]+$'), 'GET /api/public/traces/{id}',- match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 'GET /api/public/sessions/{id}',- match(route_path, '^GET /api/public/observations/[^/?#]+$'), 'GET /api/public/observations/{id}',- match(route_path, '^GET /api/public/scores/[^/?#]+$'), 'GET /api/public/scores/{id}',- match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 'GET /api/public/v2/scores/{id}',- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs',- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 'GET /api/public/datasets/{datasetName}/runs/{runName}',- NULL- ) AS legacy_route,- multiIf(- route_path IN (- 'GET /api/public/spans',- 'GET /api/public/generations',- 'GET /api/public/traces',- 'GET /api/public/observations',- 'GET /api/public/scores',- 'GET /api/public/v2/scores',- 'GET /api/public/metrics/daily',- 'GET /api/public/dataset-run-items'- ), 2,- route_path IN (- 'GET /api/public/sessions',- 'GET /api/public/metrics'- ), 1,- match(route_path, '^GET /api/public/traces/[^/?#]+$'), 3,- match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 1,- match(route_path, '^GET /api/public/observations/[^/?#]+$'), 1,- match(route_path, '^GET /api/public/scores/[^/?#]+$'), 1,- match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 1,- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 1,- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1,- NULL- ) AS clickhouse_queries_per_api_call- FROM selected-)--SELECT- project_id AS projectId,- concat('publicapi: ', legacy_route) AS entrypoint,- sum(1.0 / clickhouse_queries_per_api_call) AS count,- formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen-FROM classified-WHERE legacy_route IS NOT NULL- AND clickhouse_queries_per_api_call IS NOT NULL-GROUP BY project_id, legacy_route-ORDER BY project_id ASC, legacy_route ASC-SETTINGS skip_unavailable_shards = 1- `,- params: {- projectIds,- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),- },- tags: { route: "v4-legacy-api-usage-summary" },- preferredClickhouseService,- clickhouseSettings: { skip_unavailable_shards: 1 },+ nowMs: number;+}): Promise<Map<string, boolean | "check_failed">> => {+ const usageByProject = new Map<string, boolean | "check_failed">();+ const cachedBlobs = await readExperimentPostUsageCache(projectIds);+ projectIds.forEach((projectId, index) => {+ const blob = cachedBlobs[index];+ if (!blob) {+ usageByProject.set(projectId, false);+ return;+ }+ const stillInWindow =+ blob.lastSeen == null+ ? blob.used+ : blob.used && isWithinDetectionWindow(blob.lastSeen, nowMs);+ usageByProject.set(projectId, stillInWindow);});-- return rows.map((row) => ({- projectId: row.projectId,- entrypoint: row.entrypoint,- count: Number(row.count),- lastSeen: row.lastSeen,- }));+ return usageByProject;};-const queryLegacyApiUsageSummaries = async ({+/**+ * Deprecated public API usage per project. Redis-only: serve worker-maintained+ * entries (including when the heartbeat is stale) and treat a miss as no+ * usage. The always-mounted sidebar never calls this procedure.+ */+export const getLegacyApiUsageSummaries = async ({projectIds,- fromTimestamp,- toTimestamp,}: {projectIds: string[];- fromTimestamp: Date;- toTimestamp: Date;}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {if (projectIds.length === 0) return [];- const preferredClickhouseServices = getSystemQueryLogServices({- readService: "ReadOnly",- readUrl: env.CLICKHOUSE_READ_ONLY_URL,- });- const serviceResultSets = await querySystemQueryLogAcrossServices({- preferredClickhouseServices,- queryService: (preferredClickhouseService) =>- getLegacyApiUsageSummariesForService({- projectIds,- fromTimestamp,- toTimestamp,- preferredClickhouseService,- }),- failureMessage: "Failed to query legacy API usage from ClickHouse service",- });-- const uniqueServiceResultSets = new Map<- string,- LegacyApiUsageSummaryByProjectResultRow[]- >();- for (const rows of serviceResultSets) {- const sortedRows = [...rows].sort(- (left, right) =>- left.projectId.localeCompare(right.projectId) ||- left.entrypoint.localeCompare(right.entrypoint),- );- uniqueServiceResultSets.set(JSON.stringify(sortedRows), sortedRows);- }- const deduplicatedServiceRows = Array.from(- uniqueServiceResultSets.values(),- ).flat();-- const rowsByProjectAndEntrypoint = new Map<- string,- LegacyApiUsageSummaryByProjectResultRow- >();-- for (const row of deduplicatedServiceRows) {- const key = `${row.projectId}\u0000${row.entrypoint}`;- const existing = rowsByProjectAndEntrypoint.get(key);- rowsByProjectAndEntrypoint.set(- key,- existing- ? {- ...existing,- count: existing.count + row.count,- lastSeen:- existing.lastSeen > row.lastSeen- ? existing.lastSeen- : row.lastSeen,- }- : row,+ const nowMs = Date.now();+ const cachedBlobs = await readLegacyApiUsageCache(projectIds);+ const rows: LegacyApiUsageSummaryByProjectResultRow[] = [];+ projectIds.forEach((projectId, index) => {+ const blob = cachedBlobs[index];+ if (!blob) return;+ rows.push(+ ...trimLegacyApiUsageRows(blob.rows, nowMs).map((row) => ({+ projectId,+ ...row,+ })),);- }+ });- return Array.from(rowsByProjectAndEntrypoint.values()).sort(+ return rows.sort((left, right) =>left.projectId.localeCompare(right.projectId) ||left.entrypoint.localeCompare(right.entrypoint),);};--/**- * Deprecated public API usage per project. Reads `system.query_log` directly;- * Redis caching and the worker pipeline for this path land in a follow-up PR.- * The always-mounted sidebar never calls this procedure.- */-export const getLegacyApiUsageSummaries = async ({- projectIds,-}: {- projectIds: string[];-}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {- if (projectIds.length === 0) return [];-- const nowMs = Date.now();- const { windowStart, windowEnd } = getV4TransitionDetectionWindow(nowMs);- const rows = await queryLegacyApiUsageSummaries({- projectIds,- fromTimestamp: windowStart,- toTimestamp: windowEnd,- });- return rows- .filter((row) => isWithinDetectionWindow(row.lastSeen, nowMs))- .sort(- (left, right) =>- left.projectId.localeCompare(right.projectId) ||- left.entrypoint.localeCompare(right.entrypoint),- );-};diff --git a/web/src/features/v4/server/v4TransitionRouter.ts b/web/src/features/v4/server/v4TransitionRouter.tsindex 8014ffd1d1..0c18a6f803 100644--- a/web/src/features/v4/server/v4TransitionRouter.ts+++ b/web/src/features/v4/server/v4TransitionRouter.ts@@ -134,7 +134,7 @@ export const v4TransitionRouter = createTRPCRouter({* Migration signal for always-mounted UI (the sidebar "Action required"* pill). Postgres for eval/export signals; SDK via the same Redis + live* gap-fill path as sdkUsageSummary. Deprecated API / experiment signals- * stay `null` until the follow-up query_log worker pipeline lands.+ * read worker-maintained Redis entries when available (miss = unknown/none).*/migrationActions: protectedProjectProcedure.input(z.object({ projectId: z.string() }))diff --git a/web/src/features/v4/server/v4TransitionSdkUsage.ts b/web/src/features/v4/server/v4TransitionSdkUsage.tsindex e4cf70442a..c53c5dba61 100644--- a/web/src/features/v4/server/v4TransitionSdkUsage.ts+++ b/web/src/features/v4/server/v4TransitionSdkUsage.ts@@ -4,7 +4,8 @@* Historical series come from a per-project Redis blob; the remaining gap up* to a minute-aligned recentCutoff is queried live from events_core (with* ClickHouse use_query_cache). Detection windows are server-side — no client- * from/to. Experiment POST usage is delegated to the query_log module.+ * from/to. Experiment POST usage is read from worker-maintained Redis only+ * (miss = no usage).*/import {convertDateToClickhouseDateTime,@@ -13,7 +14,7 @@ import {type IngestionSdkAttributionStatus,} from "@langfuse/shared/src/server";import { getSdkVersionCapabilityStatus } from "@/src/features/sdk-version/lib/sdkVersionCapabilities";-import { getDatasetRunItemsPostUsageByProject } from "@/src/features/v4/server/v4TransitionQueryLogUsage";+import { getExperimentPostUsageByProject } from "@/src/features/v4/server/v4TransitionQueryLogUsage";import {getV4TransitionDetectionWindow,isV4TransitionCacheAvailable,@@ -472,48 +473,7 @@ export const getSdkUsageSeriesByProject = async ({);};-/**- * Whether each project called `POST /api/public/dataset-run-items` within the- * detection window. Reads `system.query_log` directly; Redis caching for this- * path lands in a follow-up PR with the worker pipeline.- */-const getExperimentPostUsageByProject = async ({- projectIds,- nowMs,-}: {- projectIds: string[];- nowMs: number;-}): Promise<Map<string, boolean | "check_failed">> => {- const { windowStart, windowEnd } = getV4TransitionDetectionWindow(nowMs);- const usageByProject = new Map<string, boolean | "check_failed">();-- const queryResult = await getDatasetRunItemsPostUsageByProject({- projectIds,- fromTimestamp: windowStart,- toTimestamp: windowEnd,- });- if (queryResult.status === "error") {- projectIds.forEach((projectId) =>- usageByProject.set(projectId, "check_failed"),- );- return usageByProject;- }-- const projectsUsingDatasetRunItemsPost = new Set(- queryResult.rows- .filter((row) => Number(row.count) > 0)- .map((row) => row.projectId),- );- projectIds.forEach((projectId) =>- usageByProject.set(- projectId,- projectsUsingDatasetRunItemsPost.has(projectId),- ),- );- return usageByProject;-};--const deriveExperimentInstrumentationMigration = ({+export const deriveExperimentInstrumentationMigration = ({sdkUsageSeries,postUsage,}: {diff --git a/web/src/features/v4/server/v4TransitionService.ts b/web/src/features/v4/server/v4TransitionService.tsindex eca01fd860..4812612072 100644--- a/web/src/features/v4/server/v4TransitionService.ts+++ b/web/src/features/v4/server/v4TransitionService.ts@@ -10,7 +10,15 @@ import {} from "@langfuse/shared/src/db";import type { Session } from "next-auth";import { isForceV3ExperienceProject } from "@langfuse/shared/src/server";-import { getSdkUsageSeriesByProject } from "@/src/features/v4/server/v4TransitionSdkUsage";+import {+ readExperimentPostUsageCache,+ readLegacyApiUsageCache,+} from "@/src/features/v4/server/v4TransitionCache";+import { trimLegacyApiUsageRows } from "@/src/features/v4/server/v4TransitionQueryLogUsage";+import {+ deriveExperimentInstrumentationMigration,+ getSdkUsageSeriesByProject,+} from "@/src/features/v4/server/v4TransitionSdkUsage";export { getSdkUsageSummaries } from "@/src/features/v4/server/v4TransitionSdkUsage";export { getLegacyApiUsageSummaries } from "@/src/features/v4/server/v4TransitionQueryLogUsage";@@ -209,7 +217,7 @@ export const getTraceLevelEvalSummaries = async ({* Migration signals for the always-mounted sidebar "Action required" pill.* SDK uses the same Redis + live gap-fill path as the panel summaries so the* pill cannot go stale relative to them. Deprecated API / experiment signals- * stay `null` until the follow-up query_log worker pipeline lands.+ * read worker-maintained Redis caches when available (`null` = unknown).*/export const getMigrationActions = async ({prisma,@@ -234,22 +242,46 @@ export const getMigrationActions = async ({const nowMs = Date.now();const projectIds = [projectId];- const [evalSummaries, integrationSummaries, seriesByProject] =- await Promise.all([- getTraceLevelEvalSummaries({ prisma, projectIds }),- getLegacyIntegrationSummaries({ prisma, projectIds }),- getSdkUsageSeriesByProject({ projectIds, nowMs }),- ]);+ const [+ evalSummaries,+ integrationSummaries,+ seriesByProject,+ postBlobs,+ apiBlobs,+ ] = await Promise.all([+ getTraceLevelEvalSummaries({ prisma, projectIds }),+ getLegacyIntegrationSummaries({ prisma, projectIds }),+ getSdkUsageSeriesByProject({ projectIds, nowMs }),+ readExperimentPostUsageCache(projectIds),+ readLegacyApiUsageCache(projectIds),+ ]);const sdkUsageSeries = seriesByProject.get(projectId) ?? [];+ const postBlob = postBlobs[0] ?? null;+ const apiBlob = apiBlobs[0] ?? null;++ const experimentsActionNeeded =+ postBlob === null+ ? null+ : postBlob.used === false+ ? false+ : ["required", "sdk_usage_inconclusive"].includes(+ deriveExperimentInstrumentationMigration({+ sdkUsageSeries,+ postUsage: true,+ }).status,+ );return {forceV3Experience: isForceV3ExperienceProject(projectId),sdkActionNeeded: sdkUsageSeries.some((series) => series.actionLevel === "required",),- experimentsActionNeeded: null,- apisActionNeeded: null,+ experimentsActionNeeded,+ apisActionNeeded:+ apiBlob === null+ ? null+ : trimLegacyApiUsageRows(apiBlob.rows, nowMs).length > 0,evalsActionNeeded: (evalSummaries[0]?.traceLevelEvalCount ?? 0) > 0,exportsActionNeeded:(integrationSummaries[0]?.legacyIntegrationCount ?? 0) > 0,diff --git a/worker/src/app.ts b/worker/src/app.tsindex 7b53d13d16..2573516437 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -40,6 +40,7 @@ import {TraceUpsertQueue,CloudFreeTierUsageThresholdQueue,CloudUsageMeteringQueue,+ V4LegacyApiUsageQueue,EventPropagationQueue,EvalExecutionQueue,SecondaryEvalExecutionQueue,@@ -60,6 +61,7 @@ import {postHogIntegrationProcessingProcessor,postHogIntegrationProcessor,} from "./queues/postHogIntegrationQueue";+import { v4LegacyApiUsageProcessor } from "./queues/v4LegacyApiUsageQueue";import {mixpanelIntegrationProcessingProcessor,mixpanelIntegrationProcessor,@@ -524,6 +526,25 @@ if (env.QUEUE_CONSUMER_POSTHOG_INTEGRATION_QUEUE_IS_ENABLED === "true") {);}+if (env.QUEUE_CONSUMER_V4_LEGACY_API_USAGE_QUEUE_IS_ENABLED === "true") {+ // Instantiate the queue to trigger scheduled jobs+ V4LegacyApiUsageQueue.getInstance();++ WorkerManager.register(+ QueueName.V4LegacyApiUsageQueue,+ v4LegacyApiUsageProcessor,+ {+ concurrency: 1,+ limiter: {+ // The job scans system.query_log across all ClickHouse services;+ // never run it more than once per minute even if jobs pile up.+ max: 1,+ duration: 60_000,+ },+ },+ );+}+if (env.QUEUE_CONSUMER_MIXPANEL_INTEGRATION_QUEUE_IS_ENABLED === "true") {// Instantiate the queue to trigger scheduled jobsMixpanelIntegrationQueue.getInstance();diff --git a/worker/src/env.ts b/worker/src/env.tsindex b6be7c67ce..3ef360d2c7 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -293,6 +293,9 @@ const EnvSchema = z.object({QUEUE_CONSUMER_POSTHOG_INTEGRATION_QUEUE_IS_ENABLED: z.enum(["true", "false"]).default("true"),+ QUEUE_CONSUMER_V4_LEGACY_API_USAGE_QUEUE_IS_ENABLED: z+ .enum(["true", "false"])+ .default("true"),QUEUE_CONSUMER_MIXPANEL_INTEGRATION_QUEUE_IS_ENABLED: z.enum(["true", "false"]).default("true"),diff --git a/worker/src/features/queue-metrics-runner/index.test.ts b/worker/src/features/queue-metrics-runner/index.test.tsindex 0235f53f8a..19fafb67bd 100644--- a/worker/src/features/queue-metrics-runner/index.test.ts+++ b/worker/src/features/queue-metrics-runner/index.test.ts@@ -5,6 +5,7 @@ const mocks = vi.hoisted(() => ({getJobCounts: vi.fn(),recordGauge: vi.fn(),updateActiveIngestFailureProjectsMetric: vi.fn(),+ emitV4LegacyApiUsageFreshnessMetrics: vi.fn(),}));vi.mock("@langfuse/shared/src/server", () => ({@@ -50,6 +51,11 @@ vi.mock("../../queues/shardedQueueRegistry", () => ({})),}));+vi.mock("../v4/v4LegacyApiUsageMetrics", () => ({+ emitV4LegacyApiUsageFreshnessMetrics:+ mocks.emitV4LegacyApiUsageFreshnessMetrics,+}));+import { QueueMetricsRunner } from ".";class TestQueueMetricsRunner extends QueueMetricsRunner {@@ -69,6 +75,7 @@ describe("QueueMetricsRunner", () => {waiting: 7,});mocks.updateActiveIngestFailureProjectsMetric.mockResolvedValue(undefined);+ mocks.emitV4LegacyApiUsageFreshnessMetrics.mockResolvedValue(undefined);});it("emits only the canonical gauges for a non-sharded queue", async () => {diff --git a/worker/src/features/queue-metrics-runner/index.ts b/worker/src/features/queue-metrics-runner/index.tsindex df0b01ecf8..40241c472c 100644--- a/worker/src/features/queue-metrics-runner/index.ts+++ b/worker/src/features/queue-metrics-runner/index.ts@@ -16,6 +16,7 @@ import {SHARDED_QUEUE_BASE_NAMES,resolveQueueInstance,} from "../../queues/shardedQueueRegistry";+import { emitV4LegacyApiUsageFreshnessMetrics } from "../v4/v4LegacyApiUsageMetrics";type DepthType = "waiting" | "failed" | "active";@@ -83,6 +84,23 @@ export class QueueMetricsRunner extends PeriodicRunner {}),);+ // Heartbeat/cursor freshness for the v4 legacy API usage pipeline. Only+ // when this worker consumes that queue so self-hosted deployments without+ // the job do not emit a permanent "not fresh" signal.+ if (registeredNames.has(QueueName.V4LegacyApiUsageQueue)) {+ promises.push(+ emitV4LegacyApiUsageFreshnessMetrics()+ .then(() => undefined)+ .catch((err) => {+ this.markRunFailed(err);+ logger.error(+ "Queue metrics: failed to record v4 legacy API usage freshness",+ err,+ );+ }),+ );+ }+// Non-sharded queues: only poll queues with registered workersfor (const queueName of Object.values(QueueName)) {if (SHARDED_QUEUE_BASE_NAMES.has(queueName)) continue;diff --git a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.tsnew file mode 100644index 0000000000..881ed0de00--- /dev/null+++ b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts@@ -0,0 +1,432 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ listV4LegacyApiHourStarts,+ v4ExperimentPostUsageProjectKey,+ v4LegacyApiHourBucketKey,+ v4LegacyApiUsageProjectKey,+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,+ V4_LEGACY_API_USAGE_LOCK_KEY,+} from "@langfuse/shared/src/server";++/**+ * In-memory Redis fake covering the commands used by the handler and+ * RedisLock: get/set(NX)/setex/mget/eval(check-and-delete).+ */+const redisState = vi.hoisted(() => ({+ store: new Map<string, string>(),+}));++const redisMock = vi.hoisted(() => ({+ get: vi.fn(async (key: string) => redisState.store.get(key) ?? null),+ set: vi.fn(+ async (+ key: string,+ value: string,+ _ex?: string,+ _ttl?: number,+ nx?: string,+ ) => {+ if (nx === "NX" && redisState.store.has(key)) return null;+ redisState.store.set(key, value);+ return "OK";+ },+ ),+ setex: vi.fn(async (key: string, _ttl: number, value: string) => {+ redisState.store.set(key, value);+ return "OK";+ }),+ mget: vi.fn(async (keys: string[]) =>+ keys.map((key) => redisState.store.get(key) ?? null),+ ),+ eval: vi.fn(async (_script: string, _numKeys: number, ...args: string[]) => {+ const [key, value] = args;+ if (redisState.store.get(key) === value) {+ redisState.store.delete(key);+ return 1;+ }+ return 0;+ }),+ del: vi.fn(async (...keys: string[]) => {+ let deleted = 0;+ for (const key of keys) {+ if (redisState.store.delete(key)) deleted += 1;+ }+ return deleted;+ }),+}));++const mocks = vi.hoisted(() => ({+ queryClickhouse: vi.fn(),+}));++vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const original =+ await importOriginal<typeof import("@langfuse/shared/src/server")>();+ return {+ ...original,+ redis: redisMock,+ logger: {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ },+ queryClickhouse: mocks.queryClickhouse,+ systemTableRef: (table: string) =>+ `clusterAllReplicas('test-cluster', '${table}')`,+ convertDateToClickhouseDateTime: (date: Date) =>+ date.toISOString().replace("T", " ").replace("Z", ""),+ };+});++vi.mock("@langfuse/shared/src/env", () => ({+ env: {+ CLICKHOUSE_URL: "https://clickhouse-main.example.com",+ CLICKHOUSE_READ_ONLY_URL: "https://clickhouse-read.example.com",+ CLICKHOUSE_EVENTS_READ_ONLY_URL: undefined,+ },+}));++import { handleV4LegacyApiUsageJob } from "../handleV4LegacyApiUsageJob";++// 10:30 UTC: a regular run (the deep re-scan happens at 03:xx UTC only).+const TEST_NOW = new Date("2026-06-25T10:30:00Z");+const CURRENT_HOUR_ISO = "2026-06-25T10:00:00Z";+// floor(now - 14d) to the hour.+const COVERAGE_START_CLICKHOUSE = "2026-06-11 10:00:00.000";++const usageRow = (overrides: {+ hourStart?: string;+ projectId?: string;+ route?: string;+ count?: string | number;+ lastSeen?: string;+}) => ({+ hourStart: overrides.hourStart ?? "2026-06-25T09:00:00Z",+ projectId: overrides.projectId ?? "project-a",+ route: overrides.route ?? "GET /api/public/traces",+ count: overrides.count ?? "4",+ lastSeen: overrides.lastSeen ?? "2026-06-25T09:15:00.000000Z",+});++const readJson = (key: string): unknown => {+ const value = redisState.store.get(key);+ return value ? JSON.parse(value) : null;+};++const emptyBucket = () =>+ JSON.stringify({+ version: 1,+ computedAt: "2026-06-24T00:00:00.000Z",+ apiRows: [],+ experimentPostRows: [],+ });++/** Seeds a valid (empty) bucket for every hour of the trailing window. */+const seedCoverageBuckets = (skipHourIsos: string[] = []) => {+ const skip = new Set(skipHourIsos.map((iso) => Date.parse(iso)));+ for (const hourStartMs of listV4LegacyApiHourStarts(+ Date.parse("2026-06-11T10:00:00Z"),+ Date.parse(CURRENT_HOUR_ISO),+ )) {+ if (skip.has(hourStartMs)) continue;+ redisState.store.set(v4LegacyApiHourBucketKey(hourStartMs), emptyBucket());+ }+};++/** Marks the deep re-scan as recently done so runs use the 3h margin. */+const seedRecentDeepRescan = () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,+ "2026-06-25T05:00:00.000Z",+ );+};++const bucketKeyCount = (): number =>+ Array.from(redisState.store.keys()).filter((key) =>+ key.startsWith("langfuse:v4:legacy-api-usage:hour:v1:"),+ ).length;++describe("handleV4LegacyApiUsageJob", () => {+ beforeEach(() => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(TEST_NOW);+ redisState.store.clear();+ mocks.queryClickhouse.mockResolvedValue([]);+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.clearAllMocks();+ });++ it("cold start: scans the full 14-day window once and materializes buckets, rollups, cursor, and heartbeat", async () => {+ const rows = [+ usageRow({}),+ usageRow({+ hourStart: CURRENT_HOUR_ISO,+ projectId: "project-b",+ route: "POST /api/public/dataset-run-items",+ count: "2",+ lastSeen: "2026-06-25T10:05:00.000000Z",+ }),+ ];+ // Both services return identical result sets (overlapping replicas):+ // they must be deduplicated, not summed.+ mocks.queryClickhouse.mockResolvedValue(rows);++ await handleV4LegacyApiUsageJob();++ expect(mocks.queryClickhouse).toHaveBeenCalledTimes(2);+ const services = mocks.queryClickhouse.mock.calls.map(+ ([args]) => args.preferredClickhouseService,+ );+ expect(services).toEqual(["ReadWrite", "ReadOnly"]);+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({+ fromTimestamp: COVERAGE_START_CLICKHOUSE,+ toTimestamp: "2026-06-25 10:30:00.000",+ });+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].query).toContain(+ "toStartOfHour(event_time, 'UTC')",+ );++ // One bucket per hour in the window, empty ones included: 14*24 + 1.+ expect(bucketKeyCount()).toBe(337);+ expect(+ readJson(v4LegacyApiHourBucketKey(Date.parse("2026-06-25T09:00:00Z"))),+ ).toMatchObject({+ version: 1,+ apiRows: [+ {+ projectId: "project-a",+ entrypoint: "publicapi: GET /api/public/traces",+ count: 4,+ lastSeen: "2026-06-25T09:15:00.000000Z",+ },+ ],+ experimentPostRows: [],+ });+ expect(+ readJson(v4LegacyApiHourBucketKey(Date.parse(CURRENT_HOUR_ISO))),+ ).toMatchObject({+ apiRows: [],+ experimentPostRows: [+ {+ projectId: "project-b",+ count: 2,+ lastSeen: "2026-06-25T10:05:00.000000Z",+ },+ ],+ });++ // Consumer-facing entries: identical service sets deduplicated (count 4,+ // not 8); POST usage becomes an experiment entry, not an API row.+ expect(readJson(v4LegacyApiUsageProjectKey("project-a"))).toMatchObject({+ version: 1,+ rows: [+ {+ entrypoint: "publicapi: GET /api/public/traces",+ count: 4,+ lastSeen: "2026-06-25T09:15:00.000000Z",+ },+ ],+ });+ expect(readJson(v4LegacyApiUsageProjectKey("project-b"))).toBeNull();+ expect(+ readJson(v4ExperimentPostUsageProjectKey("project-b")),+ ).toMatchObject({+ used: true,+ lastSeen: "2026-06-25T10:05:00.000000Z",+ });++ expect(redisState.store.get(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(+ CURRENT_HOUR_ISO,+ );+ expect(redisState.store.get(V4_LEGACY_API_USAGE_HEARTBEAT_KEY)).toBe(+ TEST_NOW.toISOString(),+ );+ // Lock released after the run.+ expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);+ });++ it("incremental run: re-scans only the trailing margin and merges with existing buckets", async () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ "2026-06-25T09:00:00Z",+ );+ seedRecentDeepRescan();+ seedCoverageBuckets();+ // Existing bucket from an earlier run, outside the re-scan range: its+ // counts must survive and merge into the rollup.+ redisState.store.set(+ v4LegacyApiHourBucketKey(Date.parse("2026-06-20T00:00:00Z")),+ JSON.stringify({+ version: 1,+ computedAt: "2026-06-20T01:00:00.000Z",+ apiRows: [+ {+ projectId: "project-a",+ entrypoint: "publicapi: GET /api/public/traces",+ count: 2,+ lastSeen: "2026-06-20T00:30:00.000000Z",+ },+ ],+ experimentPostRows: [],+ }),+ );+ // Distinct result sets across services are summed per row key.+ mocks.queryClickhouse+ .mockResolvedValueOnce([usageRow({ count: "4" })])+ .mockResolvedValueOnce([+ usageRow({ count: "1", lastSeen: "2026-06-25T09:20:00.000000Z" }),+ ]);++ await handleV4LegacyApiUsageJob();++ // Cursor 09:00 minus the 3h margin.+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({+ fromTimestamp: "2026-06-25 06:00:00.000",+ });++ expect(readJson(v4LegacyApiUsageProjectKey("project-a"))).toMatchObject({+ rows: [+ {+ entrypoint: "publicapi: GET /api/public/traces",+ // 2 (old bucket) + 4 + 1 (distinct service results summed).+ count: 7,+ lastSeen: "2026-06-25T09:20:00.000000Z",+ },+ ],+ });+ expect(readJson(V4_LEGACY_API_ROLLUP_PROJECTS_KEY)).toEqual({+ version: 1,+ api: ["project-a"],+ experimentPost: [],+ });+ });++ it("runs the deep re-scan when none happened within the last 24 hours", async () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ "2026-06-25T09:00:00Z",+ );+ seedCoverageBuckets();+ // Last deep re-scan 25h ago: due again, regardless of wall-clock hour.+ redisState.store.set(+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,+ "2026-06-24T09:30:00.000Z",+ );++ await handleV4LegacyApiUsageJob();++ // Cursor 09:00 minus the 24h deep margin.+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({+ fromTimestamp: "2026-06-24 09:00:00.000",+ });+ expect(redisState.store.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY)).toBe(+ TEST_NOW.toISOString(),+ );+ });++ it("extends the scan to repair missing or corrupt buckets inside the window", async () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ "2026-06-25T09:00:00Z",+ );+ seedRecentDeepRescan();+ // One bucket is missing well before the cursor margin; without repair the+ // rollup would silently lose that hour while the heartbeat stays fresh.+ seedCoverageBuckets(["2026-06-21T05:00:00Z"]);++ await handleV4LegacyApiUsageJob();++ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({+ fromTimestamp: "2026-06-21 05:00:00.000",+ });+ expect(+ redisState.store.has(+ v4LegacyApiHourBucketKey(Date.parse("2026-06-21T05:00:00Z")),+ ),+ ).toBe(true);+ });++ it("deletes per-project entries for projects that dropped out of the window", async () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ "2026-06-25T09:00:00Z",+ );+ seedRecentDeepRescan();+ seedCoverageBuckets();+ // Entries and tracking state from a previous run whose usage has aged out.+ redisState.store.set(+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,+ JSON.stringify({+ version: 1,+ api: ["project-gone"],+ experimentPost: ["project-post-gone"],+ }),+ );+ redisState.store.set(+ v4LegacyApiUsageProjectKey("project-gone"),+ JSON.stringify({ version: 1, computedAt: "old", rows: [] }),+ );+ redisState.store.set(+ v4ExperimentPostUsageProjectKey("project-post-gone"),+ JSON.stringify({+ version: 1,+ computedAt: "old",+ used: true,+ lastSeen: "2026-06-10T00:00:00.000000Z",+ }),+ );++ await handleV4LegacyApiUsageJob();++ expect(+ redisState.store.has(v4LegacyApiUsageProjectKey("project-gone")),+ ).toBe(false);+ expect(+ redisState.store.has(+ v4ExperimentPostUsageProjectKey("project-post-gone"),+ ),+ ).toBe(false);+ expect(readJson(V4_LEGACY_API_ROLLUP_PROJECTS_KEY)).toEqual({+ version: 1,+ api: [],+ experimentPost: [],+ });+ });++ it("skips the run when another worker holds the lock", async () => {+ redisState.store.set(V4_LEGACY_API_USAGE_LOCK_KEY, "someone-else");++ await handleV4LegacyApiUsageJob();++ expect(mocks.queryClickhouse).not.toHaveBeenCalled();+ expect(redisState.store.has(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(false);+ });++ it("throws and leaves the cursor untouched when a service scan fails", async () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ "2026-06-25T09:00:00Z",+ );+ mocks.queryClickhouse+ .mockResolvedValueOnce([usageRow({})])+ .mockRejectedValueOnce(new Error("query_log unavailable"));++ await expect(handleV4LegacyApiUsageJob()).rejects.toThrow(+ "query_log unavailable",+ );++ expect(redisState.store.get(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(+ "2026-06-25T09:00:00Z",+ );+ expect(redisState.store.has(V4_LEGACY_API_USAGE_HEARTBEAT_KEY)).toBe(false);+ // Lock still released on failure.+ expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);+ });+});diff --git a/worker/src/features/v4/__tests__/v4LegacyApiUsageMetrics.test.ts b/worker/src/features/v4/__tests__/v4LegacyApiUsageMetrics.test.tsnew file mode 100644index 0000000000..b081d3e5f9--- /dev/null+++ b/worker/src/features/v4/__tests__/v4LegacyApiUsageMetrics.test.ts@@ -0,0 +1,60 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const redisState = vi.hoisted(() => ({+ store: new Map<string, string>(),+}));++const mocks = vi.hoisted(() => ({+ recordGauge: vi.fn(),+}));++vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const original =+ await importOriginal<typeof import("@langfuse/shared/src/server")>();+ return {+ ...original,+ redis: {+ get: vi.fn(async (key: string) => redisState.store.get(key) ?? null),+ },+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },+ recordGauge: mocks.recordGauge,+ };+});++import { V4_LEGACY_API_USAGE_HEARTBEAT_KEY } from "@langfuse/shared/src/server";+import {+ emitV4LegacyApiUsageFreshnessMetrics,+ V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC,+} from "../v4LegacyApiUsageMetrics";++describe("emitV4LegacyApiUsageFreshnessMetrics", () => {+ beforeEach(() => {+ redisState.store.clear();+ vi.clearAllMocks();+ });++ it("emits heartbeat age in seconds for a present heartbeat", async () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,+ "2026-06-25T10:00:00.000Z",+ );++ await emitV4LegacyApiUsageFreshnessMetrics(+ Date.parse("2026-06-25T10:30:00Z"),+ );++ expect(mocks.recordGauge).toHaveBeenCalledWith(+ V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC,+ 1800,+ { unit: "seconds" },+ );+ });++ it("emits nothing when the heartbeat is missing (no-data alert)", async () => {+ await emitV4LegacyApiUsageFreshnessMetrics(+ Date.parse("2026-06-25T10:30:00Z"),+ );++ expect(mocks.recordGauge).not.toHaveBeenCalled();+ });+});diff --git a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts b/worker/src/features/v4/handleV4LegacyApiUsageJob.tsnew file mode 100644index 0000000000..a5c0b8a14b--- /dev/null+++ b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts@@ -0,0 +1,568 @@+import {+ aggregateV4LegacyApiHourBuckets,+ convertDateToClickhouseDateTime,+ listV4LegacyApiHourStarts,+ logger,+ queryClickhouse,+ redis,+ systemTableRef,+ v4ExperimentPostUsageProjectKey,+ v4LegacyApiHourBucketKey,+ v4LegacyApiHourBucketSchema,+ v4LegacyApiHourStartIso,+ v4LegacyApiRollupProjectsSchema,+ v4LegacyApiUsageProjectKey,+ V4_LEGACY_API_DEEP_RESCAN_HOURS,+ V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ v4LegacyApiHourBucketTtlSeconds,+ V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,+ V4_LEGACY_API_RESCAN_HOURS,+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,+ V4_LEGACY_API_USAGE_LOCK_KEY,+ V4_LEGACY_API_USAGE_WINDOW_MS,+ type PreferredClickhouseService,+ type V4LegacyApiHourBucket,+} from "@langfuse/shared/src/server";+import { env } from "@langfuse/shared/src/env";+import { RedisLock } from "../../utils/RedisLock";++/**+ * Incremental producer for the v4 legacy API usage pipeline.+ *+ * Scans `system.query_log` for deprecated public API calls and experiment+ * `POST /api/public/dataset-run-items` calls, environment-wide (one scan+ * serves every project), and materializes the results in Redis so the v4+ * transition UI never has to run this scan from a web request. See+ * `@langfuse/shared/src/server` (`v4/legacyApiUsage`) for the data contracts.+ *+ * Steady state each hourly run scans only the trailing re-scan margin+ * (~3 hours; 24 hours once a day for late query_log flushes) instead of the+ * full detection-window scan. Buckets are written by full overwrite, so re-scans and+ * backfills after missed runs are idempotent. The full-window scan happens+ * exactly once, on cold start (no cursor).+ */++const HOUR_MS = 60 * 60 * 1000;++/**+ * Cold-start / deep-rescan query_log sweeps can take minutes; raise the+ * ClickHouse client request timeout (and derived max_execution_time) above+ * the shared 30s default.+ */+export const V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS = 10 * 60 * 1000;++const EXPERIMENT_POST_ROUTE = "POST /api/public/dataset-run-items";++type HourUsageRow = {+ hourStart: string;+ projectId: string;+ route: string;+ count: string | number;+ lastSeen: string;+};++type MergedHourUsageRow = {+ hourStart: string;+ projectId: string;+ route: string;+ count: number;+ lastSeen: string;+};++/**+ * `system.query_log` is local to each ClickHouse service, so every service+ * that terminates public API reads must be scanned.+ */+const getQueryLogServices = (): PreferredClickhouseService[] => {+ const seenUrls = new Set([new URL(env.CLICKHOUSE_URL).toString()]);+ const services: PreferredClickhouseService[] = ["ReadWrite"];+ const candidates: [PreferredClickhouseService, string | undefined][] = [+ ["ReadOnly", env.CLICKHOUSE_READ_ONLY_URL],+ ["EventsReadOnly", env.CLICKHOUSE_EVENTS_READ_ONLY_URL],+ ];+ for (const [service, url] of candidates) {+ if (!url) continue;+ const normalizedUrl = new URL(url).toString();+ if (seenUrls.has(normalizedUrl)) continue;+ seenUrls.add(normalizedUrl);+ services.push(service);+ }+ return services;+};++const queryHourUsageRowsForService = async ({+ fromTimestamp,+ toTimestamp,+ preferredClickhouseService,+}: {+ fromTimestamp: Date;+ toTimestamp: Date;+ preferredClickhouseService: PreferredClickhouseService;+}): Promise<HourUsageRow[]> =>+ queryClickhouse<HourUsageRow>({+ query: `+WITH selected AS (+ SELECT+ JSONExtractString(log_comment, 'projectId') AS project_id,+ event_time,+ event_time_microseconds,+ splitByChar('?', JSONExtractString(log_comment, 'route'))[1] AS route_path+ FROM ${systemTableRef("system.query_log")}+ WHERE+ event_time >= {fromTimestamp: DateTime64(3)}+ AND event_time <= {toTimestamp: DateTime64(3)}+ AND event_date >= toDate({fromTimestamp: DateTime64(3)})+ AND event_date <= toDate({toTimestamp: DateTime64(3)})+ AND type = 'QueryFinish'+ AND JSONExtractString(log_comment, 'tag_schema_version') = '1'+ AND JSONExtractString(log_comment, 'surface') = 'publicapi'+ AND JSONExtractString(log_comment, 'projectId') != ''+),+classified AS (+ SELECT+ project_id,+ event_time,+ event_time_microseconds,+ multiIf(+ route_path IN (+ 'GET /api/public/spans',+ 'GET /api/public/generations',+ 'GET /api/public/traces',+ 'GET /api/public/sessions',+ 'GET /api/public/observations',+ 'GET /api/public/scores',+ 'GET /api/public/v2/scores',+ 'GET /api/public/metrics',+ 'GET /api/public/metrics/daily',+ 'GET /api/public/dataset-run-items',+ 'POST /api/public/dataset-run-items'+ ), route_path,+ match(route_path, '^GET /api/public/traces/[^/?#]+$'), 'GET /api/public/traces/{id}',+ match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 'GET /api/public/sessions/{id}',+ match(route_path, '^GET /api/public/observations/[^/?#]+$'), 'GET /api/public/observations/{id}',+ match(route_path, '^GET /api/public/scores/[^/?#]+$'), 'GET /api/public/scores/{id}',+ match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 'GET /api/public/v2/scores/{id}',+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs',+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 'GET /api/public/datasets/{datasetName}/runs/{runName}',+ NULL+ ) AS legacy_route,+ multiIf(+ route_path IN (+ 'GET /api/public/spans',+ 'GET /api/public/generations',+ 'GET /api/public/traces',+ 'GET /api/public/observations',+ 'GET /api/public/scores',+ 'GET /api/public/v2/scores',+ 'GET /api/public/metrics/daily',+ 'GET /api/public/dataset-run-items'+ ), 2,+ route_path IN (+ 'GET /api/public/sessions',+ 'GET /api/public/metrics',+ 'POST /api/public/dataset-run-items'+ ), 1,+ match(route_path, '^GET /api/public/traces/[^/?#]+$'), 3,+ match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 1,+ match(route_path, '^GET /api/public/observations/[^/?#]+$'), 1,+ match(route_path, '^GET /api/public/scores/[^/?#]+$'), 1,+ match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 1,+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 1,+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1,+ NULL+ ) AS clickhouse_queries_per_api_call+ FROM selected+)+SELECT+ formatDateTime(toStartOfHour(event_time, 'UTC'), '%Y-%m-%dT%H:%i:%SZ', 'UTC') AS hourStart,+ project_id AS projectId,+ legacy_route AS route,+ sum(1.0 / clickhouse_queries_per_api_call) AS count,+ formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen+FROM classified+WHERE legacy_route IS NOT NULL+ AND clickhouse_queries_per_api_call IS NOT NULL+GROUP BY hourStart, projectId, route+ORDER BY hourStart ASC, projectId ASC, route ASC+SETTINGS skip_unavailable_shards = 1+ `,+ params: {+ fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),+ toTimestamp: convertDateToClickhouseDateTime(toTimestamp),+ },+ tags: { route: "v4-legacy-api-usage-pipeline" },+ preferredClickhouseService,+ // Shared client derives max_execution_time = request_timeout/1000 + 5s+ // and enables HTTP progress headers when request_timeout > 30s default.+ clickhouseConfigs: {+ request_timeout: V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS,+ },+ clickhouseSettings: { skip_unavailable_shards: 1 },+ });++/**+ * Merges per-service results. Services pointing at overlapping replicas can+ * return identical result sets; those are deduplicated wholesale before+ * counts are summed, mirroring the request-path fallback behavior.+ */+const mergeServiceRows = (+ serviceRowSets: HourUsageRow[][],+): MergedHourUsageRow[] => {+ const uniqueServiceResultSets = new Map<string, HourUsageRow[]>();+ for (const rows of serviceRowSets) {+ const sortedRows = [...rows].sort(+ (left, right) =>+ left.hourStart.localeCompare(right.hourStart) ||+ left.projectId.localeCompare(right.projectId) ||+ left.route.localeCompare(right.route),+ );+ uniqueServiceResultSets.set(JSON.stringify(sortedRows), sortedRows);+ }++ const merged = new Map<string, MergedHourUsageRow>();+ for (const rows of uniqueServiceResultSets.values()) {+ for (const row of rows) {+ const key = [row.hourStart, row.projectId, row.route].join("\u0000");+ const existing = merged.get(key);+ merged.set(+ key,+ existing+ ? {+ ...existing,+ count: existing.count + Number(row.count),+ lastSeen:+ existing.lastSeen > row.lastSeen+ ? existing.lastSeen+ : row.lastSeen,+ }+ : {+ hourStart: row.hourStart,+ projectId: row.projectId,+ route: row.route,+ count: Number(row.count),+ lastSeen: row.lastSeen,+ },+ );+ }+ }+ return Array.from(merged.values());+};++const buildHourBuckets = ({+ scannedHourStartsMs,+ mergedRows,+ now,+}: {+ scannedHourStartsMs: number[];+ mergedRows: MergedHourUsageRow[];+ now: Date;+}): Map<number, V4LegacyApiHourBucket> => {+ // Every scanned hour gets a bucket, empty ones included: buckets are+ // written by full overwrite so repeated scans stay idempotent.+ const buckets = new Map<number, V4LegacyApiHourBucket>(+ scannedHourStartsMs.map((hourStartMs) => [+ hourStartMs,+ {+ version: 1,+ computedAt: now.toISOString(),+ apiRows: [],+ experimentPostRows: [],+ },+ ]),+ );+ for (const row of mergedRows) {+ const hourStartMs = Date.parse(row.hourStart);+ const bucket = buckets.get(hourStartMs);+ if (!bucket) {+ logger.warn("v4 legacy API usage row outside scanned hours", {+ hourStart: row.hourStart,+ });+ continue;+ }+ if (row.route === EXPERIMENT_POST_ROUTE) {+ bucket.experimentPostRows.push({+ projectId: row.projectId,+ count: row.count,+ lastSeen: row.lastSeen,+ });+ } else {+ bucket.apiRows.push({+ projectId: row.projectId,+ entrypoint: `publicapi: ${row.route}`,+ count: row.count,+ lastSeen: row.lastSeen,+ });+ }+ }+ return buckets;+};++const CHUNK_SIZE = 500;++const setexInChunks = async (+ entries: { key: string; ttlSeconds: number; value: string }[],+): Promise<void> => {+ for (let index = 0; index < entries.length; index += CHUNK_SIZE) {+ await Promise.all(+ entries+ .slice(index, index + CHUNK_SIZE)+ .map((entry) => redis!.setex(entry.key, entry.ttlSeconds, entry.value)),+ );+ }+};++const mgetInChunks = async (keys: string[]): Promise<(string | null)[]> => {+ const values: (string | null)[] = [];+ for (let index = 0; index < keys.length; index += CHUNK_SIZE) {+ values.push(...(await redis!.mget(keys.slice(index, index + CHUNK_SIZE))));+ }+ return values;+};++const delInChunks = async (keys: string[]): Promise<void> => {+ for (let index = 0; index < keys.length; index += CHUNK_SIZE) {+ await redis!.del(...keys.slice(index, index + CHUNK_SIZE));+ }+};++const parseHourBucket = (+ rawBucket: string | null,+): V4LegacyApiHourBucket | null => {+ if (!rawBucket) return null;+ try {+ const parsed = v4LegacyApiHourBucketSchema.safeParse(JSON.parse(rawBucket));+ return parsed.success ? parsed.data : null;+ } catch {+ return null;+ }+};++export const handleV4LegacyApiUsageJob = async (+ now = new Date(),+): Promise<void> => {+ if (!redis) {+ logger.warn(+ "Skipping v4 legacy API usage job: Redis is not available to store results",+ );+ return;+ }++ const lock = new RedisLock(V4_LEGACY_API_USAGE_LOCK_KEY, {+ // Generous TTL: a cold-start scan can take a while, and an expired lock+ // would let a second replica interleave writes and publish a torn rollup.+ ttlSeconds: 60 * 60,+ name: "v4-legacy-api-usage",+ // Without Redis there is nowhere to write results, so never run unlocked.+ onUnavailable: "fail",+ });++ const result = await lock.withLock(async () => {+ const nowMs = now.getTime();+ const currentHourStartMs = Math.floor(nowMs / HOUR_MS) * HOUR_MS;+ const coverageStartMs =+ Math.floor((nowMs - V4_LEGACY_API_USAGE_WINDOW_MS) / HOUR_MS) * HOUR_MS;+ const coverageHourStartsMs = listV4LegacyApiHourStarts(+ coverageStartMs,+ currentHourStartMs,+ );++ const [cursorIso, deepRescanAtIso, rawPreviousRollupProjects] =+ await Promise.all([+ redis!.get(V4_LEGACY_API_USAGE_CURSOR_KEY),+ redis!.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY),+ redis!.get(V4_LEGACY_API_ROLLUP_PROJECTS_KEY),+ ]);+ const cursorMs = cursorIso ? Date.parse(cursorIso) : NaN;+ const deepRescanAtMs = deepRescanAtIso ? Date.parse(deepRescanAtIso) : NaN;+ // The deep re-scan is driven by elapsed time since the last one (not a+ // wall-clock hour), so a missed run only delays it instead of skipping a+ // whole day.+ const deepRescanDue =+ !Number.isFinite(deepRescanAtMs) ||+ nowMs - deepRescanAtMs >= V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS;+ const rescanMs =+ (deepRescanDue+ ? V4_LEGACY_API_DEEP_RESCAN_HOURS+ : V4_LEGACY_API_RESCAN_HOURS) * HOUR_MS;+ // Resume from the cursor (self-healing backfill after missed runs) minus+ // the re-scan margin for late query_log flushes; never beyond the window.+ const cursorScanStartMs = Number.isFinite(cursorMs)+ ? Math.max(+ coverageStartMs,+ Math.min(cursorMs, currentHourStartMs) - rescanMs,+ )+ : coverageStartMs;++ // Bucket holes (evicted or unparsable entries older than the scan start)+ // would otherwise silently drop usage from the rollup while the fresh+ // heartbeat blocks the web fallback; extend the scan to repair them.+ const existingBucketsByHourMs = new Map<+ number,+ V4LegacyApiHourBucket | null+ >();+ const rawExistingBuckets = await mgetInChunks(+ coverageHourStartsMs.map(v4LegacyApiHourBucketKey),+ );+ coverageHourStartsMs.forEach((hourStartMs, index) => {+ existingBucketsByHourMs.set(+ hourStartMs,+ parseHourBucket(rawExistingBuckets[index] ?? null),+ );+ });+ const oldestHoleMs = coverageHourStartsMs.find(+ (hourStartMs) =>+ hourStartMs < cursorScanStartMs &&+ existingBucketsByHourMs.get(hourStartMs) == null,+ );+ const scanStartMs = oldestHoleMs ?? cursorScanStartMs;++ const scannedHourStartsMs = listV4LegacyApiHourStarts(+ scanStartMs,+ currentHourStartMs,+ );+ logger.info("Running v4 legacy API usage job", {+ scanStart: v4LegacyApiHourStartIso(scanStartMs),+ hours: scannedHourStartsMs.length,+ coldStart: !Number.isFinite(cursorMs),+ deepRescan: deepRescanDue,+ repairedHole: oldestHoleMs !== undefined,+ });++ // All services must succeed; on failure the job throws, the cursor stays+ // put, and the next run backfills the same hours.+ const serviceRowSets = await Promise.all(+ getQueryLogServices().map((preferredClickhouseService) =>+ queryHourUsageRowsForService({+ fromTimestamp: new Date(scanStartMs),+ toTimestamp: now,+ preferredClickhouseService,+ }),+ ),+ );+ const buckets = buildHourBuckets({+ scannedHourStartsMs,+ mergedRows: mergeServiceRows(serviceRowSets),+ now,+ });++ await setexInChunks(+ Array.from(buckets.entries()).map(([hourStartMs, bucket]) => ({+ key: v4LegacyApiHourBucketKey(hourStartMs),+ // Age-based TTL: backfilled/missing hours expire with the window,+ // not a flat 15d from this write.+ ttlSeconds: v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs),+ value: JSON.stringify(bucket),+ })),+ );++ // Rebuild the consumer-facing per-project entries from every bucket in+ // the trailing window: freshly scanned hours plus the pre-read existing+ // buckets outside the scan range (hole repair above guarantees each+ // coverage hour is one of the two).+ const rollup = aggregateV4LegacyApiHourBuckets(+ coverageHourStartsMs.flatMap((hourStartMs) => {+ const bucket =+ buckets.get(hourStartMs) ?? existingBucketsByHourMs.get(hourStartMs);+ return bucket ? [bucket] : [];+ }),+ );++ const computedAt = now.toISOString();+ await setexInChunks([+ ...Array.from(rollup.apiRowsByProjectId.entries()).map(+ ([projectId, rows]) => ({+ key: v4LegacyApiUsageProjectKey(projectId),+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,+ value: JSON.stringify({ version: 1, computedAt, rows }),+ }),+ ),+ ...Array.from(rollup.experimentPostLastSeenByProjectId.entries()).map(+ ([projectId, lastSeen]) => ({+ key: v4ExperimentPostUsageProjectKey(projectId),+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,+ value: JSON.stringify({+ version: 1,+ computedAt,+ used: true,+ lastSeen,+ }),+ }),+ ),+ ]);++ // Delete entries for projects that dropped out of the trailing window+ // since the previous run; otherwise their last written entry would keep+ // reporting usage for up to the entry TTL.+ const previousRollupProjects = (() => {+ try {+ const parsed = v4LegacyApiRollupProjectsSchema.safeParse(+ JSON.parse(rawPreviousRollupProjects ?? "null"),+ );+ return parsed.success ? parsed.data : null;+ } catch {+ return null;+ }+ })();+ await delInChunks([+ ...(previousRollupProjects?.api ?? [])+ .filter((projectId) => !rollup.apiRowsByProjectId.has(projectId))+ .map(v4LegacyApiUsageProjectKey),+ ...(previousRollupProjects?.experimentPost ?? [])+ .filter(+ (projectId) =>+ !rollup.experimentPostLastSeenByProjectId.has(projectId),+ )+ .map(v4ExperimentPostUsageProjectKey),+ ]);+ await redis!.setex(+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ JSON.stringify({+ version: 1,+ api: Array.from(rollup.apiRowsByProjectId.keys()),+ experimentPost: Array.from(+ rollup.experimentPostLastSeenByProjectId.keys(),+ ),+ }),+ );++ // Cursor and heartbeat move only after a fully successful run. While the+ // heartbeat is fresh, web treats missing per-project entries as "no+ // usage" instead of falling back to ClickHouse.+ await redis!.setex(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ v4LegacyApiHourStartIso(currentHourStartMs),+ );+ if (deepRescanDue) {+ await redis!.setex(+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ computedAt,+ );+ }+ await redis!.setex(+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ computedAt,+ );++ logger.info("Completed v4 legacy API usage job", {+ scannedHours: scannedHourStartsMs.length,+ projectsWithLegacyApiUsage: rollup.apiRowsByProjectId.size,+ projectsWithExperimentPostUsage:+ rollup.experimentPostLastSeenByProjectId.size,+ });+ return "completed" as const;+ });++ if (result === null) {+ logger.info(+ "Skipping v4 legacy API usage job: another worker holds the lock",+ );+ }+};diff --git a/worker/src/features/v4/v4LegacyApiUsageMetrics.ts b/worker/src/features/v4/v4LegacyApiUsageMetrics.tsnew file mode 100644index 0000000000..331a9c19e5--- /dev/null+++ b/worker/src/features/v4/v4LegacyApiUsageMetrics.ts@@ -0,0 +1,44 @@+import {+ logger,+ recordGauge,+ redis,+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,+} from "@langfuse/shared/src/server";++/**+ * Freshness metric for the v4 legacy API usage pipeline.+ *+ * A single gauge is enough to alert on staleness: the worker only refreshes+ * the heartbeat after a fully successful run, so a stuck, failing, or missing+ * job lets `heartbeat_age_seconds` climb past the freshness window. Emitted+ * from QueueMetricsRunner so age keeps rising between the hourly job runs.+ *+ * Alert: `avg:langfuse.v4.legacy_api_usage.heartbeat_age_seconds > 10800` (3h).+ */+export const V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC =+ "langfuse.v4.legacy_api_usage.heartbeat_age_seconds";++export const emitV4LegacyApiUsageFreshnessMetrics = async (+ nowMs = Date.now(),+): Promise<void> => {+ if (!redis) return;++ const heartbeatIso = await redis.get(V4_LEGACY_API_USAGE_HEARTBEAT_KEY);+ const heartbeatMs = heartbeatIso ? Date.parse(heartbeatIso) : NaN;+ // Missing/unparsable heartbeat (never ran) surfaces as a no-data alert; the+ // key's 15d TTL means a running pipeline always has a value to age off.+ if (!Number.isFinite(heartbeatMs)) return;++ try {+ recordGauge(+ V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC,+ Math.max(0, Math.floor((nowMs - heartbeatMs) / 1000)),+ { unit: "seconds" },+ );+ } catch (error) {+ logger.error(+ "v4 legacy API usage: failed to emit heartbeat freshness metric",+ error,+ );+ }+};diff --git a/worker/src/queues/v4LegacyApiUsageQueue.ts b/worker/src/queues/v4LegacyApiUsageQueue.tsnew file mode 100644index 0000000000..c88d5766d8--- /dev/null+++ b/worker/src/queues/v4LegacyApiUsageQueue.ts@@ -0,0 +1,15 @@+import { Processor } from "bullmq";+import { logger, QueueJobs } from "@langfuse/shared/src/server";+import { handleV4LegacyApiUsageJob } from "../features/v4/handleV4LegacyApiUsageJob";++export const v4LegacyApiUsageProcessor: Processor = async (job) => {+ if (job.name === QueueJobs.V4LegacyApiUsageJob) {+ logger.info("Executing V4 Legacy API Usage Job");+ try {+ return await handleV4LegacyApiUsageJob();+ } catch (error) {+ logger.error("Error executing V4LegacyApiUsageJob", error);+ throw error;+ }+ }+};