Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): maintain v4 legacy API usage in Redis via incremental worker job (#16164)

Max Deichmannae901b330f3e
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index ebca27edee..b3308bb988 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -104,6 +104,7 @@ export * from "./redis/batchExport";
export * from "./redis/batchActionQueue";
export * from "./redis/ingestionQueue";
export * from "./redis/postHogIntegrationQueue";
+export * from "./redis/v4LegacyApiUsageQueue";
export * from "./redis/postHogIntegrationProcessingQueue";
export * from "./redis/mixpanelIntegrationQueue";
export * from "./redis/mixpanelIntegrationProcessingQueue";
@@ -175,6 +176,7 @@ export * from "./utils/formatAuthProvider";
export * from "./traceDeletionProcessor";
export * from "./deletionGuard";
export * from "./forceV3Experience";
+export * from "./v4/legacyApiUsage";
export * from "./analytics-integrations/types";
// Re-annotate these deprecated routing wrappers at the public server barrel.
diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.ts
index f30d34377e..9ace762f48 100644
--- a/packages/shared/src/server/queues.ts
+++ b/packages/shared/src/server/queues.ts
@@ -415,6 +415,7 @@ export enum QueueName {
NotificationQueue = "notification-queue",
MonitorQueue = "monitor-queue",
InAppAgentRunQueue = "in-app-agent-run-queue",
+ V4LegacyApiUsageQueue = "v4-legacy-api-usage-queue",
}
export enum QueueJobs {
@@ -453,6 +454,7 @@ export enum QueueJobs {
NotificationJob = "notification-job",
MonitorJob = "monitor-job",
InAppAgentRunJob = "in-app-agent-run-job",
+ V4LegacyApiUsageJob = "v4-legacy-api-usage-job",
}
export type TQueueJobTypes = {
diff --git a/packages/shared/src/server/redis/getQueue.ts b/packages/shared/src/server/redis/getQueue.ts
index c964be2eb7..4e65c0bfa0 100644
--- a/packages/shared/src/server/redis/getQueue.ts
+++ b/packages/shared/src/server/redis/getQueue.ts
@@ -29,6 +29,7 @@ import { EventPropagationQueue } from "./eventPropagationQueue";
import { NotificationQueue } from "./notificationQueue";
import { MonitorQueue } from "./monitorQueue";
import { InAppAgentRunQueue } from "./inAppAgentRunQueue";
+import { V4LegacyApiUsageQueue } from "./v4LegacyApiUsageQueue";
// Sharded queues require a sharding key.
// Use the queue class directly, for example IngestionQueue.getInstance({ shardingKey }).
@@ -105,6 +106,8 @@ export function getQueue(
return MonitorQueue.getInstance();
case QueueName.InAppAgentRunQueue:
return InAppAgentRunQueue.getInstance();
+ case QueueName.V4LegacyApiUsageQueue:
+ return V4LegacyApiUsageQueue.getInstance();
default: {
const _exhaustiveCheckDefault: never = queueName;
throw new Error(`Queue ${queueName} not found`);
diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts
new file mode 100644
index 0000000000..de43592d6d
--- /dev/null
+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts
@@ -0,0 +1,58 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createBullMQQueueOptionsWithRedis } from "./redis";
+import { logger } from "../logger";
+
+// Hourly, offset from other scheduled jobs (:5 metering, :30 posthog, :35
+// free-tier). The worker re-scans a trailing margin each run, so the exact
+// minute only affects freshness, not correctness.
+export const V4_LEGACY_API_USAGE_CRON_PATTERN = "25 * * * *";
+
+export class V4LegacyApiUsageQueue {
+ private static instance: Queue | null = null;
+
+ public static getInstance(): Queue | null {
+ if (V4LegacyApiUsageQueue.instance) {
+ return V4LegacyApiUsageQueue.instance;
+ }
+
+ const queueOptionsWithRedis = createBullMQQueueOptionsWithRedis(
+ QueueName.V4LegacyApiUsageQueue,
+ );
+ V4LegacyApiUsageQueue.instance = queueOptionsWithRedis
+ ? new Queue(QueueName.V4LegacyApiUsageQueue, {
+ ...queueOptionsWithRedis,
+ defaultJobOptions: {
+ removeOnComplete: true,
+ removeOnFail: 100,
+ attempts: 5,
+ backoff: {
+ type: "exponential",
+ delay: 30_000,
+ },
+ },
+ })
+ : null;
+
+ V4LegacyApiUsageQueue.instance?.on("error", (err) => {
+ logger.error("V4LegacyApiUsageQueue error", err);
+ });
+
+ if (V4LegacyApiUsageQueue.instance) {
+ logger.debug("Scheduling jobs for V4LegacyApiUsageQueue");
+ V4LegacyApiUsageQueue.instance
+ .add(
+ QueueJobs.V4LegacyApiUsageJob,
+ {},
+ {
+ repeat: { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },
+ },
+ )
+ .catch((err) => {
+ logger.error("Error adding V4LegacyApiUsageJob schedule", err);
+ });
+ }
+
+ return V4LegacyApiUsageQueue.instance;
+ }
+}
diff --git a/packages/shared/src/server/v4/legacyApiUsage.test.ts b/packages/shared/src/server/v4/legacyApiUsage.test.ts
new file mode 100644
index 0000000000..7a5d40a879
--- /dev/null
+++ b/packages/shared/src/server/v4/legacyApiUsage.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+import {
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ v4LegacyApiHourBucketTtlSeconds,
+} from "./legacyApiUsage";
+
+describe("v4LegacyApiHourBucketTtlSeconds", () => {
+ const HOUR_MS = 60 * 60 * 1000;
+ const nowMs = Date.parse("2026-06-25T00:30:00.000Z");
+
+ it("gives nearly the full GC horizon for the current hour", () => {
+ const hourStartMs = Date.parse("2026-06-25T00:00:00.000Z");
+ const ttl = v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs);
+ expect(ttl).toBe(
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS -
+ Math.floor((nowMs - hourStartMs) / 1000),
+ );
+ });
+
+ it("shortens TTL for hole-repaired older hours so they expire with the window", () => {
+ const tenDaysAgo = nowMs - 10 * 24 * HOUR_MS;
+ const hourStartMs = Math.floor(tenDaysAgo / HOUR_MS) * HOUR_MS;
+ const ttl = v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs);
+ // ~5 days remaining of the 15d horizon
+ expect(ttl).toBeGreaterThan(4 * 24 * 60 * 60);
+ expect(ttl).toBeLessThan(6 * 24 * 60 * 60);
+ });
+
+ it("clamps to at least 1 second when the hour is already past the horizon", () => {
+ const hourStartMs = nowMs - 20 * 24 * HOUR_MS;
+ expect(v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs)).toBe(1);
+ });
+});
diff --git a/packages/shared/src/server/v4/legacyApiUsage.ts b/packages/shared/src/server/v4/legacyApiUsage.ts
new file mode 100644
index 0000000000..767e6ffc99
--- /dev/null
+++ b/packages/shared/src/server/v4/legacyApiUsage.ts
@@ -0,0 +1,286 @@
+import { z } from "zod/v4";
+
+/**
+ * Contracts for the v4 legacy API usage pipeline.
+ *
+ * Deprecated public API usage (and experiment `POST /api/public/dataset-run-items`
+ * usage) is derived from `system.query_log`, which is expensive to scan:
+ * filters live in unindexed JSON `log_comment` fields and the log is local to
+ * every ClickHouse service, so each check fans out across services and
+ * replicas. Instead of running that scan from web requests, a worker job
+ * maintains the data incrementally in Redis:
+ *
+ * - Hour buckets (`hourBucketKey`): environment-wide usage rows per hour,
+ * written by full overwrite so re-scans and backfills are idempotent. TTL is
+ * remaining life until `hourStart + GC horizon` (not a flat clock from write
+ * time), so hole-repaired older hours expire with the sliding window. The
+ * worker never deletes buckets; Redis expiry is the GC.
+ * - Cursor (`V4_LEGACY_API_USAGE_CURSOR_KEY`): the last hour the worker
+ * materialized. Missed runs backfill from here, so gaps self-heal.
+ * - Per-project usage entries (`legacyApiUsageProjectKey`,
+ * `experimentPostUsageProjectKey`, 12h TTL): the aggregated trailing-window
+ * answer web requests read. Refreshed every worker run, so they only expire
+ * when the pipeline is down. Projects without usage get no entry.
+ * - Heartbeat (`V4_LEGACY_API_USAGE_HEARTBEAT_KEY`): proof the pipeline ran
+ * recently. While fresh, a missing per-project entry authoritatively means
+ * "no usage" and web must not fall back to querying ClickHouse.
+ *
+ * Worker Datadog freshness gauge
+ * `langfuse.v4.legacy_api_usage.heartbeat_age_seconds` exposes staleness for
+ * alerting (alert when it exceeds `V4_LEGACY_API_HEARTBEAT_FRESHNESS_MS`, 3h).
+ *
+ * This module is pure (keys, schemas, constants, aggregation); Redis and
+ * ClickHouse I/O live with the producer (worker) and consumer (web).
+ */
+
+const HOUR_MS = 60 * 60 * 1000;
+
+/** Detection window served to the UI; must match the web-side window. */
+export const V4_LEGACY_API_USAGE_WINDOW_MS = 14 * 24 * HOUR_MS;
+
+/** GC horizon for hour buckets: window plus one day of slack. */
+export const V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS = 15 * 24 * 60 * 60;
+
+/**
+ * Redis TTL for an hour bucket keyed by that hour's start, not by write time.
+ * A hole-repaired bucket from 10 days ago must expire in ~5 days (horizon
+ * minus age), not live another full 15 days from the repair write.
+ */
+export const v4LegacyApiHourBucketTtlSeconds = (
+ hourStartMs: number,
+ nowMs: number,
+): number => {
+ const expireAtMs = hourStartMs + V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS * 1000;
+ return Math.max(1, Math.floor((expireAtMs - nowMs) / 1000));
+};
+
+/**
+ * Per-project entries outlive several missed worker runs; the worker
+ * refreshes them hourly, so they only expire when the pipeline is down and
+ * web falls back to its request-path cache behavior.
+ */
+export const V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS = 12 * 60 * 60;
+
+/**
+ * Heartbeats older than this stop being authoritative: absence of a
+ * per-project entry then no longer means "no usage".
+ */
+export const V4_LEGACY_API_HEARTBEAT_FRESHNESS_MS = 3 * HOUR_MS;
+
+/** Trailing hours re-scanned every run to absorb late query_log flushes. */
+export const V4_LEGACY_API_RESCAN_HOURS = 3;
+/** Deeper re-scan for stragglers beyond the hourly margin. */
+export const V4_LEGACY_API_DEEP_RESCAN_HOURS = 24;
+/**
+ * Minimum spacing between deep re-scans. Tracked via a persisted timestamp
+ * (not a wall-clock hour) so a missed run only delays the deep sweep instead
+ * of skipping it for a day.
+ */
+export const V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS = 24 * HOUR_MS;
+
+export const V4_LEGACY_API_USAGE_CURSOR_KEY =
+ "langfuse:v4:legacy-api-usage:cursor:v1";
+export const V4_LEGACY_API_USAGE_HEARTBEAT_KEY =
+ "langfuse:v4:legacy-api-usage:heartbeat:v1";
+export const V4_LEGACY_API_USAGE_LOCK_KEY =
+ "langfuse:v4:legacy-api-usage:lock:v1";
+export const V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY =
+ "langfuse:v4:legacy-api-usage:deep-rescan-at:v1";
+/**
+ * Project IDs the worker wrote per-project entries for in its last run; the
+ * next run deletes entries for projects that dropped out of the window so
+ * stale usage does not linger for the entry TTL.
+ */
+export const V4_LEGACY_API_ROLLUP_PROJECTS_KEY =
+ "langfuse:v4:legacy-api-usage:rollup-projects:v1";
+
+/** Hour-aligned ISO without milliseconds, e.g. "2026-06-25T13:00:00Z". */
+export const v4LegacyApiHourStartIso = (hourStartMs: number): string =>
+ new Date(hourStartMs).toISOString().replace(".000Z", "Z");
+
+// Hour buckets stay v1 across window changes: they hold per-hour raw facts
+// that are independent of the window length. Widening the window simply
+// leaves older hours missing, which the worker's hole repair backfills on
+// its next run.
+export const v4LegacyApiHourBucketKey = (hourStartMs: number): string =>
+ `langfuse:v4:legacy-api-usage:hour:v1:${v4LegacyApiHourStartIso(hourStartMs)}`;
+
+// v1: per-project entries. Age trimming and worker rollup deletion handle
+// semantic invalidation when the detection window or aggregation changes.
+export const v4LegacyApiUsageProjectKey = (projectId: string): string =>
+ `langfuse:v4:legacy-api-usage:v1:${projectId}`;
+
+export const v4ExperimentPostUsageProjectKey = (projectId: string): string =>
+ `langfuse:v4:experiment-post-usage:v1:${projectId}`;
+
+const legacyApiUsageRowSchema = z.object({
+ entrypoint: z.string(),
+ count: z.number(),
+ lastSeen: z.string(),
+});
+
+export type V4LegacyApiUsageRow = z.infer<typeof legacyApiUsageRowSchema>;
+
+export const v4LegacyApiHourBucketSchema = z.object({
+ version: z.literal(1),
+ computedAt: z.string(),
+ apiRows: z.array(
+ legacyApiUsageRowSchema.extend({
+ projectId: z.string(),
+ }),
+ ),
+ experimentPostRows: z.array(
+ z.object({
+ projectId: z.string(),
+ count: z.number(),
+ // Optional so pre-lastSeen hour buckets keep parsing; new writes always set it.
+ lastSeen: z.string().optional(),
+ }),
+ ),
+});
+
+export type V4LegacyApiHourBucket = z.infer<typeof v4LegacyApiHourBucketSchema>;
+
+/** Per-project entry consumed by web (`legacyApiUsageSummary`). */
+export const v4LegacyApiUsageBlobSchema = z.object({
+ version: z.literal(1),
+ computedAt: z.string(),
+ rows: z.array(legacyApiUsageRowSchema),
+});
+
+export type V4LegacyApiUsageBlob = z.infer<typeof v4LegacyApiUsageBlobSchema>;
+
+/** Per-project entry consumed by web (experiment instrumentation check). */
+export const v4ExperimentPostUsageBlobSchema = z.object({
+ version: z.literal(1),
+ computedAt: z.string(),
+ used: z.boolean(),
+ /**
+ * Latest POST /api/public/dataset-run-items in the window; null when unused.
+ * Default keeps older worker blobs readable until the next refresh.
+ */
+ lastSeen: z.string().nullable().default(null),
+});
+
+export type V4ExperimentPostUsageBlob = z.infer<
+ typeof v4ExperimentPostUsageBlobSchema
+>;
+
+export const v4LegacyApiRollupProjectsSchema = z.object({
+ version: z.literal(1),
+ api: z.array(z.string()),
+ experimentPost: z.array(z.string()),
+});
+
+export type V4LegacyApiRollupProjects = z.infer<
+ typeof v4LegacyApiRollupProjectsSchema
+>;
+
+export const isV4LegacyApiHeartbeatFresh = (
+ heartbeatIso: string | null | undefined,
+ nowMs: number,
+): boolean => {
+ if (!heartbeatIso) return false;
+ const heartbeatMs = Date.parse(heartbeatIso);
+ return (
+ Number.isFinite(heartbeatMs) &&
+ nowMs - heartbeatMs <= V4_LEGACY_API_HEARTBEAT_FRESHNESS_MS
+ );
+};
+
+/** Hour starts in `[fromMs, toMs]`, inclusive, aligned to full hours. */
+export const listV4LegacyApiHourStarts = (
+ fromMs: number,
+ toMs: number,
+): number[] => {
+ const hourStarts: number[] = [];
+ for (
+ let hourStartMs = Math.floor(fromMs / HOUR_MS) * HOUR_MS;
+ hourStartMs <= toMs;
+ hourStartMs += HOUR_MS
+ ) {
+ hourStarts.push(hourStartMs);
+ }
+ return hourStarts;
+};
+
+export type V4LegacyApiUsageRollup = {
+ apiRowsByProjectId: Map<string, V4LegacyApiUsageRow[]>;
+ /**
+ * projectId → latest experiment POST lastSeen within the aggregated buckets.
+ * `null` means usage was observed in a legacy hour bucket that lacked lastSeen.
+ */
+ experimentPostLastSeenByProjectId: Map<string, string | null>;
+};
+
+/**
+ * Aggregates hour buckets into the per-project trailing-window answer:
+ * counts add per entrypoint, `lastSeen` takes the maximum. Buckets carry raw
+ * facts only, so aggregation is associative and order-independent.
+ */
+export const aggregateV4LegacyApiHourBuckets = (
+ buckets: V4LegacyApiHourBucket[],
+): V4LegacyApiUsageRollup => {
+ const rowsByProjectAndEntrypoint = new Map<
+ string,
+ Map<string, V4LegacyApiUsageRow>
+ >();
+ const experimentPostLastSeenByProjectId = new Map<string, string | null>();
+
+ for (const bucket of buckets) {
+ for (const row of bucket.apiRows) {
+ const rowsByEntrypoint =
+ rowsByProjectAndEntrypoint.get(row.projectId) ??
+ new Map<string, V4LegacyApiUsageRow>();
+ const existing = rowsByEntrypoint.get(row.entrypoint);
+ rowsByEntrypoint.set(
+ row.entrypoint,
+ existing
+ ? {
+ entrypoint: row.entrypoint,
+ count: existing.count + row.count,
+ lastSeen:
+ existing.lastSeen > row.lastSeen
+ ? existing.lastSeen
+ : row.lastSeen,
+ }
+ : {
+ entrypoint: row.entrypoint,
+ count: row.count,
+ lastSeen: row.lastSeen,
+ },
+ );
+ rowsByProjectAndEntrypoint.set(row.projectId, rowsByEntrypoint);
+ }
+ for (const row of bucket.experimentPostRows) {
+ if (row.count <= 0) continue;
+ const existingLastSeen = experimentPostLastSeenByProjectId.get(
+ row.projectId,
+ );
+ if (!row.lastSeen) {
+ if (existingLastSeen === undefined) {
+ experimentPostLastSeenByProjectId.set(row.projectId, null);
+ }
+ continue;
+ }
+ experimentPostLastSeenByProjectId.set(
+ row.projectId,
+ existingLastSeen && existingLastSeen > row.lastSeen
+ ? existingLastSeen
+ : row.lastSeen,
+ );
+ }
+ }
+
+ const apiRowsByProjectId = new Map<string, V4LegacyApiUsageRow[]>();
+ for (const [projectId, rowsByEntrypoint] of rowsByProjectAndEntrypoint) {
+ apiRowsByProjectId.set(
+ projectId,
+ Array.from(rowsByEntrypoint.values()).sort((left, right) =>
+ left.entrypoint.localeCompare(right.entrypoint),
+ ),
+ );
+ }
+
+ return { apiRowsByProjectId, experimentPostLastSeenByProjectId };
+};
diff --git a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
index 83a3b80195..c5ba599a01 100644
--- a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
+++ b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
@@ -9,17 +9,88 @@ vi.mock("@/src/server/auth", () => ({
}));
/**
- * Mutable Redis stub shared with the mocked "@langfuse/shared/src/server"
- * barrel. Defaults to status "end" (cache unavailable, direct-query mode);
- * cache tests flip status to "ready" and stub mget/setex.
+ * In-memory Redis stand-in for cache tests. Avoids a real localhost:6379
+ * dependency so the redis-cluster CI matrix (no standalone Redis) stays green.
+ * Most tests keep `available=false` to exercise the direct-query path.
*/
-const redisMock = vi.hoisted(() => ({
- status: "end" as string,
- disconnect: vi.fn(),
- mget: vi.fn(),
- setex: vi.fn(),
+const redisAvailability = vi.hoisted(() => ({
+ available: false,
}));
+const memoryRedis = vi.hoisted(() => {
+ type Entry = { value: string; expiresAtMs: number | null };
+ const store = new Map<string, Entry>();
+ const nowMs = () => Date.now();
+ const isLive = (entry: Entry | undefined) =>
+ entry != null && (entry.expiresAtMs == null || entry.expiresAtMs > nowMs());
+
+ const client = {
+ get status() {
+ return redisAvailability.available ? "ready" : "end";
+ },
+ disconnect: vi.fn(),
+ ping: async () => "PONG",
+ get: async (key: string) => {
+ const entry = store.get(key);
+ if (!isLive(entry)) {
+ store.delete(key);
+ return null;
+ }
+ return entry!.value;
+ },
+ mget: async (...keysOrArray: string[] | [string[]]) => {
+ const keys =
+ keysOrArray.length === 1 && Array.isArray(keysOrArray[0])
+ ? keysOrArray[0]
+ : (keysOrArray as string[]);
+ return Promise.all(keys.map((key) => client.get(key)));
+ },
+ setex: async (key: string, ttlSeconds: number, value: string) => {
+ store.set(key, {
+ value,
+ expiresAtMs: nowMs() + ttlSeconds * 1000,
+ });
+ return "OK";
+ },
+ del: async (...keys: string[]) => {
+ let removed = 0;
+ for (const key of keys) {
+ if (store.delete(key)) removed += 1;
+ }
+ return removed;
+ },
+ ttl: async (key: string) => {
+ const entry = store.get(key);
+ if (!isLive(entry)) {
+ store.delete(key);
+ return -2;
+ }
+ if (entry!.expiresAtMs == null) return -1;
+ return Math.max(0, Math.ceil((entry!.expiresAtMs - nowMs()) / 1000));
+ },
+ scan: async (
+ cursor: string,
+ _matchToken: string,
+ pattern: string,
+ _countToken: string,
+ _count: number,
+ ) => {
+ const prefix = pattern.endsWith("*") ? pattern.slice(0, -1) : pattern;
+ const keys = [...store.keys()].filter((key) =>
+ pattern.endsWith("*") ? key.startsWith(prefix) : key === pattern,
+ );
+ // Single-page scan: first call returns all keys, next cursor is "0".
+ if (cursor !== "0") return ["0", [] as string[]];
+ return ["0", keys];
+ },
+ __store: store,
+ __reset() {
+ store.clear();
+ },
+ };
+ return client;
+});
+
const sharedServerMock = vi.hoisted(() => ({
queryClickhouse: vi.fn(),
isForceV3ExperienceProject: vi.fn(() => false),
@@ -119,10 +190,12 @@ const sharedEnvMock = vi.hoisted(() => ({
vi.mock("@langfuse/shared/src/env", () => ({ env: sharedEnvMock }));
-vi.mock("@langfuse/shared/src/server", async () => {
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
+ const actual = await importOriginal();
const { ROOT_CONTEXT } = await import("@opentelemetry/api");
return {
+ ...(actual as object),
...sharedServerMock,
getTraceById: vi.fn(),
logger: {
@@ -131,7 +204,7 @@ vi.mock("@langfuse/shared/src/server", async () => {
warn: vi.fn(),
error: vi.fn(),
},
- redis: redisMock,
+ redis: memoryRedis,
ClickHouseClientManager: {
getInstance: () => ({
closeAllConnections: vi.fn(),
@@ -390,7 +463,6 @@ const accessibleProjectsFindManyArgs = {
const TEST_NOW = new Date("2026-06-25T00:30:00Z");
const HOT_START_ISO = "2026-06-25T00:00:00.000Z";
const HOT_START_CLICKHOUSE = "2026-06-25 00:00:00.000";
-const WINDOW_END_CLICKHOUSE = "2026-06-25 01:00:00.000";
const WINDOW_START_CLICKHOUSE = "2026-06-11 01:00:00.000";
// Recent SDK gap cutoff: TEST_NOW floored to the minute (zero
// seconds/millis) so repeated calls within the minute share one ClickHouse
@@ -398,28 +470,65 @@ const WINDOW_START_CLICKHOUSE = "2026-06-11 01:00:00.000";
const RECENT_CUTOFF_CLICKHOUSE = "2026-06-25 00:30:00.000";
const sdkUsageCacheKey = `langfuse:v4:sdk-usage:v1:${projectId}`;
+const legacyApiUsageCacheKey = `langfuse:v4:legacy-api-usage:v1:${projectId}`;
+const experimentPostUsageCacheKey = `langfuse:v4:experiment-post-usage:v1:${projectId}`;
-/** In-memory Redis fake: mget/setex against a Map, status "ready". */
-const enableRedisCache = (initialEntries: Record<string, string> = {}) => {
- const store = new Map<string, string>(Object.entries(initialEntries));
- redisMock.status = "ready";
- redisMock.mget.mockImplementation(async (keys: string[]) =>
- keys.map((key) => store.get(key) ?? null),
- );
- redisMock.setex.mockImplementation(
- async (key: string, _ttlSeconds: number, value: string) => {
- store.set(key, value);
- return "OK";
- },
+const experimentPostBlob = (
+ used: boolean,
+ lastSeen: string | null = used ? "2026-06-24T15:00:00.000000Z" : null,
+) =>
+ JSON.stringify({
+ version: 1,
+ computedAt: "2026-06-25T00:00:00.000Z",
+ used,
+ lastSeen,
+ });
+
+const legacyApiBlob = (
+ rows: { entrypoint: string; count: number; lastSeen: string }[],
+) =>
+ JSON.stringify({
+ version: 1,
+ computedAt: "2026-06-25T00:00:00.000Z",
+ rows,
+ });
+
+const legacyApiUsageHeartbeatKey = "langfuse:v4:legacy-api-usage:heartbeat:v1";
+
+const DEFAULT_SEED_TTL_SECONDS = 60 * 60;
+
+const clearV4CacheKeys = async () => {
+ memoryRedis.__reset();
+};
+
+/** Seed in-memory Redis keys and enable the cache for the code under test. */
+const seedRedisCache = async (
+ initialEntries: Record<string, string> = {},
+ ttlSeconds = DEFAULT_SEED_TTL_SECONDS,
+) => {
+ await clearV4CacheKeys();
+ redisAvailability.available = true;
+ if (Object.keys(initialEntries).length === 0) return;
+ await Promise.all(
+ Object.entries(initialEntries).map(([key, value]) =>
+ memoryRedis.setex(key, ttlSeconds, value),
+ ),
);
- return store;
};
+const readRedisJson = async (key: string) => {
+ const raw = await memoryRedis.get(key);
+ return raw === null ? null : JSON.parse(raw);
+};
+
+const readRedisTtl = async (key: string) => memoryRedis.ttl(key);
+
describe("v4TransitionRouter", () => {
- beforeEach(() => {
+ beforeEach(async () => {
vi.useFakeTimers({ toFake: ["Date"] });
vi.setSystemTime(TEST_NOW);
- redisMock.status = "end";
+ redisAvailability.available = false;
+ await clearV4CacheKeys();
sharedEnvMock.CLICKHOUSE_READ_ONLY_URL =
"https://clickhouse-read.example.com";
sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =
@@ -434,35 +543,31 @@ describe("v4TransitionRouter", () => {
]);
});
+ afterEach(async () => {
+ redisAvailability.available = false;
+ await clearV4CacheKeys();
+ });
+
afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
});
- it("summarizes legacy public API usage for a project with route classification", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
+ it("serves legacy public API usage from Redis without querying ClickHouse", async () => {
+ await seedRedisCache({
+ [legacyApiUsageCacheKey]: legacyApiBlob([
{
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "0.6666666666666666",
- lastSeen: "2026-06-24T12:34:56.789123Z",
- },
- {
- projectId,
entrypoint: "publicapi: GET /api/public/datasets/{datasetName}/runs",
- count: "1",
+ count: 1,
lastSeen: "2026-06-24T14:00:00.000000Z",
},
- ])
- .mockResolvedValueOnce([
{
- projectId,
entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "1.3333333333333335",
+ count: 2,
lastSeen: "2026-06-24T15:00:00.000000Z",
},
- ]);
+ ]),
+ });
const caller = createCaller();
const rows = await caller.legacyApiUsageSummary({
@@ -483,187 +588,34 @@ describe("v4TransitionRouter", () => {
lastSeen: "2026-06-24T15:00:00.000000Z",
},
]);
-
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
- const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];
- expect(clickhouseQuery?.query).toContain(
- "FROM clusterAllReplicas('test-cluster', 'system.query_log')",
- );
- expect(clickhouseQuery?.query).not.toContain("toStartOfInterval");
- expect(clickhouseQuery?.query).not.toContain("bucket_time");
- expect(clickhouseQuery?.query).toContain(
- "splitByChar('?', JSONExtractString(log_comment, 'route'))[1]",
- );
- expect(clickhouseQuery?.query).toContain(
- "sum(1.0 / clickhouse_queries_per_api_call) AS count",
- );
- expect(clickhouseQuery?.query).toContain(
- "formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen",
- );
- expect(clickhouseQuery?.query).toContain(
- "SETTINGS skip_unavailable_shards = 1",
- );
- expect(clickhouseQuery?.query).toContain("AND type = 'QueryFinish'");
- expect(clickhouseQuery?.query).toContain(
- "JSONExtractString(log_comment, 'tag_schema_version') = '1'",
- );
- expect(clickhouseQuery?.query).toContain(
- "JSONExtractString(log_comment, 'surface') = 'publicapi'",
- );
- expect(clickhouseQuery?.query).toContain(
- "JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}",
- );
- expect(clickhouseQuery?.query).toContain(
- "GROUP BY project_id, legacy_route",
- );
- expect(clickhouseQuery?.query).not.toContain("hostName()");
- expect(clickhouseQuery?.params).toMatchObject({
- projectIds: [projectId],
- });
- expect(clickhouseQuery?.tags).toEqual({
- route: "v4-legacy-api-usage-summary",
- });
- expect(clickhouseQuery?.clickhouseSettings).toEqual({
- skip_unavailable_shards: 1,
- });
- expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({
- preferredClickhouseService: "ReadOnly",
- });
- expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({
- preferredClickhouseService: "ReadWrite",
- });
-
- [
- "GET /api/public/spans",
- "GET /api/public/generations",
- "GET /api/public/traces",
- "GET /api/public/sessions",
- "GET /api/public/observations",
- "GET /api/public/scores",
- "GET /api/public/v2/scores",
- "GET /api/public/metrics",
- "GET /api/public/metrics/daily",
- "GET /api/public/dataset-run-items",
- ].forEach((route) => expect(clickhouseQuery?.query).toContain(route));
-
- [
- "GET /api/public/traces/{id}",
- "GET /api/public/sessions/{id}",
- "GET /api/public/observations/{id}",
- "GET /api/public/scores/{id}",
- "GET /api/public/v2/scores/{id}",
- "GET /api/public/datasets/{datasetName}/runs",
- "GET /api/public/datasets/{datasetName}/runs/{runName}",
- ].forEach((route) => expect(clickhouseQuery?.query).toContain(route));
- expect(clickhouseQuery?.query).toContain(
- "match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs'",
- );
- expect(clickhouseQuery?.query).toContain(
- "match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1",
- );
- expect(clickhouseQuery?.query).not.toContain("DELETE /api/public/datasets");
- expect(clickhouseQuery?.query).toContain(
- "'GET /api/public/traces',\n 'GET /api/public/observations',\n 'GET /api/public/scores',\n 'GET /api/public/v2/scores',\n 'GET /api/public/metrics/daily',\n 'GET /api/public/dataset-run-items'\n ), 2",
- );
- expect(clickhouseQuery?.query).toContain(
- "'GET /api/public/sessions',\n 'GET /api/public/metrics'\n ), 1",
- );
- expect(clickhouseQuery?.query).toContain(
- "match(route_path, '^GET /api/public/traces/[^/?#]+$'), 3",
- );
- });
-
- it("queries the main service once when no separate read replica is configured", async () => {
- sharedEnvMock.CLICKHOUSE_READ_ONLY_URL = sharedEnvMock.CLICKHOUSE_URL;
- const caller = createCaller();
-
- await caller.legacyApiUsageSummary({
- projectId,
- });
-
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
- expect(mockedQueryClickhouse).toHaveBeenCalledWith(
- expect.objectContaining({ preferredClickhouseService: "ReadWrite" }),
- );
+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();
});
- it("returns legacy API usage when one ClickHouse service is unavailable", async () => {
- mockedQueryClickhouse
- .mockRejectedValueOnce(new Error("read replica unavailable"))
- .mockResolvedValueOnce([
- {
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "1",
- lastSeen: "2026-06-24T15:00:00.000000Z",
- },
- ]);
+ it("treats missing legacy API Redis entries as no usage", async () => {
+ await seedRedisCache();
const caller = createCaller();
await expect(
caller.legacyApiUsageSummary({
projectId,
}),
- ).resolves.toEqual([
- {
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: 1,
- lastSeen: "2026-06-24T15:00:00.000000Z",
- },
- ]);
-
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
+ ).resolves.toEqual([]);
+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();
+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();
});
- it("does not double-count identical results from two ClickHouse services", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- {
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "2",
- lastSeen: "2026-06-24T14:00:00.000000Z",
- },
- ])
- .mockResolvedValueOnce([
- {
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "2",
- lastSeen: "2026-06-24T14:00:00.000000Z",
- },
- ]);
- const caller = createCaller();
-
- await expect(
- caller.legacyApiUsageSummary({
+ it("queries SDK usage for only the authorized project", async () => {
+ mockedQueryClickhouse.mockResolvedValueOnce([
+ mockSdkUsageRow({
projectId,
+ sdkName: "python",
+ sdkVersion: "4.7.0",
+ publicKey: "pk-lf-python",
+ eventCount: "2",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T02:00:00Z",
}),
- ).resolves.toEqual([
- {
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: 2,
- lastSeen: "2026-06-24T14:00:00.000000Z",
- },
]);
- });
-
- it("queries SDK usage for only the authorized project", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "4.7.0",
- publicKey: "pk-lf-python",
- eventCount: "2",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T02:00:00Z",
- }),
- ])
- .mockResolvedValueOnce([]);
const caller = createCaller();
const summary = await caller.sdkUsageSummary({
@@ -686,24 +638,23 @@ describe("v4TransitionRouter", () => {
},
],
});
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
expect(mockedQueryClickhouse.mock.calls[0]?.[0].params).toMatchObject({
projectIds: [projectId],
});
- expect(mockedQueryClickhouse.mock.calls[1]?.[0].params).toMatchObject({
- projectIds: [projectId],
- });
});
it("rejects project summaries outside the caller session", async () => {
const caller = createCaller();
+ const range = {};
await expect(
- caller.sdkUsageSummary({ projectId: outsideProjectId }),
+ caller.sdkUsageSummary({ projectId: outsideProjectId, ...range }),
).rejects.toThrow("User is not a member of this project");
await expect(
caller.legacyApiUsageSummary({
projectId: outsideProjectId,
+ ...range,
}),
).rejects.toThrow("User is not a member of this project");
expect(mockedQueryClickhouse).not.toHaveBeenCalled();
@@ -1118,75 +1069,73 @@ describe("v4TransitionRouter", () => {
);
it("summarizes outdated SDK usage series by organization project", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "3.9.0",
- publicKey: "pk-lf-python",
- eventCount: "8",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T02:00:00Z",
- }),
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "4.14.1",
- publicKey: "pk-lf-python",
- eventCount: "13",
- firstSeen: "2026-06-24T03:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- }),
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "4.6.9",
- publicKey: "pk-lf-pre-v4-python",
- eventCount: "5",
- firstSeen: "2026-06-24T12:00:00Z",
- lastSeen: "2026-06-24T13:00:00Z",
- }),
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "4.7.0",
- publicKey: "pk-lf-current-python",
- eventCount: "6",
- firstSeen: "2026-06-24T13:30:00Z",
- lastSeen: "2026-06-24T14:00:00Z",
- }),
- mockSdkUsageRow({
- projectId: secondProjectId,
- sdkName: "@langfuse/tracing",
- sdkVersion: "5.3.9",
- publicKey: "pk-lf-old-js",
- eventCount: "5",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- }),
- mockSdkUsageRow({
- projectId: secondProjectId,
- source: "otel",
- sdkName: "unknown",
- sdkVersion: "unknown",
- publicKey: "pk-lf-otel",
- eventCount: "3",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- }),
- mockSdkUsageRow({
- projectId: secondProjectId,
- source: "otel-dual-write",
- sdkName: "custom-otel-writer",
- sdkVersion: "1.2.3",
- publicKey: "pk-lf-custom-otel",
- eventCount: "2",
- firstSeen: "2026-06-24T02:00:00Z",
- lastSeen: "2026-06-24T03:00:00Z",
- }),
- ])
- .mockResolvedValueOnce([{ projectId, count: "1" }]);
+ mockedQueryClickhouse.mockResolvedValueOnce([
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "3.9.0",
+ publicKey: "pk-lf-python",
+ eventCount: "8",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T02:00:00Z",
+ }),
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "4.14.1",
+ publicKey: "pk-lf-python",
+ eventCount: "13",
+ firstSeen: "2026-06-24T03:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ }),
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "4.6.9",
+ publicKey: "pk-lf-pre-v4-python",
+ eventCount: "5",
+ firstSeen: "2026-06-24T12:00:00Z",
+ lastSeen: "2026-06-24T13:00:00Z",
+ }),
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "4.7.0",
+ publicKey: "pk-lf-current-python",
+ eventCount: "6",
+ firstSeen: "2026-06-24T13:30:00Z",
+ lastSeen: "2026-06-24T14:00:00Z",
+ }),
+ mockSdkUsageRow({
+ projectId: secondProjectId,
+ sdkName: "@langfuse/tracing",
+ sdkVersion: "5.3.9",
+ publicKey: "pk-lf-old-js",
+ eventCount: "5",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ }),
+ mockSdkUsageRow({
+ projectId: secondProjectId,
+ source: "otel",
+ sdkName: "unknown",
+ sdkVersion: "unknown",
+ publicKey: "pk-lf-otel",
+ eventCount: "3",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ }),
+ mockSdkUsageRow({
+ projectId: secondProjectId,
+ source: "otel-dual-write",
+ sdkName: "custom-otel-writer",
+ sdkVersion: "1.2.3",
+ publicKey: "pk-lf-custom-otel",
+ eventCount: "2",
+ firstSeen: "2026-06-24T02:00:00Z",
+ lastSeen: "2026-06-24T03:00:00Z",
+ }),
+ ]);
const mockPrisma = {
project: {
findMany: vi.fn().mockResolvedValue([
@@ -1205,8 +1154,8 @@ describe("v4TransitionRouter", () => {
{
projectId,
experimentInstrumentationMigration: {
- status: "sdk_usage_inconclusive",
- upgradePath: "sdk",
+ status: "not_required",
+ upgradePath: null,
},
// Most recently seen first: the server re-sorts after merging cached
// and live series.
@@ -1361,7 +1310,7 @@ describe("v4TransitionRouter", () => {
expect(mockPrisma.project.findMany).toHaveBeenCalledWith(
accessibleProjectsFindManyArgs,
);
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
const usageQuery = mockedQueryClickhouse.mock.calls[0]?.[0];
expect(usageQuery?.query).toContain("FROM events_core");
expect(usageQuery?.query).not.toContain("WITH selected");
@@ -1409,46 +1358,21 @@ describe("v4TransitionRouter", () => {
use_query_cache: 1,
query_cache_ttl: 300,
});
-
- const experimentUsageQuery = mockedQueryClickhouse.mock.calls[1]?.[0];
- expect(experimentUsageQuery?.query).toContain(
- "splitByChar('?', JSONExtractString(log_comment, 'route'))[1] = 'POST /api/public/dataset-run-items'",
- );
- expect(experimentUsageQuery?.query).toContain(
- "JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}",
- );
- expect(experimentUsageQuery?.query).toContain(
- "event_date >= toDate({fromTimestamp: DateTime64(3)})",
- );
- expect(experimentUsageQuery?.query).not.toContain("hostName()");
- expect(experimentUsageQuery?.params).toMatchObject({
- projectIds: [projectId, secondProjectId],
- fromTimestamp: WINDOW_START_CLICKHOUSE,
- toTimestamp: WINDOW_END_CLICKHOUSE,
- });
- expect(experimentUsageQuery?.tags).toEqual({
- route: "v4-experiment-instrumentation-summary",
- });
- // The query_log-backed experiment check must not enable the query cache.
- expect(
- experimentUsageQuery?.clickhouseSettings?.use_query_cache,
- ).toBeUndefined();
});
- it("finds dataset-run-items POST usage when only the main service has it", async () => {
- sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =
- "https://clickhouse-events-read.example.com";
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- source: "otel",
- sdkName: "unknown",
- sdkVersion: "unknown",
- }),
- ])
- .mockRejectedValueOnce(new Error("events read replica unavailable"))
- .mockResolvedValueOnce([{ projectId, count: "1" }]);
+ it("requires an API upgrade when Redis shows dataset-run-items POST usage", async () => {
+ await seedRedisCache({
+ [experimentPostUsageCacheKey]: experimentPostBlob(true),
+ });
+ // Redis is available but the SDK blob is missing: historical + live gap.
+ mockedQueryClickhouse.mockResolvedValue([
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "3.3.5",
+ publicKey: "pk-lf-python",
+ }),
+ ]);
const caller = createCaller({
project: {
findMany: vi.fn().mockResolvedValue([{ id: projectId }]),
@@ -1463,29 +1387,29 @@ describe("v4TransitionRouter", () => {
status: "required",
upgradePath: "api",
});
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(3);
- expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({
- preferredClickhouseService: "EventsReadOnly",
- });
- expect(mockedQueryClickhouse.mock.calls[2]?.[0]).toMatchObject({
- preferredClickhouseService: "ReadWrite",
- });
+ const eventsCoreCalls = mockedQueryClickhouse.mock.calls.filter(([args]) =>
+ args.query.includes("FROM events_core"),
+ );
+ expect(eventsCoreCalls).toHaveLength(2);
+ expect(
+ mockedQueryClickhouse.mock.calls.some(([args]) =>
+ args.query.includes("system.query_log"),
+ ),
+ ).toBe(false);
});
it("summarizes SDK usage for a single project with exactly that projectId", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "4.7.0",
- publicKey: "pk-lf-python",
- eventCount: "6",
- firstSeen: "2026-06-24T13:30:00Z",
- lastSeen: "2026-06-24T14:00:00Z",
- }),
- ])
- .mockResolvedValueOnce([]);
+ mockedQueryClickhouse.mockResolvedValueOnce([
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "4.7.0",
+ publicKey: "pk-lf-python",
+ eventCount: "6",
+ firstSeen: "2026-06-24T13:30:00Z",
+ lastSeen: "2026-06-24T14:00:00Z",
+ }),
+ ]);
const caller = createCaller();
@@ -1511,64 +1435,56 @@ describe("v4TransitionRouter", () => {
],
});
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
expect(mockedQueryClickhouse.mock.calls[0]?.[0]?.params).toMatchObject({
projectIds: [projectId],
});
- expect(mockedQueryClickhouse.mock.calls[1]?.[0]?.params).toMatchObject({
- projectIds: [projectId],
- });
expect(mockedQueryClickhouse.mock.calls[0]?.[0]?.tags).toEqual({
route: "v4-sdk-usage-summary",
});
- expect(mockedQueryClickhouse.mock.calls[1]?.[0]?.tags).toEqual({
- route: "v4-experiment-instrumentation-summary",
- });
});
it("keeps source-specific series and consumes SQL-owned classification", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- {
- projectId,
- source: "otel-dual-write",
- ingestionPath: "otel",
- deliveryMode: "delayed",
- sdkName: "unknown",
- sdkVersion: "unknown",
- canonicalSdkName: null,
- sdkVersionMajor: null,
- isValidSdkVersion: false,
- attributionStatus: "missing_name_and_version",
- publicKey: "pk-lf-shared",
- v4MigrationStatus: "unknown",
- remediationType: "update_otel_instrumentation",
- actionLevel: "required",
- eventCount: "3",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T02:00:00Z",
- },
- {
- projectId,
- source: "ingestion-api-dual-write",
- ingestionPath: "ingestion_api",
- deliveryMode: "delayed",
- sdkName: "unknown",
- sdkVersion: "unknown",
- canonicalSdkName: null,
- sdkVersionMajor: null,
- isValidSdkVersion: false,
- attributionStatus: "missing_name_and_version",
- publicKey: "pk-lf-shared",
- v4MigrationStatus: "unknown",
- remediationType: "upgrade_instrumentation",
- actionLevel: "required",
- eventCount: "2",
- firstSeen: "2026-06-24T03:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- },
- ])
- .mockResolvedValueOnce([]);
+ mockedQueryClickhouse.mockResolvedValueOnce([
+ {
+ projectId,
+ source: "otel-dual-write",
+ ingestionPath: "otel",
+ deliveryMode: "delayed",
+ sdkName: "unknown",
+ sdkVersion: "unknown",
+ canonicalSdkName: null,
+ sdkVersionMajor: null,
+ isValidSdkVersion: false,
+ attributionStatus: "missing_name_and_version",
+ publicKey: "pk-lf-shared",
+ v4MigrationStatus: "unknown",
+ remediationType: "update_otel_instrumentation",
+ actionLevel: "required",
+ eventCount: "3",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T02:00:00Z",
+ },
+ {
+ projectId,
+ source: "ingestion-api-dual-write",
+ ingestionPath: "ingestion_api",
+ deliveryMode: "delayed",
+ sdkName: "unknown",
+ sdkVersion: "unknown",
+ canonicalSdkName: null,
+ sdkVersionMajor: null,
+ isValidSdkVersion: false,
+ attributionStatus: "missing_name_and_version",
+ publicKey: "pk-lf-shared",
+ v4MigrationStatus: "unknown",
+ remediationType: "upgrade_instrumentation",
+ actionLevel: "required",
+ eventCount: "2",
+ firstSeen: "2026-06-24T03:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ },
+ ]);
const caller = createCaller();
const summary = await caller.sdkUsageSummary({
@@ -1606,20 +1522,21 @@ describe("v4TransitionRouter", () => {
});
it("requires removing dataset-run-items POST usage for native OTel instrumentation", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- source: "otel",
- sdkName: "unknown",
- sdkVersion: "unknown",
- publicKey: "pk-lf-otel",
- eventCount: "3",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- }),
- ])
- .mockResolvedValueOnce([{ projectId, count: "1" }]);
+ await seedRedisCache({
+ [experimentPostUsageCacheKey]: experimentPostBlob(true),
+ });
+ mockedQueryClickhouse.mockResolvedValue([
+ mockSdkUsageRow({
+ projectId,
+ source: "otel",
+ sdkName: "unknown",
+ sdkVersion: "unknown",
+ publicKey: "pk-lf-otel",
+ eventCount: "3",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ }),
+ ]);
const caller = createCaller({
project: {
findMany: vi.fn().mockResolvedValue([{ id: projectId }]),
@@ -1639,20 +1556,19 @@ describe("v4TransitionRouter", () => {
});
});
- it("preserves SDK usage when the experiment instrumentation check fails", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "3.9.0",
- publicKey: "pk-lf-python",
- eventCount: "3",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- }),
- ])
- .mockRejectedValueOnce(new Error("query_log unavailable"));
+ it("keeps SDK usage when experiment POST Redis data is missing", async () => {
+ // No worker entry and no query_log fallback: experiment stays not_required.
+ mockedQueryClickhouse.mockResolvedValueOnce([
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "3.9.0",
+ publicKey: "pk-lf-python",
+ eventCount: "3",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ }),
+ ]);
const caller = createCaller({
project: {
findMany: vi.fn().mockResolvedValue([{ id: projectId }]),
@@ -1666,7 +1582,7 @@ describe("v4TransitionRouter", () => {
expect(summary).toMatchObject({
projectId,
experimentInstrumentationMigration: {
- status: "check_failed",
+ status: "not_required",
upgradePath: null,
},
sdkUsageSeries: [
@@ -1682,19 +1598,20 @@ describe("v4TransitionRouter", () => {
});
it("requires an API upgrade when POST usage predates the experiment runner", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "3.3.5",
- publicKey: "pk-lf-python",
- eventCount: "3",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- }),
- ])
- .mockResolvedValueOnce([{ projectId, count: "1" }]);
+ await seedRedisCache({
+ [experimentPostUsageCacheKey]: experimentPostBlob(true),
+ });
+ mockedQueryClickhouse.mockResolvedValue([
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "3.3.5",
+ publicKey: "pk-lf-python",
+ eventCount: "3",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ }),
+ ]);
const caller = createCaller({
project: {
findMany: vi.fn().mockResolvedValue([{ id: projectId }]),
@@ -1712,19 +1629,20 @@ describe("v4TransitionRouter", () => {
});
it("does not require an upgrade for current experiment instrumentation", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "4.0.0",
- publicKey: "pk-lf-python",
- eventCount: "3",
- firstSeen: "2026-06-24T01:00:00Z",
- lastSeen: "2026-06-24T04:00:00Z",
- }),
- ])
- .mockResolvedValueOnce([{ projectId, count: "1" }]);
+ await seedRedisCache({
+ [experimentPostUsageCacheKey]: experimentPostBlob(true),
+ });
+ mockedQueryClickhouse.mockResolvedValue([
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "4.0.0",
+ publicKey: "pk-lf-python",
+ eventCount: "3",
+ firstSeen: "2026-06-24T01:00:00Z",
+ lastSeen: "2026-06-24T04:00:00Z",
+ }),
+ ]);
const caller = createCaller({
project: {
findMany: vi.fn().mockResolvedValue([{ id: projectId }]),
@@ -1741,36 +1659,23 @@ describe("v4TransitionRouter", () => {
});
});
- it("summarizes legacy public API usage by organization project", async () => {
- mockedQueryClickhouse
- .mockResolvedValueOnce([
+ it("summarizes legacy public API usage by organization project from Redis", async () => {
+ await seedRedisCache({
+ [legacyApiUsageCacheKey]: legacyApiBlob([
{
- projectId,
entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "0.6666666666666666",
- lastSeen: "2026-06-24T12:34:56.789123Z",
- },
- {
- projectId: secondProjectId,
- entrypoint: "publicapi: GET /api/public/metrics",
- count: 3,
- lastSeen: "2026-06-24T15:00:00.000000Z",
- },
- ])
- .mockResolvedValueOnce([
- {
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "0.3333333333333334",
+ count: 1,
lastSeen: "2026-06-24T14:00:00.000000Z",
},
+ ]),
+ [`langfuse:v4:legacy-api-usage:v1:${secondProjectId}`]: legacyApiBlob([
{
- projectId: secondProjectId,
entrypoint: "publicapi: GET /api/public/metrics",
- count: 2,
+ count: 5,
lastSeen: "2026-06-24T16:00:00.000000Z",
},
- ]);
+ ]),
+ });
const mockPrisma = {
project: {
findMany: vi.fn().mockResolvedValue([
@@ -1799,39 +1704,10 @@ describe("v4TransitionRouter", () => {
lastSeen: "2026-06-24T16:00:00.000000Z",
},
]);
-
expect(mockPrisma.project.findMany).toHaveBeenCalledWith(
accessibleProjectsFindManyArgs,
);
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
- const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];
- expect(clickhouseQuery?.query).not.toContain("toStartOfInterval");
- expect(clickhouseQuery?.query).not.toContain("bucket_time");
- expect(clickhouseQuery?.query).toContain(
- "JSONExtractString(log_comment, 'projectId') AS project_id",
- );
- expect(clickhouseQuery?.query).toContain(
- "JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}",
- );
- expect(clickhouseQuery?.query).toContain("project_id AS projectId");
- expect(clickhouseQuery?.query).toContain(
- "formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen",
- );
- expect(clickhouseQuery?.query).toContain(
- "GROUP BY project_id, legacy_route",
- );
- expect(clickhouseQuery?.params).toMatchObject({
- projectIds: [projectId, secondProjectId],
- });
- expect(clickhouseQuery?.tags).toEqual({
- route: "v4-legacy-api-usage-summary",
- });
- expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({
- preferredClickhouseService: "ReadOnly",
- });
- expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({
- preferredClickhouseService: "ReadWrite",
- });
+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();
});
describe("redis caching", () => {
@@ -1876,7 +1752,7 @@ describe("v4TransitionRouter", () => {
});
it("splits SDK usage into a cached-historical and a live-gap query on cache miss", async () => {
- enableRedisCache();
+ await seedRedisCache();
const historicalRow = mockSdkUsageRow({
projectId,
sdkName: "python",
@@ -1896,8 +1772,6 @@ describe("v4TransitionRouter", () => {
lastSeen: "2026-06-25T00:15:00Z",
});
mockedQueryClickhouse.mockImplementation(async (args) => {
- // SDK path also fans out a query_log experiment-POST check; only
- // events_core participates in the history/gap split under test.
if (!args.query.includes("FROM events_core")) return [];
return (args.params?.toTimestamp as string) === HOT_START_CLICKHOUSE
? [historicalRow]
@@ -1953,38 +1827,37 @@ describe("v4TransitionRouter", () => {
});
}
- // Only the historical SDK slice is cached (1h TTL); the gap stays live.
- // query_log experiment usage is not Redis-cached in this PR.
- expect(redisMock.setex).toHaveBeenCalledTimes(1);
- const sdkWrite = redisMock.setex.mock.calls.find(
- ([key]) => key === sdkUsageCacheKey,
- );
- expect(sdkWrite?.[1]).toBe(60 * 60);
- expect(JSON.parse(sdkWrite?.[2] as string)).toMatchObject({
+ // Only the historical slice is cached (1h TTL); the gap stays live.
+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({
version: 1,
hotStart: HOT_START_ISO,
series: [expect.objectContaining({ eventCount: 5 })],
});
+ expect(await readRedisTtl(sdkUsageCacheKey)).toBeGreaterThan(55 * 60);
+ expect(await readRedisTtl(sdkUsageCacheKey)).toBeLessThanOrEqual(60 * 60);
+ // Experiment POST is Redis-only from the worker; the request path must
+ // not shadow-write empty entries on miss.
+ await expect(
+ readRedisJson(experimentPostUsageCacheKey),
+ ).resolves.toBeNull();
});
it("serves cached SDK history with a live gap query on cache hit", async () => {
- enableRedisCache({
+ await seedRedisCache({
[sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries()]),
+ [experimentPostUsageCacheKey]: experimentPostBlob(false),
});
- mockedQueryClickhouse.mockImplementation(async (args) => {
- if (!args.query.includes("FROM events_core")) return [];
- return [
- mockSdkUsageRow({
- projectId,
- sdkName: "python",
- sdkVersion: "3.9.0",
- publicKey: "pk-lf-python",
- eventCount: "2",
- firstSeen: "2026-06-25T00:05:00Z",
- lastSeen: "2026-06-25T00:15:00Z",
- }),
- ];
- });
+ mockedQueryClickhouse.mockResolvedValue([
+ mockSdkUsageRow({
+ projectId,
+ sdkName: "python",
+ sdkVersion: "3.9.0",
+ publicKey: "pk-lf-python",
+ eventCount: "2",
+ firstSeen: "2026-06-25T00:05:00Z",
+ lastSeen: "2026-06-25T00:15:00Z",
+ }),
+ ]);
const caller = createCaller();
const summary = await caller.sdkUsageSummary({
@@ -1999,25 +1872,30 @@ describe("v4TransitionRouter", () => {
lastSeen: "2026-06-25T00:15:00Z",
}),
]);
- // No dataset-run-items POST usage in the uncached query_log check.
+ // Cached experiment usage: not_required without a query_log scan.
expect(summary.experimentInstrumentationMigration).toEqual({
status: "not_required",
upgradePath: null,
});
- const eventsCoreCalls = mockedQueryClickhouse.mock.calls.filter(
- ([args]) => args.query.includes("FROM events_core"),
+ // Exactly one ClickHouse query: the live gap on events_core.
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
+ expect(mockedQueryClickhouse.mock.calls[0]?.[0].query).toContain(
+ "FROM events_core",
);
- expect(eventsCoreCalls).toHaveLength(1);
- expect(eventsCoreCalls[0]?.[0].params).toMatchObject({
+ expect(mockedQueryClickhouse.mock.calls[0]?.[0].params).toMatchObject({
fromTimestamp: HOT_START_CLICKHOUSE,
toTimestamp: RECENT_CUTOFF_CLICKHOUSE,
});
- expect(redisMock.setex).not.toHaveBeenCalled();
+ // Cache hit must not rewrite the historical blob.
+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({
+ hotStart: HOT_START_ISO,
+ series: [expect.objectContaining({ eventCount: 5 })],
+ });
});
it("drops cached SDK series that aged out of the 14-day window", async () => {
- enableRedisCache({
+ await seedRedisCache({
[sdkUsageCacheKey]: sdkUsageBlob([
cachedSdkSeries(),
cachedSdkSeries({
@@ -2027,11 +1905,9 @@ describe("v4TransitionRouter", () => {
lastSeen: "2026-06-10T00:00:00Z",
}),
]),
+ [experimentPostUsageCacheKey]: experimentPostBlob(false),
});
- mockedQueryClickhouse.mockImplementation(async (args) => {
- if (!args.query.includes("FROM events_core")) return [];
- return [];
- });
+ mockedQueryClickhouse.mockResolvedValue([]);
const caller = createCaller();
const summary = await caller.sdkUsageSummary({
@@ -2044,13 +1920,11 @@ describe("v4TransitionRouter", () => {
});
it("treats blobs with corrupt JSON as cache misses", async () => {
- enableRedisCache({
+ await seedRedisCache({
[sdkUsageCacheKey]: "not-json",
+ [experimentPostUsageCacheKey]: experimentPostBlob(false),
});
- mockedQueryClickhouse.mockImplementation(async (args) => {
- if (!args.query.includes("FROM events_core")) return [];
- return [];
- });
+ mockedQueryClickhouse.mockResolvedValue([]);
const caller = createCaller();
await caller.sdkUsageSummary({
@@ -2062,9 +1936,155 @@ describe("v4TransitionRouter", () => {
([args]) => args.query.includes("FROM events_core"),
);
expect(eventsCoreCalls).toHaveLength(2);
- expect(
- redisMock.setex.mock.calls.some(([key]) => key === sdkUsageCacheKey),
- ).toBe(true);
+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({
+ version: 1,
+ hotStart: HOT_START_ISO,
+ });
+ });
+
+ it("serves legacy API usage from the cache without querying ClickHouse", async () => {
+ await seedRedisCache({
+ [legacyApiUsageCacheKey]: legacyApiBlob([
+ {
+ entrypoint: "publicapi: GET /api/public/traces",
+ count: 4,
+ lastSeen: "2026-06-24T12:00:00.000000Z",
+ },
+ {
+ // Aged out of the window: trimmed at read time.
+ entrypoint: "publicapi: GET /api/public/sessions",
+ count: 2,
+ lastSeen: "2026-06-10T00:00:00.000000Z",
+ },
+ ]),
+ });
+ const caller = createCaller();
+
+ const rows = await caller.legacyApiUsageSummary({
+ projectId,
+ });
+
+ expect(rows).toEqual([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces",
+ count: 4,
+ lastSeen: "2026-06-24T12:00:00.000000Z",
+ },
+ ]);
+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();
+ });
+
+ it("does not query or shadow-write legacy API Redis entries on miss", async () => {
+ await seedRedisCache();
+ const mockPrisma = {
+ project: {
+ findMany: vi.fn().mockResolvedValue([
+ { id: projectId, name: "V4 Transition Project" },
+ { id: secondProjectId, name: "Second Project" },
+ ]),
+ },
+ };
+ const caller = createCaller(mockPrisma);
+
+ await expect(
+ caller.legacyApiUsageSummaryByProject({
+ orgId,
+ }),
+ ).resolves.toEqual([]);
+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();
+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();
+ await expect(
+ readRedisJson(`langfuse:v4:legacy-api-usage:v1:${secondProjectId}`),
+ ).resolves.toBeNull();
+ });
+
+ describe("worker pipeline heartbeat", () => {
+ it("serves experiment POST Redis data even when the heartbeat is stale", async () => {
+ await seedRedisCache({
+ [legacyApiUsageHeartbeatKey]: "2026-06-24T20:30:00.000Z",
+ [sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries()]),
+ [experimentPostUsageCacheKey]: experimentPostBlob(true),
+ });
+ mockedQueryClickhouse.mockResolvedValue([]);
+ const caller = createCaller();
+
+ const summary = await caller.sdkUsageSummary({
+ projectId,
+ });
+ // python 3.9.0 + POST usage → inconclusive (needs current experiment SDK).
+ expect(summary.experimentInstrumentationMigration).toEqual({
+ status: "sdk_usage_inconclusive",
+ upgradePath: "sdk",
+ });
+ expect(
+ mockedQueryClickhouse.mock.calls.some(([args]) =>
+ args.query.includes("system.query_log"),
+ ),
+ ).toBe(false);
+ });
+
+ it("treats missing entries as no usage while the heartbeat is fresh", async () => {
+ await seedRedisCache({
+ // Worker ran 30 minutes ago; it only writes entries for projects
+ // WITH usage, so absence is authoritative.
+ [legacyApiUsageHeartbeatKey]: "2026-06-25T00:00:00.000Z",
+ [sdkUsageCacheKey]: sdkUsageBlob([cachedSdkSeries()]),
+ });
+ mockedQueryClickhouse.mockResolvedValue([]);
+ const caller = createCaller();
+
+ await expect(
+ caller.legacyApiUsageSummary({
+ projectId,
+ }),
+ ).resolves.toEqual([]);
+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();
+ // The request path must not shadow the worker's entries either.
+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();
+
+ const summary = await caller.sdkUsageSummary({
+ projectId,
+ });
+ expect(summary.experimentInstrumentationMigration).toEqual({
+ status: "not_required",
+ upgradePath: null,
+ });
+ // Only the SDK live-gap query on events_core; no query_log scan.
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
+ expect(mockedQueryClickhouse.mock.calls[0]?.[0].query).toContain(
+ "FROM events_core",
+ );
+ });
+
+ it("serves legacy API Redis data even when the heartbeat is stale", async () => {
+ await seedRedisCache({
+ // 4h old: past the 3h freshness horizon — still serve Redis, no CH.
+ [legacyApiUsageHeartbeatKey]: "2026-06-24T20:30:00.000Z",
+ [legacyApiUsageCacheKey]: legacyApiBlob([
+ {
+ entrypoint: "publicapi: GET /api/public/traces",
+ count: 4,
+ lastSeen: "2026-06-24T12:00:00.000000Z",
+ },
+ ]),
+ });
+ const caller = createCaller();
+
+ const rows = await caller.legacyApiUsageSummary({
+ projectId,
+ });
+
+ expect(rows).toEqual([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces",
+ count: 4,
+ lastSeen: "2026-06-24T12:00:00.000000Z",
+ },
+ ]);
+ expect(mockedQueryClickhouse).not.toHaveBeenCalled();
+ });
});
describe("migrationActions", () => {
@@ -2084,7 +2104,7 @@ describe("v4TransitionRouter", () => {
});
it("on cold cache, runs history+gap queries, writes back, and derives the SDK flag", async () => {
- enableRedisCache();
+ await seedRedisCache();
mockedQueryClickhouse.mockImplementation(async (args) => {
if (!args.query.includes("FROM events_core")) return [];
return (args.params?.toTimestamp as string) === HOT_START_CLICKHOUSE
@@ -2116,16 +2136,24 @@ describe("v4TransitionRouter", () => {
([args]) => args.query.includes("FROM events_core"),
);
expect(eventsCoreCalls).toHaveLength(2);
- expect(
- redisMock.setex.mock.calls.some(([key]) => key === sdkUsageCacheKey),
- ).toBe(true);
+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({
+ version: 1,
+ hotStart: HOT_START_ISO,
+ series: [expect.objectContaining({ eventCount: 5 })],
+ });
+ await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();
+ await expect(
+ readRedisJson(experimentPostUsageCacheKey),
+ ).resolves.toBeNull();
});
it("on cache hit, runs the live gap query and merges for the SDK flag", async () => {
- enableRedisCache({
+ await seedRedisCache({
[sdkUsageCacheKey]: sdkUsageBlob([
cachedSdkSeries({ actionLevel: "required" }),
]),
+ [experimentPostUsageCacheKey]: experimentPostBlob(false),
+ [legacyApiUsageCacheKey]: legacyApiBlob([]),
});
mockedQueryClickhouse.mockImplementation(async (args) => {
if (!args.query.includes("FROM events_core")) return [];
@@ -2136,9 +2164,8 @@ describe("v4TransitionRouter", () => {
await expect(caller.migrationActions({ projectId })).resolves.toEqual({
forceV3Experience: false,
sdkActionNeeded: true,
- // query_log-backed categories stay unknown until the follow-up PR.
- experimentsActionNeeded: null,
- apisActionNeeded: null,
+ experimentsActionNeeded: false,
+ apisActionNeeded: false,
evalsActionNeeded: true,
exportsActionNeeded: false,
});
@@ -2151,11 +2178,15 @@ describe("v4TransitionRouter", () => {
fromTimestamp: HOT_START_CLICKHOUSE,
toTimestamp: RECENT_CUTOFF_CLICKHOUSE,
});
- expect(redisMock.setex).not.toHaveBeenCalled();
+ // Cache hit must not rewrite the historical SDK blob.
+ await expect(readRedisJson(sdkUsageCacheKey)).resolves.toMatchObject({
+ hotStart: HOT_START_ISO,
+ series: [expect.objectContaining({ eventCount: 5 })],
+ });
});
it("short-circuits partner-managed (forced v3) projects without any I/O", async () => {
- enableRedisCache();
+ await seedRedisCache();
sharedServerMock.isForceV3ExperienceProject.mockReturnValueOnce(true);
const prismaMock = mockPrismaForActions();
const caller = createCaller(prismaMock);
@@ -2170,12 +2201,12 @@ describe("v4TransitionRouter", () => {
});
expect(prismaMock.jobConfiguration.groupBy).not.toHaveBeenCalled();
- expect(redisMock.mget).not.toHaveBeenCalled();
expect(mockedQueryClickhouse).not.toHaveBeenCalled();
});
it("when Redis is unavailable, queries the full window and derives the SDK flag", async () => {
- redisMock.status = "end";
+ // Default beforeEach leaves the cache gated off (status proxy → "end").
+ redisAvailability.available = false;
mockedQueryClickhouse.mockImplementation(async (args) => {
if (!args.query.includes("FROM events_core")) return [];
return [
@@ -2204,7 +2235,6 @@ describe("v4TransitionRouter", () => {
([args]) => args.query.includes("FROM events_core"),
);
expect(eventsCoreCalls).toHaveLength(1);
- expect(redisMock.mget).not.toHaveBeenCalled();
});
});
});
diff --git a/web/src/features/v4/server/v4TransitionCache.ts b/web/src/features/v4/server/v4TransitionCache.ts
index 9751d67cbf..4ee56d056f 100644
--- a/web/src/features/v4/server/v4TransitionCache.ts
+++ b/web/src/features/v4/server/v4TransitionCache.ts
@@ -1,8 +1,18 @@
import { z } from "zod/v4";
-import { logger, redis } from "@langfuse/shared/src/server";
+import {
+ logger,
+ redis,
+ v4ExperimentPostUsageBlobSchema,
+ v4ExperimentPostUsageProjectKey,
+ v4LegacyApiUsageBlobSchema,
+ v4LegacyApiUsageProjectKey,
+ type V4ExperimentPostUsageBlob,
+ type V4LegacyApiUsageBlob,
+ type V4LegacyApiUsageRow,
+} from "@langfuse/shared/src/server";
/**
- * Redis caching for the v4 transition SDK usage check.
+ * Redis caching for the v4 transition usage checks.
*
* SDK ingestion is read from the indexed `events_core` table. A per-project
* blob covers the detection window up to an hour-aligned `hotStart` boundary;
@@ -11,9 +21,9 @@ import { logger, redis } from "@langfuse/shared/src/server";
* left-edge overcount on the sliding 14-day window (and stale classification
* fields embedded in the blob) stay bounded to ~1h.
*
- * Deprecated public API and experiment POST usage come from `system.query_log`
- * and are intentionally not cached here — that path lives in a follow-up PR
- * with a worker-maintained pipeline. Sidebar and panel SDK reads share this
+ * Deprecated public API and experiment POST usage are maintained by the
+ * worker pipeline into the shared Redis keys; the request path only reads
+ * those entries (miss = no usage). Sidebar and panel SDK reads share this
* blob + gap-fill path so both stay consistent.
*
* Helpers degrade gracefully: no Redis, Redis errors, or unparsable entries
@@ -29,7 +39,6 @@ export const MIGRATION_INGRESS_EVENT_SOURCES = [
/** Historical blob TTL: 60 minutes (1 hour). */
export const SDK_USAGE_CACHE_TTL_SECONDS = 60 * 60;
-
const sdkUsageSeriesSchema = z.object({
source: z.enum(MIGRATION_INGRESS_EVENT_SOURCES),
ingestionPath: z.enum(["otel", "ingestion_api"]),
@@ -71,8 +80,14 @@ const sdkUsageBlobSchema = z.object({
export type SdkUsageCacheBlob = z.infer<typeof sdkUsageBlobSchema>;
+export type CachedLegacyApiUsageRow = V4LegacyApiUsageRow;
+export type LegacyApiUsageCacheBlob = V4LegacyApiUsageBlob;
+export type ExperimentPostUsageCacheBlob = V4ExperimentPostUsageBlob;
+
const sdkUsageKey = (projectId: string) =>
`langfuse:v4:sdk-usage:v1:${projectId}`;
+const legacyApiUsageKey = v4LegacyApiUsageProjectKey;
+const experimentPostUsageKey = v4ExperimentPostUsageProjectKey;
export const isV4TransitionCacheAvailable = (): boolean =>
redis != null && redis.status !== "end" && redis.status !== "close";
@@ -142,6 +157,19 @@ export const writeSdkUsageCache = (
})),
);
+export const readLegacyApiUsageCache = (
+ projectIds: string[],
+): Promise<(LegacyApiUsageCacheBlob | null)[]> =>
+ readBlobs(projectIds.map(legacyApiUsageKey), v4LegacyApiUsageBlobSchema);
+
+export const readExperimentPostUsageCache = (
+ projectIds: string[],
+): Promise<(ExperimentPostUsageCacheBlob | null)[]> =>
+ readBlobs(
+ projectIds.map(experimentPostUsageKey),
+ v4ExperimentPostUsageBlobSchema,
+ );
+
const MINUTE_MS = 60 * 1000;
const HOUR_MS = 60 * MINUTE_MS;
diff --git a/web/src/features/v4/server/v4TransitionQueryLogUsage.ts b/web/src/features/v4/server/v4TransitionQueryLogUsage.ts
index e2b7f37923..03c3ece75d 100644
--- a/web/src/features/v4/server/v4TransitionQueryLogUsage.ts
+++ b/web/src/features/v4/server/v4TransitionQueryLogUsage.ts
@@ -1,31 +1,18 @@
/**
- * ClickHouse system.query_log helpers for v4 transition checks
- * (legacy public API usage and experiment POST /dataset-run-items).
+ * Redis-backed v4 transition usage reads for deprecated public API and
+ * experiment POST /dataset-run-items checks.
*
- * Legacy API stays on a direct query_log read (no Redis worker yet). The
- * public summary uses the server detection window so behavior matches the
- * pre-split service; internal helpers still accept from/to for reuse.
+ * Both paths are worker-maintained Redis only (including when the pipeline
+ * heartbeat is stale). A missing entry means no usage; the request path never
+ * scans `system.query_log`.
*/
-import { env } from "@langfuse/shared/src/env";
import {
- convertDateToClickhouseDateTime,
- logger,
- queryClickhouse,
- systemTableRef,
- type PreferredClickhouseService,
-} from "@langfuse/shared/src/server";
-import {
- getV4TransitionDetectionWindow,
+ readExperimentPostUsageCache,
+ readLegacyApiUsageCache,
V4_TRANSITION_DETECTION_WINDOW_MS,
+ type CachedLegacyApiUsageRow,
} from "@/src/features/v4/server/v4TransitionCache";
-type LegacyApiUsageSummaryByProjectRow = {
- projectId: string;
- entrypoint: string;
- count: string | number;
- lastSeen: string;
-};
-
export type LegacyApiUsageSummaryByProjectResultRow = {
projectId: string;
entrypoint: string;
@@ -33,64 +20,6 @@ export type LegacyApiUsageSummaryByProjectResultRow = {
lastSeen: string;
};
-export type DatasetRunItemsPostUsageByProjectRow = {
- projectId: string;
- count: string | number;
-};
-
-type SystemQueryLogReadService = "ReadOnly" | "EventsReadOnly";
-
-const getSystemQueryLogServices = ({
- readService,
- readUrl,
-}: {
- readService: SystemQueryLogReadService;
- readUrl: string | undefined;
-}): PreferredClickhouseService[] =>
- readUrl &&
- new URL(readUrl).toString() !== new URL(env.CLICKHOUSE_URL).toString()
- ? [readService, "ReadWrite"]
- : ["ReadWrite"];
-
-const querySystemQueryLogAcrossServices = async <T>({
- preferredClickhouseServices,
- queryService,
- failureMessage,
-}: {
- preferredClickhouseServices: PreferredClickhouseService[];
- queryService: (service: PreferredClickhouseService) => Promise<T[]>;
- failureMessage: string;
-}): Promise<T[][]> => {
- const settledServiceRows = await Promise.allSettled(
- preferredClickhouseServices.map(queryService),
- );
- const firstSuccessfulResult = settledServiceRows.find(
- (result) => result.status === "fulfilled",
- );
- if (!firstSuccessfulResult) {
- const firstFailure = settledServiceRows.find(
- (result) => result.status === "rejected",
- );
- throw firstFailure?.reason ?? new Error("ClickHouse services unavailable");
- }
-
- settledServiceRows.forEach((result, index) => {
- if (result.status === "rejected") {
- logger.warn(failureMessage, {
- preferredClickhouseService: preferredClickhouseServices[index],
- error:
- result.reason instanceof Error
- ? result.reason.message
- : "Unknown error",
- });
- }
- });
-
- return settledServiceRows.flatMap((result) =>
- result.status === "fulfilled" ? [result.value] : [],
- );
-};
-
/**
* Read-time aging-out rule: drop entries whose last activity left the trailing
* detection window (same rule as the SDK path).
@@ -98,298 +27,71 @@ const querySystemQueryLogAcrossServices = async <T>({
const isWithinDetectionWindow = (lastSeen: string, nowMs: number): boolean =>
Date.parse(lastSeen) >= nowMs - V4_TRANSITION_DETECTION_WINDOW_MS;
-export type DatasetRunItemsPostUsageResult =
- | {
- status: "success";
- rows: DatasetRunItemsPostUsageByProjectRow[];
- }
- | {
- status: "error";
- rows: DatasetRunItemsPostUsageByProjectRow[];
- };
-
-/** POST /api/public/dataset-run-items usage from system.query_log. */
-export const getDatasetRunItemsPostUsageByProject = async ({
- projectIds,
- fromTimestamp,
- toTimestamp,
-}: {
- projectIds: string[];
- fromTimestamp: Date;
- toTimestamp: Date;
-}): Promise<DatasetRunItemsPostUsageResult> => {
- if (projectIds.length === 0) {
- return { status: "success", rows: [] };
- }
-
- const systemQueryLogServices = getSystemQueryLogServices({
- readService: "EventsReadOnly",
- readUrl:
- env.CLICKHOUSE_EVENTS_READ_ONLY_URL ?? env.CLICKHOUSE_READ_ONLY_URL,
- });
-
- try {
- const serviceRows = await querySystemQueryLogAcrossServices({
- preferredClickhouseServices: systemQueryLogServices,
- queryService: (preferredClickhouseService) =>
- queryClickhouse<DatasetRunItemsPostUsageByProjectRow>({
- query: `
-SELECT
- JSONExtractString(log_comment, 'projectId') AS projectId,
- count() AS count
-FROM ${systemTableRef("system.query_log")}
-WHERE
- event_time >= {fromTimestamp: DateTime64(3)}
- AND event_time <= {toTimestamp: DateTime64(3)}
- AND event_date >= toDate({fromTimestamp: DateTime64(3)})
- AND event_date <= toDate({toTimestamp: DateTime64(3)})
- AND type = 'QueryFinish'
- AND JSONExtractString(log_comment, 'tag_schema_version') = '1'
- AND JSONExtractString(log_comment, 'surface') = 'publicapi'
- AND JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}
- AND splitByChar('?', JSONExtractString(log_comment, 'route'))[1] = 'POST /api/public/dataset-run-items'
-GROUP BY projectId
-SETTINGS skip_unavailable_shards = 1
- `,
- params: {
- projectIds,
- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),
- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
- },
- tags: { route: "v4-experiment-instrumentation-summary" },
- preferredClickhouseService,
- clickhouseSettings: { skip_unavailable_shards: 1 },
- }),
- failureMessage:
- "Failed to query dataset-run-items POST usage from ClickHouse service",
- });
- return {
- status: "success",
- rows: serviceRows.flat(),
- };
- } catch (error: unknown) {
- logger.warn(
- "Failed to query dataset-run-items POST usage for v4 migration",
- error,
- );
- return {
- status: "error",
- rows: [],
- };
- }
-};
+export const trimLegacyApiUsageRows = (
+ rows: CachedLegacyApiUsageRow[],
+ nowMs: number,
+): CachedLegacyApiUsageRow[] =>
+ rows.filter((row) => isWithinDetectionWindow(row.lastSeen, nowMs));
-const getLegacyApiUsageSummariesForService = async ({
+/**
+ * Whether each project called `POST /api/public/dataset-run-items` within the
+ * detection window. Redis-only: serve worker-maintained entries (including
+ * when the heartbeat is stale) and treat a miss as no usage. No `query_log`
+ * fallback on the request path.
+ */
+export const getExperimentPostUsageByProject = async ({
projectIds,
- fromTimestamp,
- toTimestamp,
- preferredClickhouseService,
+ nowMs,
}: {
projectIds: string[];
- fromTimestamp: Date;
- toTimestamp: Date;
- preferredClickhouseService: PreferredClickhouseService;
-}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {
- if (projectIds.length === 0) return [];
-
- const rows = await queryClickhouse<LegacyApiUsageSummaryByProjectRow>({
- query: `
-WITH selected AS (
- SELECT
- JSONExtractString(log_comment, 'projectId') AS project_id,
- event_time_microseconds,
- splitByChar('?', JSONExtractString(log_comment, 'route'))[1] AS route_path
- FROM ${systemTableRef("system.query_log")}
- WHERE
- event_time >= {fromTimestamp: DateTime64(3)}
- AND event_time <= {toTimestamp: DateTime64(3)}
- AND event_date >= toDate({fromTimestamp: DateTime64(3)})
- AND event_date <= toDate({toTimestamp: DateTime64(3)})
- AND type = 'QueryFinish'
- AND JSONExtractString(log_comment, 'tag_schema_version') = '1'
- AND JSONExtractString(log_comment, 'surface') = 'publicapi'
- AND JSONExtractString(log_comment, 'projectId') IN {projectIds: Array(String)}
-),
-classified AS (
- SELECT
- project_id,
- event_time_microseconds,
- multiIf(
- route_path IN (
- 'GET /api/public/spans',
- 'GET /api/public/generations',
- 'GET /api/public/traces',
- 'GET /api/public/sessions',
- 'GET /api/public/observations',
- 'GET /api/public/scores',
- 'GET /api/public/v2/scores',
- 'GET /api/public/metrics',
- 'GET /api/public/metrics/daily',
- 'GET /api/public/dataset-run-items'
- ), route_path,
- match(route_path, '^GET /api/public/traces/[^/?#]+$'), 'GET /api/public/traces/{id}',
- match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 'GET /api/public/sessions/{id}',
- match(route_path, '^GET /api/public/observations/[^/?#]+$'), 'GET /api/public/observations/{id}',
- match(route_path, '^GET /api/public/scores/[^/?#]+$'), 'GET /api/public/scores/{id}',
- match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 'GET /api/public/v2/scores/{id}',
- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs',
- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 'GET /api/public/datasets/{datasetName}/runs/{runName}',
- NULL
- ) AS legacy_route,
- multiIf(
- route_path IN (
- 'GET /api/public/spans',
- 'GET /api/public/generations',
- 'GET /api/public/traces',
- 'GET /api/public/observations',
- 'GET /api/public/scores',
- 'GET /api/public/v2/scores',
- 'GET /api/public/metrics/daily',
- 'GET /api/public/dataset-run-items'
- ), 2,
- route_path IN (
- 'GET /api/public/sessions',
- 'GET /api/public/metrics'
- ), 1,
- match(route_path, '^GET /api/public/traces/[^/?#]+$'), 3,
- match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 1,
- match(route_path, '^GET /api/public/observations/[^/?#]+$'), 1,
- match(route_path, '^GET /api/public/scores/[^/?#]+$'), 1,
- match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 1,
- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 1,
- match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1,
- NULL
- ) AS clickhouse_queries_per_api_call
- FROM selected
-)
-
-SELECT
- project_id AS projectId,
- concat('publicapi: ', legacy_route) AS entrypoint,
- sum(1.0 / clickhouse_queries_per_api_call) AS count,
- formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen
-FROM classified
-WHERE legacy_route IS NOT NULL
- AND clickhouse_queries_per_api_call IS NOT NULL
-GROUP BY project_id, legacy_route
-ORDER BY project_id ASC, legacy_route ASC
-SETTINGS skip_unavailable_shards = 1
- `,
- params: {
- projectIds,
- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),
- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
- },
- tags: { route: "v4-legacy-api-usage-summary" },
- preferredClickhouseService,
- clickhouseSettings: { skip_unavailable_shards: 1 },
+ nowMs: number;
+}): Promise<Map<string, boolean | "check_failed">> => {
+ const usageByProject = new Map<string, boolean | "check_failed">();
+ const cachedBlobs = await readExperimentPostUsageCache(projectIds);
+ projectIds.forEach((projectId, index) => {
+ const blob = cachedBlobs[index];
+ if (!blob) {
+ usageByProject.set(projectId, false);
+ return;
+ }
+ const stillInWindow =
+ blob.lastSeen == null
+ ? blob.used
+ : blob.used && isWithinDetectionWindow(blob.lastSeen, nowMs);
+ usageByProject.set(projectId, stillInWindow);
});
-
- return rows.map((row) => ({
- projectId: row.projectId,
- entrypoint: row.entrypoint,
- count: Number(row.count),
- lastSeen: row.lastSeen,
- }));
+ return usageByProject;
};
-const queryLegacyApiUsageSummaries = async ({
+/**
+ * Deprecated public API usage per project. Redis-only: serve worker-maintained
+ * entries (including when the heartbeat is stale) and treat a miss as no
+ * usage. The always-mounted sidebar never calls this procedure.
+ */
+export const getLegacyApiUsageSummaries = async ({
projectIds,
- fromTimestamp,
- toTimestamp,
}: {
projectIds: string[];
- fromTimestamp: Date;
- toTimestamp: Date;
}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {
if (projectIds.length === 0) return [];
- const preferredClickhouseServices = getSystemQueryLogServices({
- readService: "ReadOnly",
- readUrl: env.CLICKHOUSE_READ_ONLY_URL,
- });
- const serviceResultSets = await querySystemQueryLogAcrossServices({
- preferredClickhouseServices,
- queryService: (preferredClickhouseService) =>
- getLegacyApiUsageSummariesForService({
- projectIds,
- fromTimestamp,
- toTimestamp,
- preferredClickhouseService,
- }),
- failureMessage: "Failed to query legacy API usage from ClickHouse service",
- });
-
- const uniqueServiceResultSets = new Map<
- string,
- LegacyApiUsageSummaryByProjectResultRow[]
- >();
- for (const rows of serviceResultSets) {
- const sortedRows = [...rows].sort(
- (left, right) =>
- left.projectId.localeCompare(right.projectId) ||
- left.entrypoint.localeCompare(right.entrypoint),
- );
- uniqueServiceResultSets.set(JSON.stringify(sortedRows), sortedRows);
- }
- const deduplicatedServiceRows = Array.from(
- uniqueServiceResultSets.values(),
- ).flat();
-
- const rowsByProjectAndEntrypoint = new Map<
- string,
- LegacyApiUsageSummaryByProjectResultRow
- >();
-
- for (const row of deduplicatedServiceRows) {
- const key = `${row.projectId}\u0000${row.entrypoint}`;
- const existing = rowsByProjectAndEntrypoint.get(key);
- rowsByProjectAndEntrypoint.set(
- key,
- existing
- ? {
- ...existing,
- count: existing.count + row.count,
- lastSeen:
- existing.lastSeen > row.lastSeen
- ? existing.lastSeen
- : row.lastSeen,
- }
- : row,
+ const nowMs = Date.now();
+ const cachedBlobs = await readLegacyApiUsageCache(projectIds);
+ const rows: LegacyApiUsageSummaryByProjectResultRow[] = [];
+ projectIds.forEach((projectId, index) => {
+ const blob = cachedBlobs[index];
+ if (!blob) return;
+ rows.push(
+ ...trimLegacyApiUsageRows(blob.rows, nowMs).map((row) => ({
+ projectId,
+ ...row,
+ })),
);
- }
+ });
- return Array.from(rowsByProjectAndEntrypoint.values()).sort(
+ return rows.sort(
(left, right) =>
left.projectId.localeCompare(right.projectId) ||
left.entrypoint.localeCompare(right.entrypoint),
);
};
-
-/**
- * Deprecated public API usage per project. Reads `system.query_log` directly;
- * Redis caching and the worker pipeline for this path land in a follow-up PR.
- * The always-mounted sidebar never calls this procedure.
- */
-export const getLegacyApiUsageSummaries = async ({
- projectIds,
-}: {
- projectIds: string[];
-}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {
- if (projectIds.length === 0) return [];
-
- const nowMs = Date.now();
- const { windowStart, windowEnd } = getV4TransitionDetectionWindow(nowMs);
- const rows = await queryLegacyApiUsageSummaries({
- projectIds,
- fromTimestamp: windowStart,
- toTimestamp: windowEnd,
- });
- return rows
- .filter((row) => isWithinDetectionWindow(row.lastSeen, nowMs))
- .sort(
- (left, right) =>
- left.projectId.localeCompare(right.projectId) ||
- left.entrypoint.localeCompare(right.entrypoint),
- );
-};
diff --git a/web/src/features/v4/server/v4TransitionRouter.ts b/web/src/features/v4/server/v4TransitionRouter.ts
index 8014ffd1d1..0c18a6f803 100644
--- a/web/src/features/v4/server/v4TransitionRouter.ts
+++ b/web/src/features/v4/server/v4TransitionRouter.ts
@@ -134,7 +134,7 @@ export const v4TransitionRouter = createTRPCRouter({
* Migration signal for always-mounted UI (the sidebar "Action required"
* pill). Postgres for eval/export signals; SDK via the same Redis + live
* gap-fill path as sdkUsageSummary. Deprecated API / experiment signals
- * stay `null` until the follow-up query_log worker pipeline lands.
+ * read worker-maintained Redis entries when available (miss = unknown/none).
*/
migrationActions: protectedProjectProcedure
.input(z.object({ projectId: z.string() }))
diff --git a/web/src/features/v4/server/v4TransitionSdkUsage.ts b/web/src/features/v4/server/v4TransitionSdkUsage.ts
index e4cf70442a..c53c5dba61 100644
--- a/web/src/features/v4/server/v4TransitionSdkUsage.ts
+++ b/web/src/features/v4/server/v4TransitionSdkUsage.ts
@@ -4,7 +4,8 @@
* Historical series come from a per-project Redis blob; the remaining gap up
* to a minute-aligned recentCutoff is queried live from events_core (with
* ClickHouse use_query_cache). Detection windows are server-side — no client
- * from/to. Experiment POST usage is delegated to the query_log module.
+ * from/to. Experiment POST usage is read from worker-maintained Redis only
+ * (miss = no usage).
*/
import {
convertDateToClickhouseDateTime,
@@ -13,7 +14,7 @@ import {
type IngestionSdkAttributionStatus,
} from "@langfuse/shared/src/server";
import { getSdkVersionCapabilityStatus } from "@/src/features/sdk-version/lib/sdkVersionCapabilities";
-import { getDatasetRunItemsPostUsageByProject } from "@/src/features/v4/server/v4TransitionQueryLogUsage";
+import { getExperimentPostUsageByProject } from "@/src/features/v4/server/v4TransitionQueryLogUsage";
import {
getV4TransitionDetectionWindow,
isV4TransitionCacheAvailable,
@@ -472,48 +473,7 @@ export const getSdkUsageSeriesByProject = async ({
);
};
-/**
- * Whether each project called `POST /api/public/dataset-run-items` within the
- * detection window. Reads `system.query_log` directly; Redis caching for this
- * path lands in a follow-up PR with the worker pipeline.
- */
-const getExperimentPostUsageByProject = async ({
- projectIds,
- nowMs,
-}: {
- projectIds: string[];
- nowMs: number;
-}): Promise<Map<string, boolean | "check_failed">> => {
- const { windowStart, windowEnd } = getV4TransitionDetectionWindow(nowMs);
- const usageByProject = new Map<string, boolean | "check_failed">();
-
- const queryResult = await getDatasetRunItemsPostUsageByProject({
- projectIds,
- fromTimestamp: windowStart,
- toTimestamp: windowEnd,
- });
- if (queryResult.status === "error") {
- projectIds.forEach((projectId) =>
- usageByProject.set(projectId, "check_failed"),
- );
- return usageByProject;
- }
-
- const projectsUsingDatasetRunItemsPost = new Set(
- queryResult.rows
- .filter((row) => Number(row.count) > 0)
- .map((row) => row.projectId),
- );
- projectIds.forEach((projectId) =>
- usageByProject.set(
- projectId,
- projectsUsingDatasetRunItemsPost.has(projectId),
- ),
- );
- return usageByProject;
-};
-
-const deriveExperimentInstrumentationMigration = ({
+export const deriveExperimentInstrumentationMigration = ({
sdkUsageSeries,
postUsage,
}: {
diff --git a/web/src/features/v4/server/v4TransitionService.ts b/web/src/features/v4/server/v4TransitionService.ts
index eca01fd860..4812612072 100644
--- a/web/src/features/v4/server/v4TransitionService.ts
+++ b/web/src/features/v4/server/v4TransitionService.ts
@@ -10,7 +10,15 @@ import {
} from "@langfuse/shared/src/db";
import type { Session } from "next-auth";
import { isForceV3ExperienceProject } from "@langfuse/shared/src/server";
-import { getSdkUsageSeriesByProject } from "@/src/features/v4/server/v4TransitionSdkUsage";
+import {
+ readExperimentPostUsageCache,
+ readLegacyApiUsageCache,
+} from "@/src/features/v4/server/v4TransitionCache";
+import { trimLegacyApiUsageRows } from "@/src/features/v4/server/v4TransitionQueryLogUsage";
+import {
+ deriveExperimentInstrumentationMigration,
+ getSdkUsageSeriesByProject,
+} from "@/src/features/v4/server/v4TransitionSdkUsage";
export { getSdkUsageSummaries } from "@/src/features/v4/server/v4TransitionSdkUsage";
export { getLegacyApiUsageSummaries } from "@/src/features/v4/server/v4TransitionQueryLogUsage";
@@ -209,7 +217,7 @@ export const getTraceLevelEvalSummaries = async ({
* Migration signals for the always-mounted sidebar "Action required" pill.
* SDK uses the same Redis + live gap-fill path as the panel summaries so the
* pill cannot go stale relative to them. Deprecated API / experiment signals
- * stay `null` until the follow-up query_log worker pipeline lands.
+ * read worker-maintained Redis caches when available (`null` = unknown).
*/
export const getMigrationActions = async ({
prisma,
@@ -234,22 +242,46 @@ export const getMigrationActions = async ({
const nowMs = Date.now();
const projectIds = [projectId];
- const [evalSummaries, integrationSummaries, seriesByProject] =
- await Promise.all([
- getTraceLevelEvalSummaries({ prisma, projectIds }),
- getLegacyIntegrationSummaries({ prisma, projectIds }),
- getSdkUsageSeriesByProject({ projectIds, nowMs }),
- ]);
+ const [
+ evalSummaries,
+ integrationSummaries,
+ seriesByProject,
+ postBlobs,
+ apiBlobs,
+ ] = await Promise.all([
+ getTraceLevelEvalSummaries({ prisma, projectIds }),
+ getLegacyIntegrationSummaries({ prisma, projectIds }),
+ getSdkUsageSeriesByProject({ projectIds, nowMs }),
+ readExperimentPostUsageCache(projectIds),
+ readLegacyApiUsageCache(projectIds),
+ ]);
const sdkUsageSeries = seriesByProject.get(projectId) ?? [];
+ const postBlob = postBlobs[0] ?? null;
+ const apiBlob = apiBlobs[0] ?? null;
+
+ const experimentsActionNeeded =
+ postBlob === null
+ ? null
+ : postBlob.used === false
+ ? false
+ : ["required", "sdk_usage_inconclusive"].includes(
+ deriveExperimentInstrumentationMigration({
+ sdkUsageSeries,
+ postUsage: true,
+ }).status,
+ );
return {
forceV3Experience: isForceV3ExperienceProject(projectId),
sdkActionNeeded: sdkUsageSeries.some(
(series) => series.actionLevel === "required",
),
- experimentsActionNeeded: null,
- apisActionNeeded: null,
+ experimentsActionNeeded,
+ apisActionNeeded:
+ apiBlob === null
+ ? null
+ : trimLegacyApiUsageRows(apiBlob.rows, nowMs).length > 0,
evalsActionNeeded: (evalSummaries[0]?.traceLevelEvalCount ?? 0) > 0,
exportsActionNeeded:
(integrationSummaries[0]?.legacyIntegrationCount ?? 0) > 0,
diff --git a/worker/src/app.ts b/worker/src/app.ts
index 7b53d13d16..2573516437 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -40,6 +40,7 @@ import {
TraceUpsertQueue,
CloudFreeTierUsageThresholdQueue,
CloudUsageMeteringQueue,
+ V4LegacyApiUsageQueue,
EventPropagationQueue,
EvalExecutionQueue,
SecondaryEvalExecutionQueue,
@@ -60,6 +61,7 @@ import {
postHogIntegrationProcessingProcessor,
postHogIntegrationProcessor,
} from "./queues/postHogIntegrationQueue";
+import { v4LegacyApiUsageProcessor } from "./queues/v4LegacyApiUsageQueue";
import {
mixpanelIntegrationProcessingProcessor,
mixpanelIntegrationProcessor,
@@ -524,6 +526,25 @@ if (env.QUEUE_CONSUMER_POSTHOG_INTEGRATION_QUEUE_IS_ENABLED === "true") {
);
}
+if (env.QUEUE_CONSUMER_V4_LEGACY_API_USAGE_QUEUE_IS_ENABLED === "true") {
+ // Instantiate the queue to trigger scheduled jobs
+ V4LegacyApiUsageQueue.getInstance();
+
+ WorkerManager.register(
+ QueueName.V4LegacyApiUsageQueue,
+ v4LegacyApiUsageProcessor,
+ {
+ concurrency: 1,
+ limiter: {
+ // The job scans system.query_log across all ClickHouse services;
+ // never run it more than once per minute even if jobs pile up.
+ max: 1,
+ duration: 60_000,
+ },
+ },
+ );
+}
+
if (env.QUEUE_CONSUMER_MIXPANEL_INTEGRATION_QUEUE_IS_ENABLED === "true") {
// Instantiate the queue to trigger scheduled jobs
MixpanelIntegrationQueue.getInstance();
diff --git a/worker/src/env.ts b/worker/src/env.ts
index b6be7c67ce..3ef360d2c7 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -293,6 +293,9 @@ const EnvSchema = z.object({
QUEUE_CONSUMER_POSTHOG_INTEGRATION_QUEUE_IS_ENABLED: z
.enum(["true", "false"])
.default("true"),
+ QUEUE_CONSUMER_V4_LEGACY_API_USAGE_QUEUE_IS_ENABLED: z
+ .enum(["true", "false"])
+ .default("true"),
QUEUE_CONSUMER_MIXPANEL_INTEGRATION_QUEUE_IS_ENABLED: z
.enum(["true", "false"])
.default("true"),
diff --git a/worker/src/features/queue-metrics-runner/index.test.ts b/worker/src/features/queue-metrics-runner/index.test.ts
index 0235f53f8a..19fafb67bd 100644
--- a/worker/src/features/queue-metrics-runner/index.test.ts
+++ b/worker/src/features/queue-metrics-runner/index.test.ts
@@ -5,6 +5,7 @@ const mocks = vi.hoisted(() => ({
getJobCounts: vi.fn(),
recordGauge: vi.fn(),
updateActiveIngestFailureProjectsMetric: vi.fn(),
+ emitV4LegacyApiUsageFreshnessMetrics: vi.fn(),
}));
vi.mock("@langfuse/shared/src/server", () => ({
@@ -50,6 +51,11 @@ vi.mock("../../queues/shardedQueueRegistry", () => ({
})),
}));
+vi.mock("../v4/v4LegacyApiUsageMetrics", () => ({
+ emitV4LegacyApiUsageFreshnessMetrics:
+ mocks.emitV4LegacyApiUsageFreshnessMetrics,
+}));
+
import { QueueMetricsRunner } from ".";
class TestQueueMetricsRunner extends QueueMetricsRunner {
@@ -69,6 +75,7 @@ describe("QueueMetricsRunner", () => {
waiting: 7,
});
mocks.updateActiveIngestFailureProjectsMetric.mockResolvedValue(undefined);
+ mocks.emitV4LegacyApiUsageFreshnessMetrics.mockResolvedValue(undefined);
});
it("emits only the canonical gauges for a non-sharded queue", async () => {
diff --git a/worker/src/features/queue-metrics-runner/index.ts b/worker/src/features/queue-metrics-runner/index.ts
index df0b01ecf8..40241c472c 100644
--- a/worker/src/features/queue-metrics-runner/index.ts
+++ b/worker/src/features/queue-metrics-runner/index.ts
@@ -16,6 +16,7 @@ import {
SHARDED_QUEUE_BASE_NAMES,
resolveQueueInstance,
} from "../../queues/shardedQueueRegistry";
+import { emitV4LegacyApiUsageFreshnessMetrics } from "../v4/v4LegacyApiUsageMetrics";
type DepthType = "waiting" | "failed" | "active";
@@ -83,6 +84,23 @@ export class QueueMetricsRunner extends PeriodicRunner {
}),
);
+ // Heartbeat/cursor freshness for the v4 legacy API usage pipeline. Only
+ // when this worker consumes that queue so self-hosted deployments without
+ // the job do not emit a permanent "not fresh" signal.
+ if (registeredNames.has(QueueName.V4LegacyApiUsageQueue)) {
+ promises.push(
+ emitV4LegacyApiUsageFreshnessMetrics()
+ .then(() => undefined)
+ .catch((err) => {
+ this.markRunFailed(err);
+ logger.error(
+ "Queue metrics: failed to record v4 legacy API usage freshness",
+ err,
+ );
+ }),
+ );
+ }
+
// Non-sharded queues: only poll queues with registered workers
for (const queueName of Object.values(QueueName)) {
if (SHARDED_QUEUE_BASE_NAMES.has(queueName)) continue;
diff --git a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
new file mode 100644
index 0000000000..881ed0de00
--- /dev/null
+++ b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
@@ -0,0 +1,432 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ listV4LegacyApiHourStarts,
+ v4ExperimentPostUsageProjectKey,
+ v4LegacyApiHourBucketKey,
+ v4LegacyApiUsageProjectKey,
+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,
+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,
+ V4_LEGACY_API_USAGE_LOCK_KEY,
+} from "@langfuse/shared/src/server";
+
+/**
+ * In-memory Redis fake covering the commands used by the handler and
+ * RedisLock: get/set(NX)/setex/mget/eval(check-and-delete).
+ */
+const redisState = vi.hoisted(() => ({
+ store: new Map<string, string>(),
+}));
+
+const redisMock = vi.hoisted(() => ({
+ get: vi.fn(async (key: string) => redisState.store.get(key) ?? null),
+ set: vi.fn(
+ async (
+ key: string,
+ value: string,
+ _ex?: string,
+ _ttl?: number,
+ nx?: string,
+ ) => {
+ if (nx === "NX" && redisState.store.has(key)) return null;
+ redisState.store.set(key, value);
+ return "OK";
+ },
+ ),
+ setex: vi.fn(async (key: string, _ttl: number, value: string) => {
+ redisState.store.set(key, value);
+ return "OK";
+ }),
+ mget: vi.fn(async (keys: string[]) =>
+ keys.map((key) => redisState.store.get(key) ?? null),
+ ),
+ eval: vi.fn(async (_script: string, _numKeys: number, ...args: string[]) => {
+ const [key, value] = args;
+ if (redisState.store.get(key) === value) {
+ redisState.store.delete(key);
+ return 1;
+ }
+ return 0;
+ }),
+ del: vi.fn(async (...keys: string[]) => {
+ let deleted = 0;
+ for (const key of keys) {
+ if (redisState.store.delete(key)) deleted += 1;
+ }
+ return deleted;
+ }),
+}));
+
+const mocks = vi.hoisted(() => ({
+ queryClickhouse: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
+ const original =
+ await importOriginal<typeof import("@langfuse/shared/src/server")>();
+ return {
+ ...original,
+ redis: redisMock,
+ logger: {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ },
+ queryClickhouse: mocks.queryClickhouse,
+ systemTableRef: (table: string) =>
+ `clusterAllReplicas('test-cluster', '${table}')`,
+ convertDateToClickhouseDateTime: (date: Date) =>
+ date.toISOString().replace("T", " ").replace("Z", ""),
+ };
+});
+
+vi.mock("@langfuse/shared/src/env", () => ({
+ env: {
+ CLICKHOUSE_URL: "https://clickhouse-main.example.com",
+ CLICKHOUSE_READ_ONLY_URL: "https://clickhouse-read.example.com",
+ CLICKHOUSE_EVENTS_READ_ONLY_URL: undefined,
+ },
+}));
+
+import { handleV4LegacyApiUsageJob } from "../handleV4LegacyApiUsageJob";
+
+// 10:30 UTC: a regular run (the deep re-scan happens at 03:xx UTC only).
+const TEST_NOW = new Date("2026-06-25T10:30:00Z");
+const CURRENT_HOUR_ISO = "2026-06-25T10:00:00Z";
+// floor(now - 14d) to the hour.
+const COVERAGE_START_CLICKHOUSE = "2026-06-11 10:00:00.000";
+
+const usageRow = (overrides: {
+ hourStart?: string;
+ projectId?: string;
+ route?: string;
+ count?: string | number;
+ lastSeen?: string;
+}) => ({
+ hourStart: overrides.hourStart ?? "2026-06-25T09:00:00Z",
+ projectId: overrides.projectId ?? "project-a",
+ route: overrides.route ?? "GET /api/public/traces",
+ count: overrides.count ?? "4",
+ lastSeen: overrides.lastSeen ?? "2026-06-25T09:15:00.000000Z",
+});
+
+const readJson = (key: string): unknown => {
+ const value = redisState.store.get(key);
+ return value ? JSON.parse(value) : null;
+};
+
+const emptyBucket = () =>
+ JSON.stringify({
+ version: 1,
+ computedAt: "2026-06-24T00:00:00.000Z",
+ apiRows: [],
+ experimentPostRows: [],
+ });
+
+/** Seeds a valid (empty) bucket for every hour of the trailing window. */
+const seedCoverageBuckets = (skipHourIsos: string[] = []) => {
+ const skip = new Set(skipHourIsos.map((iso) => Date.parse(iso)));
+ for (const hourStartMs of listV4LegacyApiHourStarts(
+ Date.parse("2026-06-11T10:00:00Z"),
+ Date.parse(CURRENT_HOUR_ISO),
+ )) {
+ if (skip.has(hourStartMs)) continue;
+ redisState.store.set(v4LegacyApiHourBucketKey(hourStartMs), emptyBucket());
+ }
+};
+
+/** Marks the deep re-scan as recently done so runs use the 3h margin. */
+const seedRecentDeepRescan = () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,
+ "2026-06-25T05:00:00.000Z",
+ );
+};
+
+const bucketKeyCount = (): number =>
+ Array.from(redisState.store.keys()).filter((key) =>
+ key.startsWith("langfuse:v4:legacy-api-usage:hour:v1:"),
+ ).length;
+
+describe("handleV4LegacyApiUsageJob", () => {
+ beforeEach(() => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(TEST_NOW);
+ redisState.store.clear();
+ mocks.queryClickhouse.mockResolvedValue([]);
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.clearAllMocks();
+ });
+
+ it("cold start: scans the full 14-day window once and materializes buckets, rollups, cursor, and heartbeat", async () => {
+ const rows = [
+ usageRow({}),
+ usageRow({
+ hourStart: CURRENT_HOUR_ISO,
+ projectId: "project-b",
+ route: "POST /api/public/dataset-run-items",
+ count: "2",
+ lastSeen: "2026-06-25T10:05:00.000000Z",
+ }),
+ ];
+ // Both services return identical result sets (overlapping replicas):
+ // they must be deduplicated, not summed.
+ mocks.queryClickhouse.mockResolvedValue(rows);
+
+ await handleV4LegacyApiUsageJob();
+
+ expect(mocks.queryClickhouse).toHaveBeenCalledTimes(2);
+ const services = mocks.queryClickhouse.mock.calls.map(
+ ([args]) => args.preferredClickhouseService,
+ );
+ expect(services).toEqual(["ReadWrite", "ReadOnly"]);
+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({
+ fromTimestamp: COVERAGE_START_CLICKHOUSE,
+ toTimestamp: "2026-06-25 10:30:00.000",
+ });
+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].query).toContain(
+ "toStartOfHour(event_time, 'UTC')",
+ );
+
+ // One bucket per hour in the window, empty ones included: 14*24 + 1.
+ expect(bucketKeyCount()).toBe(337);
+ expect(
+ readJson(v4LegacyApiHourBucketKey(Date.parse("2026-06-25T09:00:00Z"))),
+ ).toMatchObject({
+ version: 1,
+ apiRows: [
+ {
+ projectId: "project-a",
+ entrypoint: "publicapi: GET /api/public/traces",
+ count: 4,
+ lastSeen: "2026-06-25T09:15:00.000000Z",
+ },
+ ],
+ experimentPostRows: [],
+ });
+ expect(
+ readJson(v4LegacyApiHourBucketKey(Date.parse(CURRENT_HOUR_ISO))),
+ ).toMatchObject({
+ apiRows: [],
+ experimentPostRows: [
+ {
+ projectId: "project-b",
+ count: 2,
+ lastSeen: "2026-06-25T10:05:00.000000Z",
+ },
+ ],
+ });
+
+ // Consumer-facing entries: identical service sets deduplicated (count 4,
+ // not 8); POST usage becomes an experiment entry, not an API row.
+ expect(readJson(v4LegacyApiUsageProjectKey("project-a"))).toMatchObject({
+ version: 1,
+ rows: [
+ {
+ entrypoint: "publicapi: GET /api/public/traces",
+ count: 4,
+ lastSeen: "2026-06-25T09:15:00.000000Z",
+ },
+ ],
+ });
+ expect(readJson(v4LegacyApiUsageProjectKey("project-b"))).toBeNull();
+ expect(
+ readJson(v4ExperimentPostUsageProjectKey("project-b")),
+ ).toMatchObject({
+ used: true,
+ lastSeen: "2026-06-25T10:05:00.000000Z",
+ });
+
+ expect(redisState.store.get(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(
+ CURRENT_HOUR_ISO,
+ );
+ expect(redisState.store.get(V4_LEGACY_API_USAGE_HEARTBEAT_KEY)).toBe(
+ TEST_NOW.toISOString(),
+ );
+ // Lock released after the run.
+ expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);
+ });
+
+ it("incremental run: re-scans only the trailing margin and merges with existing buckets", async () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ "2026-06-25T09:00:00Z",
+ );
+ seedRecentDeepRescan();
+ seedCoverageBuckets();
+ // Existing bucket from an earlier run, outside the re-scan range: its
+ // counts must survive and merge into the rollup.
+ redisState.store.set(
+ v4LegacyApiHourBucketKey(Date.parse("2026-06-20T00:00:00Z")),
+ JSON.stringify({
+ version: 1,
+ computedAt: "2026-06-20T01:00:00.000Z",
+ apiRows: [
+ {
+ projectId: "project-a",
+ entrypoint: "publicapi: GET /api/public/traces",
+ count: 2,
+ lastSeen: "2026-06-20T00:30:00.000000Z",
+ },
+ ],
+ experimentPostRows: [],
+ }),
+ );
+ // Distinct result sets across services are summed per row key.
+ mocks.queryClickhouse
+ .mockResolvedValueOnce([usageRow({ count: "4" })])
+ .mockResolvedValueOnce([
+ usageRow({ count: "1", lastSeen: "2026-06-25T09:20:00.000000Z" }),
+ ]);
+
+ await handleV4LegacyApiUsageJob();
+
+ // Cursor 09:00 minus the 3h margin.
+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({
+ fromTimestamp: "2026-06-25 06:00:00.000",
+ });
+
+ expect(readJson(v4LegacyApiUsageProjectKey("project-a"))).toMatchObject({
+ rows: [
+ {
+ entrypoint: "publicapi: GET /api/public/traces",
+ // 2 (old bucket) + 4 + 1 (distinct service results summed).
+ count: 7,
+ lastSeen: "2026-06-25T09:20:00.000000Z",
+ },
+ ],
+ });
+ expect(readJson(V4_LEGACY_API_ROLLUP_PROJECTS_KEY)).toEqual({
+ version: 1,
+ api: ["project-a"],
+ experimentPost: [],
+ });
+ });
+
+ it("runs the deep re-scan when none happened within the last 24 hours", async () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ "2026-06-25T09:00:00Z",
+ );
+ seedCoverageBuckets();
+ // Last deep re-scan 25h ago: due again, regardless of wall-clock hour.
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,
+ "2026-06-24T09:30:00.000Z",
+ );
+
+ await handleV4LegacyApiUsageJob();
+
+ // Cursor 09:00 minus the 24h deep margin.
+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({
+ fromTimestamp: "2026-06-24 09:00:00.000",
+ });
+ expect(redisState.store.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY)).toBe(
+ TEST_NOW.toISOString(),
+ );
+ });
+
+ it("extends the scan to repair missing or corrupt buckets inside the window", async () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ "2026-06-25T09:00:00Z",
+ );
+ seedRecentDeepRescan();
+ // One bucket is missing well before the cursor margin; without repair the
+ // rollup would silently lose that hour while the heartbeat stays fresh.
+ seedCoverageBuckets(["2026-06-21T05:00:00Z"]);
+
+ await handleV4LegacyApiUsageJob();
+
+ expect(mocks.queryClickhouse.mock.calls[0]?.[0].params).toMatchObject({
+ fromTimestamp: "2026-06-21 05:00:00.000",
+ });
+ expect(
+ redisState.store.has(
+ v4LegacyApiHourBucketKey(Date.parse("2026-06-21T05:00:00Z")),
+ ),
+ ).toBe(true);
+ });
+
+ it("deletes per-project entries for projects that dropped out of the window", async () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ "2026-06-25T09:00:00Z",
+ );
+ seedRecentDeepRescan();
+ seedCoverageBuckets();
+ // Entries and tracking state from a previous run whose usage has aged out.
+ redisState.store.set(
+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,
+ JSON.stringify({
+ version: 1,
+ api: ["project-gone"],
+ experimentPost: ["project-post-gone"],
+ }),
+ );
+ redisState.store.set(
+ v4LegacyApiUsageProjectKey("project-gone"),
+ JSON.stringify({ version: 1, computedAt: "old", rows: [] }),
+ );
+ redisState.store.set(
+ v4ExperimentPostUsageProjectKey("project-post-gone"),
+ JSON.stringify({
+ version: 1,
+ computedAt: "old",
+ used: true,
+ lastSeen: "2026-06-10T00:00:00.000000Z",
+ }),
+ );
+
+ await handleV4LegacyApiUsageJob();
+
+ expect(
+ redisState.store.has(v4LegacyApiUsageProjectKey("project-gone")),
+ ).toBe(false);
+ expect(
+ redisState.store.has(
+ v4ExperimentPostUsageProjectKey("project-post-gone"),
+ ),
+ ).toBe(false);
+ expect(readJson(V4_LEGACY_API_ROLLUP_PROJECTS_KEY)).toEqual({
+ version: 1,
+ api: [],
+ experimentPost: [],
+ });
+ });
+
+ it("skips the run when another worker holds the lock", async () => {
+ redisState.store.set(V4_LEGACY_API_USAGE_LOCK_KEY, "someone-else");
+
+ await handleV4LegacyApiUsageJob();
+
+ expect(mocks.queryClickhouse).not.toHaveBeenCalled();
+ expect(redisState.store.has(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(false);
+ });
+
+ it("throws and leaves the cursor untouched when a service scan fails", async () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ "2026-06-25T09:00:00Z",
+ );
+ mocks.queryClickhouse
+ .mockResolvedValueOnce([usageRow({})])
+ .mockRejectedValueOnce(new Error("query_log unavailable"));
+
+ await expect(handleV4LegacyApiUsageJob()).rejects.toThrow(
+ "query_log unavailable",
+ );
+
+ expect(redisState.store.get(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(
+ "2026-06-25T09:00:00Z",
+ );
+ expect(redisState.store.has(V4_LEGACY_API_USAGE_HEARTBEAT_KEY)).toBe(false);
+ // Lock still released on failure.
+ expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);
+ });
+});
diff --git a/worker/src/features/v4/__tests__/v4LegacyApiUsageMetrics.test.ts b/worker/src/features/v4/__tests__/v4LegacyApiUsageMetrics.test.ts
new file mode 100644
index 0000000000..b081d3e5f9
--- /dev/null
+++ b/worker/src/features/v4/__tests__/v4LegacyApiUsageMetrics.test.ts
@@ -0,0 +1,60 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const redisState = vi.hoisted(() => ({
+ store: new Map<string, string>(),
+}));
+
+const mocks = vi.hoisted(() => ({
+ recordGauge: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
+ const original =
+ await importOriginal<typeof import("@langfuse/shared/src/server")>();
+ return {
+ ...original,
+ redis: {
+ get: vi.fn(async (key: string) => redisState.store.get(key) ?? null),
+ },
+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+ recordGauge: mocks.recordGauge,
+ };
+});
+
+import { V4_LEGACY_API_USAGE_HEARTBEAT_KEY } from "@langfuse/shared/src/server";
+import {
+ emitV4LegacyApiUsageFreshnessMetrics,
+ V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC,
+} from "../v4LegacyApiUsageMetrics";
+
+describe("emitV4LegacyApiUsageFreshnessMetrics", () => {
+ beforeEach(() => {
+ redisState.store.clear();
+ vi.clearAllMocks();
+ });
+
+ it("emits heartbeat age in seconds for a present heartbeat", async () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,
+ "2026-06-25T10:00:00.000Z",
+ );
+
+ await emitV4LegacyApiUsageFreshnessMetrics(
+ Date.parse("2026-06-25T10:30:00Z"),
+ );
+
+ expect(mocks.recordGauge).toHaveBeenCalledWith(
+ V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC,
+ 1800,
+ { unit: "seconds" },
+ );
+ });
+
+ it("emits nothing when the heartbeat is missing (no-data alert)", async () => {
+ await emitV4LegacyApiUsageFreshnessMetrics(
+ Date.parse("2026-06-25T10:30:00Z"),
+ );
+
+ expect(mocks.recordGauge).not.toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
new file mode 100644
index 0000000000..a5c0b8a14b
--- /dev/null
+++ b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
@@ -0,0 +1,568 @@
+import {
+ aggregateV4LegacyApiHourBuckets,
+ convertDateToClickhouseDateTime,
+ listV4LegacyApiHourStarts,
+ logger,
+ queryClickhouse,
+ redis,
+ systemTableRef,
+ v4ExperimentPostUsageProjectKey,
+ v4LegacyApiHourBucketKey,
+ v4LegacyApiHourBucketSchema,
+ v4LegacyApiHourStartIso,
+ v4LegacyApiRollupProjectsSchema,
+ v4LegacyApiUsageProjectKey,
+ V4_LEGACY_API_DEEP_RESCAN_HOURS,
+ V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ v4LegacyApiHourBucketTtlSeconds,
+ V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,
+ V4_LEGACY_API_RESCAN_HOURS,
+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,
+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,
+ V4_LEGACY_API_USAGE_LOCK_KEY,
+ V4_LEGACY_API_USAGE_WINDOW_MS,
+ type PreferredClickhouseService,
+ type V4LegacyApiHourBucket,
+} from "@langfuse/shared/src/server";
+import { env } from "@langfuse/shared/src/env";
+import { RedisLock } from "../../utils/RedisLock";
+
+/**
+ * Incremental producer for the v4 legacy API usage pipeline.
+ *
+ * Scans `system.query_log` for deprecated public API calls and experiment
+ * `POST /api/public/dataset-run-items` calls, environment-wide (one scan
+ * serves every project), and materializes the results in Redis so the v4
+ * transition UI never has to run this scan from a web request. See
+ * `@langfuse/shared/src/server` (`v4/legacyApiUsage`) for the data contracts.
+ *
+ * Steady state each hourly run scans only the trailing re-scan margin
+ * (~3 hours; 24 hours once a day for late query_log flushes) instead of the
+ * full detection-window scan. Buckets are written by full overwrite, so re-scans and
+ * backfills after missed runs are idempotent. The full-window scan happens
+ * exactly once, on cold start (no cursor).
+ */
+
+const HOUR_MS = 60 * 60 * 1000;
+
+/**
+ * Cold-start / deep-rescan query_log sweeps can take minutes; raise the
+ * ClickHouse client request timeout (and derived max_execution_time) above
+ * the shared 30s default.
+ */
+export const V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS = 10 * 60 * 1000;
+
+const EXPERIMENT_POST_ROUTE = "POST /api/public/dataset-run-items";
+
+type HourUsageRow = {
+ hourStart: string;
+ projectId: string;
+ route: string;
+ count: string | number;
+ lastSeen: string;
+};
+
+type MergedHourUsageRow = {
+ hourStart: string;
+ projectId: string;
+ route: string;
+ count: number;
+ lastSeen: string;
+};
+
+/**
+ * `system.query_log` is local to each ClickHouse service, so every service
+ * that terminates public API reads must be scanned.
+ */
+const getQueryLogServices = (): PreferredClickhouseService[] => {
+ const seenUrls = new Set([new URL(env.CLICKHOUSE_URL).toString()]);
+ const services: PreferredClickhouseService[] = ["ReadWrite"];
+ const candidates: [PreferredClickhouseService, string | undefined][] = [
+ ["ReadOnly", env.CLICKHOUSE_READ_ONLY_URL],
+ ["EventsReadOnly", env.CLICKHOUSE_EVENTS_READ_ONLY_URL],
+ ];
+ for (const [service, url] of candidates) {
+ if (!url) continue;
+ const normalizedUrl = new URL(url).toString();
+ if (seenUrls.has(normalizedUrl)) continue;
+ seenUrls.add(normalizedUrl);
+ services.push(service);
+ }
+ return services;
+};
+
+const queryHourUsageRowsForService = async ({
+ fromTimestamp,
+ toTimestamp,
+ preferredClickhouseService,
+}: {
+ fromTimestamp: Date;
+ toTimestamp: Date;
+ preferredClickhouseService: PreferredClickhouseService;
+}): Promise<HourUsageRow[]> =>
+ queryClickhouse<HourUsageRow>({
+ query: `
+WITH selected AS (
+ SELECT
+ JSONExtractString(log_comment, 'projectId') AS project_id,
+ event_time,
+ event_time_microseconds,
+ splitByChar('?', JSONExtractString(log_comment, 'route'))[1] AS route_path
+ FROM ${systemTableRef("system.query_log")}
+ WHERE
+ event_time >= {fromTimestamp: DateTime64(3)}
+ AND event_time <= {toTimestamp: DateTime64(3)}
+ AND event_date >= toDate({fromTimestamp: DateTime64(3)})
+ AND event_date <= toDate({toTimestamp: DateTime64(3)})
+ AND type = 'QueryFinish'
+ AND JSONExtractString(log_comment, 'tag_schema_version') = '1'
+ AND JSONExtractString(log_comment, 'surface') = 'publicapi'
+ AND JSONExtractString(log_comment, 'projectId') != ''
+),
+classified AS (
+ SELECT
+ project_id,
+ event_time,
+ event_time_microseconds,
+ multiIf(
+ route_path IN (
+ 'GET /api/public/spans',
+ 'GET /api/public/generations',
+ 'GET /api/public/traces',
+ 'GET /api/public/sessions',
+ 'GET /api/public/observations',
+ 'GET /api/public/scores',
+ 'GET /api/public/v2/scores',
+ 'GET /api/public/metrics',
+ 'GET /api/public/metrics/daily',
+ 'GET /api/public/dataset-run-items',
+ 'POST /api/public/dataset-run-items'
+ ), route_path,
+ match(route_path, '^GET /api/public/traces/[^/?#]+$'), 'GET /api/public/traces/{id}',
+ match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 'GET /api/public/sessions/{id}',
+ match(route_path, '^GET /api/public/observations/[^/?#]+$'), 'GET /api/public/observations/{id}',
+ match(route_path, '^GET /api/public/scores/[^/?#]+$'), 'GET /api/public/scores/{id}',
+ match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 'GET /api/public/v2/scores/{id}',
+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs',
+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 'GET /api/public/datasets/{datasetName}/runs/{runName}',
+ NULL
+ ) AS legacy_route,
+ multiIf(
+ route_path IN (
+ 'GET /api/public/spans',
+ 'GET /api/public/generations',
+ 'GET /api/public/traces',
+ 'GET /api/public/observations',
+ 'GET /api/public/scores',
+ 'GET /api/public/v2/scores',
+ 'GET /api/public/metrics/daily',
+ 'GET /api/public/dataset-run-items'
+ ), 2,
+ route_path IN (
+ 'GET /api/public/sessions',
+ 'GET /api/public/metrics',
+ 'POST /api/public/dataset-run-items'
+ ), 1,
+ match(route_path, '^GET /api/public/traces/[^/?#]+$'), 3,
+ match(route_path, '^GET /api/public/sessions/[^/?#]+$'), 1,
+ match(route_path, '^GET /api/public/observations/[^/?#]+$'), 1,
+ match(route_path, '^GET /api/public/scores/[^/?#]+$'), 1,
+ match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 1,
+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 1,
+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1,
+ NULL
+ ) AS clickhouse_queries_per_api_call
+ FROM selected
+)
+SELECT
+ formatDateTime(toStartOfHour(event_time, 'UTC'), '%Y-%m-%dT%H:%i:%SZ', 'UTC') AS hourStart,
+ project_id AS projectId,
+ legacy_route AS route,
+ sum(1.0 / clickhouse_queries_per_api_call) AS count,
+ formatDateTime(max(event_time_microseconds), '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS lastSeen
+FROM classified
+WHERE legacy_route IS NOT NULL
+ AND clickhouse_queries_per_api_call IS NOT NULL
+GROUP BY hourStart, projectId, route
+ORDER BY hourStart ASC, projectId ASC, route ASC
+SETTINGS skip_unavailable_shards = 1
+ `,
+ params: {
+ fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),
+ toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
+ },
+ tags: { route: "v4-legacy-api-usage-pipeline" },
+ preferredClickhouseService,
+ // Shared client derives max_execution_time = request_timeout/1000 + 5s
+ // and enables HTTP progress headers when request_timeout > 30s default.
+ clickhouseConfigs: {
+ request_timeout: V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS,
+ },
+ clickhouseSettings: { skip_unavailable_shards: 1 },
+ });
+
+/**
+ * Merges per-service results. Services pointing at overlapping replicas can
+ * return identical result sets; those are deduplicated wholesale before
+ * counts are summed, mirroring the request-path fallback behavior.
+ */
+const mergeServiceRows = (
+ serviceRowSets: HourUsageRow[][],
+): MergedHourUsageRow[] => {
+ const uniqueServiceResultSets = new Map<string, HourUsageRow[]>();
+ for (const rows of serviceRowSets) {
+ const sortedRows = [...rows].sort(
+ (left, right) =>
+ left.hourStart.localeCompare(right.hourStart) ||
+ left.projectId.localeCompare(right.projectId) ||
+ left.route.localeCompare(right.route),
+ );
+ uniqueServiceResultSets.set(JSON.stringify(sortedRows), sortedRows);
+ }
+
+ const merged = new Map<string, MergedHourUsageRow>();
+ for (const rows of uniqueServiceResultSets.values()) {
+ for (const row of rows) {
+ const key = [row.hourStart, row.projectId, row.route].join("\u0000");
+ const existing = merged.get(key);
+ merged.set(
+ key,
+ existing
+ ? {
+ ...existing,
+ count: existing.count + Number(row.count),
+ lastSeen:
+ existing.lastSeen > row.lastSeen
+ ? existing.lastSeen
+ : row.lastSeen,
+ }
+ : {
+ hourStart: row.hourStart,
+ projectId: row.projectId,
+ route: row.route,
+ count: Number(row.count),
+ lastSeen: row.lastSeen,
+ },
+ );
+ }
+ }
+ return Array.from(merged.values());
+};
+
+const buildHourBuckets = ({
+ scannedHourStartsMs,
+ mergedRows,
+ now,
+}: {
+ scannedHourStartsMs: number[];
+ mergedRows: MergedHourUsageRow[];
+ now: Date;
+}): Map<number, V4LegacyApiHourBucket> => {
+ // Every scanned hour gets a bucket, empty ones included: buckets are
+ // written by full overwrite so repeated scans stay idempotent.
+ const buckets = new Map<number, V4LegacyApiHourBucket>(
+ scannedHourStartsMs.map((hourStartMs) => [
+ hourStartMs,
+ {
+ version: 1,
+ computedAt: now.toISOString(),
+ apiRows: [],
+ experimentPostRows: [],
+ },
+ ]),
+ );
+ for (const row of mergedRows) {
+ const hourStartMs = Date.parse(row.hourStart);
+ const bucket = buckets.get(hourStartMs);
+ if (!bucket) {
+ logger.warn("v4 legacy API usage row outside scanned hours", {
+ hourStart: row.hourStart,
+ });
+ continue;
+ }
+ if (row.route === EXPERIMENT_POST_ROUTE) {
+ bucket.experimentPostRows.push({
+ projectId: row.projectId,
+ count: row.count,
+ lastSeen: row.lastSeen,
+ });
+ } else {
+ bucket.apiRows.push({
+ projectId: row.projectId,
+ entrypoint: `publicapi: ${row.route}`,
+ count: row.count,
+ lastSeen: row.lastSeen,
+ });
+ }
+ }
+ return buckets;
+};
+
+const CHUNK_SIZE = 500;
+
+const setexInChunks = async (
+ entries: { key: string; ttlSeconds: number; value: string }[],
+): Promise<void> => {
+ for (let index = 0; index < entries.length; index += CHUNK_SIZE) {
+ await Promise.all(
+ entries
+ .slice(index, index + CHUNK_SIZE)
+ .map((entry) => redis!.setex(entry.key, entry.ttlSeconds, entry.value)),
+ );
+ }
+};
+
+const mgetInChunks = async (keys: string[]): Promise<(string | null)[]> => {
+ const values: (string | null)[] = [];
+ for (let index = 0; index < keys.length; index += CHUNK_SIZE) {
+ values.push(...(await redis!.mget(keys.slice(index, index + CHUNK_SIZE))));
+ }
+ return values;
+};
+
+const delInChunks = async (keys: string[]): Promise<void> => {
+ for (let index = 0; index < keys.length; index += CHUNK_SIZE) {
+ await redis!.del(...keys.slice(index, index + CHUNK_SIZE));
+ }
+};
+
+const parseHourBucket = (
+ rawBucket: string | null,
+): V4LegacyApiHourBucket | null => {
+ if (!rawBucket) return null;
+ try {
+ const parsed = v4LegacyApiHourBucketSchema.safeParse(JSON.parse(rawBucket));
+ return parsed.success ? parsed.data : null;
+ } catch {
+ return null;
+ }
+};
+
+export const handleV4LegacyApiUsageJob = async (
+ now = new Date(),
+): Promise<void> => {
+ if (!redis) {
+ logger.warn(
+ "Skipping v4 legacy API usage job: Redis is not available to store results",
+ );
+ return;
+ }
+
+ const lock = new RedisLock(V4_LEGACY_API_USAGE_LOCK_KEY, {
+ // Generous TTL: a cold-start scan can take a while, and an expired lock
+ // would let a second replica interleave writes and publish a torn rollup.
+ ttlSeconds: 60 * 60,
+ name: "v4-legacy-api-usage",
+ // Without Redis there is nowhere to write results, so never run unlocked.
+ onUnavailable: "fail",
+ });
+
+ const result = await lock.withLock(async () => {
+ const nowMs = now.getTime();
+ const currentHourStartMs = Math.floor(nowMs / HOUR_MS) * HOUR_MS;
+ const coverageStartMs =
+ Math.floor((nowMs - V4_LEGACY_API_USAGE_WINDOW_MS) / HOUR_MS) * HOUR_MS;
+ const coverageHourStartsMs = listV4LegacyApiHourStarts(
+ coverageStartMs,
+ currentHourStartMs,
+ );
+
+ const [cursorIso, deepRescanAtIso, rawPreviousRollupProjects] =
+ await Promise.all([
+ redis!.get(V4_LEGACY_API_USAGE_CURSOR_KEY),
+ redis!.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY),
+ redis!.get(V4_LEGACY_API_ROLLUP_PROJECTS_KEY),
+ ]);
+ const cursorMs = cursorIso ? Date.parse(cursorIso) : NaN;
+ const deepRescanAtMs = deepRescanAtIso ? Date.parse(deepRescanAtIso) : NaN;
+ // The deep re-scan is driven by elapsed time since the last one (not a
+ // wall-clock hour), so a missed run only delays it instead of skipping a
+ // whole day.
+ const deepRescanDue =
+ !Number.isFinite(deepRescanAtMs) ||
+ nowMs - deepRescanAtMs >= V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS;
+ const rescanMs =
+ (deepRescanDue
+ ? V4_LEGACY_API_DEEP_RESCAN_HOURS
+ : V4_LEGACY_API_RESCAN_HOURS) * HOUR_MS;
+ // Resume from the cursor (self-healing backfill after missed runs) minus
+ // the re-scan margin for late query_log flushes; never beyond the window.
+ const cursorScanStartMs = Number.isFinite(cursorMs)
+ ? Math.max(
+ coverageStartMs,
+ Math.min(cursorMs, currentHourStartMs) - rescanMs,
+ )
+ : coverageStartMs;
+
+ // Bucket holes (evicted or unparsable entries older than the scan start)
+ // would otherwise silently drop usage from the rollup while the fresh
+ // heartbeat blocks the web fallback; extend the scan to repair them.
+ const existingBucketsByHourMs = new Map<
+ number,
+ V4LegacyApiHourBucket | null
+ >();
+ const rawExistingBuckets = await mgetInChunks(
+ coverageHourStartsMs.map(v4LegacyApiHourBucketKey),
+ );
+ coverageHourStartsMs.forEach((hourStartMs, index) => {
+ existingBucketsByHourMs.set(
+ hourStartMs,
+ parseHourBucket(rawExistingBuckets[index] ?? null),
+ );
+ });
+ const oldestHoleMs = coverageHourStartsMs.find(
+ (hourStartMs) =>
+ hourStartMs < cursorScanStartMs &&
+ existingBucketsByHourMs.get(hourStartMs) == null,
+ );
+ const scanStartMs = oldestHoleMs ?? cursorScanStartMs;
+
+ const scannedHourStartsMs = listV4LegacyApiHourStarts(
+ scanStartMs,
+ currentHourStartMs,
+ );
+ logger.info("Running v4 legacy API usage job", {
+ scanStart: v4LegacyApiHourStartIso(scanStartMs),
+ hours: scannedHourStartsMs.length,
+ coldStart: !Number.isFinite(cursorMs),
+ deepRescan: deepRescanDue,
+ repairedHole: oldestHoleMs !== undefined,
+ });
+
+ // All services must succeed; on failure the job throws, the cursor stays
+ // put, and the next run backfills the same hours.
+ const serviceRowSets = await Promise.all(
+ getQueryLogServices().map((preferredClickhouseService) =>
+ queryHourUsageRowsForService({
+ fromTimestamp: new Date(scanStartMs),
+ toTimestamp: now,
+ preferredClickhouseService,
+ }),
+ ),
+ );
+ const buckets = buildHourBuckets({
+ scannedHourStartsMs,
+ mergedRows: mergeServiceRows(serviceRowSets),
+ now,
+ });
+
+ await setexInChunks(
+ Array.from(buckets.entries()).map(([hourStartMs, bucket]) => ({
+ key: v4LegacyApiHourBucketKey(hourStartMs),
+ // Age-based TTL: backfilled/missing hours expire with the window,
+ // not a flat 15d from this write.
+ ttlSeconds: v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs),
+ value: JSON.stringify(bucket),
+ })),
+ );
+
+ // Rebuild the consumer-facing per-project entries from every bucket in
+ // the trailing window: freshly scanned hours plus the pre-read existing
+ // buckets outside the scan range (hole repair above guarantees each
+ // coverage hour is one of the two).
+ const rollup = aggregateV4LegacyApiHourBuckets(
+ coverageHourStartsMs.flatMap((hourStartMs) => {
+ const bucket =
+ buckets.get(hourStartMs) ?? existingBucketsByHourMs.get(hourStartMs);
+ return bucket ? [bucket] : [];
+ }),
+ );
+
+ const computedAt = now.toISOString();
+ await setexInChunks([
+ ...Array.from(rollup.apiRowsByProjectId.entries()).map(
+ ([projectId, rows]) => ({
+ key: v4LegacyApiUsageProjectKey(projectId),
+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,
+ value: JSON.stringify({ version: 1, computedAt, rows }),
+ }),
+ ),
+ ...Array.from(rollup.experimentPostLastSeenByProjectId.entries()).map(
+ ([projectId, lastSeen]) => ({
+ key: v4ExperimentPostUsageProjectKey(projectId),
+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,
+ value: JSON.stringify({
+ version: 1,
+ computedAt,
+ used: true,
+ lastSeen,
+ }),
+ }),
+ ),
+ ]);
+
+ // Delete entries for projects that dropped out of the trailing window
+ // since the previous run; otherwise their last written entry would keep
+ // reporting usage for up to the entry TTL.
+ const previousRollupProjects = (() => {
+ try {
+ const parsed = v4LegacyApiRollupProjectsSchema.safeParse(
+ JSON.parse(rawPreviousRollupProjects ?? "null"),
+ );
+ return parsed.success ? parsed.data : null;
+ } catch {
+ return null;
+ }
+ })();
+ await delInChunks([
+ ...(previousRollupProjects?.api ?? [])
+ .filter((projectId) => !rollup.apiRowsByProjectId.has(projectId))
+ .map(v4LegacyApiUsageProjectKey),
+ ...(previousRollupProjects?.experimentPost ?? [])
+ .filter(
+ (projectId) =>
+ !rollup.experimentPostLastSeenByProjectId.has(projectId),
+ )
+ .map(v4ExperimentPostUsageProjectKey),
+ ]);
+ await redis!.setex(
+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ JSON.stringify({
+ version: 1,
+ api: Array.from(rollup.apiRowsByProjectId.keys()),
+ experimentPost: Array.from(
+ rollup.experimentPostLastSeenByProjectId.keys(),
+ ),
+ }),
+ );
+
+ // Cursor and heartbeat move only after a fully successful run. While the
+ // heartbeat is fresh, web treats missing per-project entries as "no
+ // usage" instead of falling back to ClickHouse.
+ await redis!.setex(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ v4LegacyApiHourStartIso(currentHourStartMs),
+ );
+ if (deepRescanDue) {
+ await redis!.setex(
+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ computedAt,
+ );
+ }
+ await redis!.setex(
+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ computedAt,
+ );
+
+ logger.info("Completed v4 legacy API usage job", {
+ scannedHours: scannedHourStartsMs.length,
+ projectsWithLegacyApiUsage: rollup.apiRowsByProjectId.size,
+ projectsWithExperimentPostUsage:
+ rollup.experimentPostLastSeenByProjectId.size,
+ });
+ return "completed" as const;
+ });
+
+ if (result === null) {
+ logger.info(
+ "Skipping v4 legacy API usage job: another worker holds the lock",
+ );
+ }
+};
diff --git a/worker/src/features/v4/v4LegacyApiUsageMetrics.ts b/worker/src/features/v4/v4LegacyApiUsageMetrics.ts
new file mode 100644
index 0000000000..331a9c19e5
--- /dev/null
+++ b/worker/src/features/v4/v4LegacyApiUsageMetrics.ts
@@ -0,0 +1,44 @@
+import {
+ logger,
+ recordGauge,
+ redis,
+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,
+} from "@langfuse/shared/src/server";
+
+/**
+ * Freshness metric for the v4 legacy API usage pipeline.
+ *
+ * A single gauge is enough to alert on staleness: the worker only refreshes
+ * the heartbeat after a fully successful run, so a stuck, failing, or missing
+ * job lets `heartbeat_age_seconds` climb past the freshness window. Emitted
+ * from QueueMetricsRunner so age keeps rising between the hourly job runs.
+ *
+ * Alert: `avg:langfuse.v4.legacy_api_usage.heartbeat_age_seconds > 10800` (3h).
+ */
+export const V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC =
+ "langfuse.v4.legacy_api_usage.heartbeat_age_seconds";
+
+export const emitV4LegacyApiUsageFreshnessMetrics = async (
+ nowMs = Date.now(),
+): Promise<void> => {
+ if (!redis) return;
+
+ const heartbeatIso = await redis.get(V4_LEGACY_API_USAGE_HEARTBEAT_KEY);
+ const heartbeatMs = heartbeatIso ? Date.parse(heartbeatIso) : NaN;
+ // Missing/unparsable heartbeat (never ran) surfaces as a no-data alert; the
+ // key's 15d TTL means a running pipeline always has a value to age off.
+ if (!Number.isFinite(heartbeatMs)) return;
+
+ try {
+ recordGauge(
+ V4_LEGACY_API_USAGE_HEARTBEAT_AGE_METRIC,
+ Math.max(0, Math.floor((nowMs - heartbeatMs) / 1000)),
+ { unit: "seconds" },
+ );
+ } catch (error) {
+ logger.error(
+ "v4 legacy API usage: failed to emit heartbeat freshness metric",
+ error,
+ );
+ }
+};
diff --git a/worker/src/queues/v4LegacyApiUsageQueue.ts b/worker/src/queues/v4LegacyApiUsageQueue.ts
new file mode 100644
index 0000000000..c88d5766d8
--- /dev/null
+++ b/worker/src/queues/v4LegacyApiUsageQueue.ts
@@ -0,0 +1,15 @@
+import { Processor } from "bullmq";
+import { logger, QueueJobs } from "@langfuse/shared/src/server";
+import { handleV4LegacyApiUsageJob } from "../features/v4/handleV4LegacyApiUsageJob";
+
+export const v4LegacyApiUsageProcessor: Processor = async (job) => {
+ if (job.name === QueueJobs.V4LegacyApiUsageJob) {
+ logger.info("Executing V4 Legacy API Usage Job");
+ try {
+ return await handleV4LegacyApiUsageJob();
+ } catch (error) {
+ logger.error("Error executing V4LegacyApiUsageJob", error);
+ throw error;
+ }
+ }
+};