🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): add structured logging to the v4 legacy API usage job (#16184)
Max Deichmann931c9755d936
diff --git a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.tsindex 1e43a1ce88..2184187c70 100644--- a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts+++ b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts@@ -62,6 +62,12 @@ const redisMock = vi.hoisted(() => ({const mocks = vi.hoisted(() => ({queryClickhouse: vi.fn(),+ logger: {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ },}));vi.mock("@langfuse/shared/src/server", async (importOriginal) => {@@ -70,12 +76,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {return {...original,redis: redisMock,- logger: {- debug: vi.fn(),- info: vi.fn(),- warn: vi.fn(),- error: vi.fn(),- },+ logger: mocks.logger,queryClickhouse: mocks.queryClickhouse,systemTableRef: (table: string) =>`clusterAllReplicas('test-cluster', '${table}')`,@@ -94,6 +95,21 @@ vi.mock("@langfuse/shared/src/env", () => ({import { handleV4LegacyApiUsageJob } from "../handleV4LegacyApiUsageJob";+const loggedInfo = (message: string) =>+ mocks.logger.info.mock.calls.find(+ ([loggedMessage]) => loggedMessage === message,+ )?.[1];++const loggedWarn = (message: string) =>+ mocks.logger.warn.mock.calls.find(+ ([loggedMessage]) => loggedMessage === message,+ )?.[1];++const loggedError = (message: string) =>+ mocks.logger.error.mock.calls.find(+ ([loggedMessage]) => loggedMessage === message,+ )?.[1];+// 10:30 UTC: a regular run (the deep re-scan happens at 03:xx UTC only).const TEST_NOW = new Date("2026-06-25T10:30:00Z");const CURRENT_HOUR_ISO = "2026-06-25T10:00:00Z";@@ -252,6 +268,33 @@ describe("handleV4LegacyApiUsageJob", () => {);// Lock released after the run.expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);++ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({+ scanStart: "2026-06-11T10:00:00Z",+ hours: 337,+ coldStart: true,+ repairedHole: false,+ missingBucketCount: 337,+ clickhouseServices: ["ReadWrite", "ReadOnly"],+ });+ expect(+ loggedInfo("v4 legacy API usage: merged query_log rows"),+ ).toMatchObject({+ inputServiceCount: 2,+ uniqueResultSetCount: 1,+ mergedRowCount: 2,+ uniqueProjects: 2,+ apiRowCount: 1,+ experimentPostRowCount: 1,+ });+ expect(loggedInfo("Completed v4 legacy API usage job")).toMatchObject({+ scannedHours: 337,+ nonEmptyBuckets: 2,+ projectsWithLegacyApiUsage: 1,+ projectsWithExperimentPostUsage: 1,+ cursor: CURRENT_HOUR_ISO,+ deepRescanRecorded: true,+ });});it("incremental run: re-scans only the trailing margin and merges with existing buckets", async () => {@@ -308,6 +351,21 @@ describe("handleV4LegacyApiUsageJob", () => {api: ["project-a"],experimentPost: [],});+ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({+ scanStart: "2026-06-25T06:00:00Z",+ hours: 5,+ coldStart: false,+ deepRescan: false,+ repairedHole: false,+ missingBucketCount: 0,+ });+ expect(+ loggedInfo("v4 legacy API usage: merged query_log rows"),+ ).toMatchObject({+ uniqueResultSetCount: 2,+ serviceRowCounts: [1, 1],+ mergedRowCount: 1,+ });});it("runs the deep re-scan when none happened within the last 24 hours", async () => {@@ -353,6 +411,11 @@ describe("handleV4LegacyApiUsageJob", () => {v4LegacyApiHourBucketKey(Date.parse("2026-06-21T05:00:00Z")),),).toBe(true);+ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({+ repairedHole: true,+ repairedHoleHour: "2026-06-21T05:00:00Z",+ missingBucketCount: 1,+ });});it("deletes per-project entries for projects that dropped out of the window", async () => {@@ -400,6 +463,16 @@ describe("handleV4LegacyApiUsageJob", () => {api: [],experimentPost: [],});+ expect(+ loggedInfo("v4 legacy API usage: wrote project rollup"),+ ).toMatchObject({+ apiProjectsWritten: 0,+ experimentPostProjectsWritten: 0,+ deletedApiProjects: 1,+ deletedExperimentPostProjects: 1,+ previousApiProjects: 1,+ previousExperimentPostProjects: 1,+ });});it("skips the run when another worker holds the lock", async () => {@@ -409,6 +482,9 @@ describe("handleV4LegacyApiUsageJob", () => {expect(mocks.queryClickhouse).not.toHaveBeenCalled();expect(redisState.store.has(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(false);+ expect(mocks.logger.info).toHaveBeenCalledWith(+ "Skipping v4 legacy API usage job: another worker holds the lock",+ );});it("throws and leaves the cursor untouched when a service scan fails", async () => {@@ -430,5 +506,41 @@ describe("handleV4LegacyApiUsageJob", () => {expect(redisState.store.has(V4_LEGACY_API_USAGE_HEARTBEAT_KEY)).toBe(false);// Lock still released on failure.expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);+ expect(+ loggedError("v4 legacy API usage: query_log scan failed"),+ ).toMatchObject({+ service: "ReadOnly",+ errorMessage: "query_log unavailable",+ });+ expect(loggedError("v4 legacy API usage: job failed")).toMatchObject({+ errorMessage: "query_log unavailable",+ });+ });++ it("logs unparsable hour buckets and extends the scan to repair them", async () => {+ redisState.store.set(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ "2026-06-25T09:00:00Z",+ );+ seedRecentDeepRescan();+ seedCoverageBuckets();+ redisState.store.set(+ v4LegacyApiHourBucketKey(Date.parse("2026-06-21T05:00:00Z")),+ "not-json",+ );++ await handleV4LegacyApiUsageJob();++ expect(+ loggedWarn("v4 legacy API usage: unparsable hour buckets"),+ ).toMatchObject({+ count: 1,+ hourStarts: ["2026-06-21T05:00:00Z"],+ });+ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({+ repairedHole: true,+ repairedHoleHour: "2026-06-21T05:00:00Z",+ unparsableBucketCount: 1,+ });});});diff --git a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts b/worker/src/features/v4/handleV4LegacyApiUsageJob.tsindex 8dbcdbcc6b..f11cc09817 100644--- a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts+++ b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts@@ -59,6 +59,18 @@ export const V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS = 10 * 60 * 1000;const EXPERIMENT_POST_ROUTE = "POST /api/public/dataset-run-items";+const elapsedMs = (startedAt: number): number =>+ Math.round(performance.now() - startedAt);++/** Enumerable error fields for winston JSON logs. Nesting an `Error` under a+ * metadata key serializes to `{}` because message/stack are non-enumerable. */+const serializeLogError = (+ error: unknown,+): { errorMessage: string; errorStack?: string } =>+ error instanceof Error+ ? { errorMessage: error.message, errorStack: error.stack }+ : { errorMessage: String(error) };+type HourUsageRow = {hourStart: string;projectId: string;@@ -104,9 +116,16 @@ const queryHourUsageRowsForService = async ({fromTimestamp: Date;toTimestamp: Date;preferredClickhouseService: PreferredClickhouseService;-}): Promise<HourUsageRow[]> =>- queryClickhouse<HourUsageRow>({- query: `+}): Promise<HourUsageRow[]> => {+ const startedAt = performance.now();+ logger.info("v4 legacy API usage: scanning query_log", {+ service: preferredClickhouseService,+ fromTimestamp: fromTimestamp.toISOString(),+ toTimestamp: toTimestamp.toISOString(),+ });+ try {+ const rows = await queryClickhouse<HourUsageRow>({+ query: `WITH selected AS (SELECTJSONExtractString(log_comment, 'projectId') AS project_id,@@ -192,19 +211,34 @@ GROUP BY hourStart, projectId, routeORDER BY hourStart ASC, projectId ASC, route ASCSETTINGS skip_unavailable_shards = 1`,- params: {- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),- },- tags: { route: "v4-legacy-api-usage-pipeline" },- preferredClickhouseService,- // Shared client derives max_execution_time = request_timeout/1000 + 5s- // and enables HTTP progress headers when request_timeout > 30s default.- clickhouseConfigs: {- request_timeout: V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS,- },- clickhouseSettings: { skip_unavailable_shards: 1 },- });+ params: {+ fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),+ toTimestamp: convertDateToClickhouseDateTime(toTimestamp),+ },+ tags: { route: "v4-legacy-api-usage-pipeline" },+ preferredClickhouseService,+ // Shared client derives max_execution_time = request_timeout/1000 + 5s+ // and enables HTTP progress headers when request_timeout > 30s default.+ clickhouseConfigs: {+ request_timeout: V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS,+ },+ clickhouseSettings: { skip_unavailable_shards: 1 },+ });+ logger.info("v4 legacy API usage: query_log scan completed", {+ service: preferredClickhouseService,+ durationMs: elapsedMs(startedAt),+ rowCount: rows.length,+ });+ return rows;+ } catch (error) {+ logger.error("v4 legacy API usage: query_log scan failed", {+ service: preferredClickhouseService,+ durationMs: elapsedMs(startedAt),+ ...serializeLogError(error),+ });+ throw error;+ }+};/*** Merges per-service results. Services pointing at overlapping replicas can@@ -251,7 +285,32 @@ const mergeServiceRows = ();}}- return Array.from(merged.values());+ const mergedRows = Array.from(merged.values());+ const projects = new Set<string>();+ const routes = new Set<string>();+ let apiRowCount = 0;+ let experimentPostRowCount = 0;+ for (const row of mergedRows) {+ projects.add(row.projectId);+ routes.add(row.route);+ if (row.route === EXPERIMENT_POST_ROUTE) {+ experimentPostRowCount += 1;+ } else {+ apiRowCount += 1;+ }+ }+ logger.info("v4 legacy API usage: merged query_log rows", {+ inputServiceCount: serviceRowSets.length,+ uniqueResultSetCount: uniqueServiceResultSets.size,+ serviceRowCounts: serviceRowSets.map((rows) => rows.length),+ mergedRowCount: mergedRows.length,+ uniqueProjects: projects.size,+ uniqueRoutes: routes.size,+ routes: [...routes].sort(),+ apiRowCount,+ experimentPostRowCount,+ });+ return mergedRows;};const buildHourBuckets = ({@@ -366,6 +425,7 @@ export const handleV4LegacyApiUsageJob = async (});const result = await lock.withLock(async () => {+ const jobStartedAt = performance.now();const nowMs = now.getTime();const currentHourStartMs = Math.floor(nowMs / HOUR_MS) * HOUR_MS;const coverageStartMs =@@ -374,195 +434,286 @@ export const handleV4LegacyApiUsageJob = async (coverageStartMs,currentHourStartMs,);+ const clickhouseServices = getQueryLogServices();- const [cursorIso, deepRescanAtIso, rawPreviousRollupProjects] =- await Promise.all([- redis!.get(V4_LEGACY_API_USAGE_CURSOR_KEY),- redis!.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY),- redis!.get(V4_LEGACY_API_ROLLUP_PROJECTS_KEY),- ]);- const cursorMs = cursorIso ? Date.parse(cursorIso) : NaN;- const deepRescanAtMs = deepRescanAtIso ? Date.parse(deepRescanAtIso) : NaN;- // The deep re-scan is driven by elapsed time since the last one (not a- // wall-clock hour), so a missed run only delays it instead of skipping a- // whole day.- const deepRescanDue =- !Number.isFinite(deepRescanAtMs) ||- nowMs - deepRescanAtMs >= V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS;- const rescanMs =- (deepRescanDue- ? V4_LEGACY_API_DEEP_RESCAN_HOURS- : V4_LEGACY_API_RESCAN_HOURS) * HOUR_MS;- // Resume from the cursor (self-healing backfill after missed runs) minus- // the re-scan margin for late query_log flushes; never beyond the window.- const cursorScanStartMs = Number.isFinite(cursorMs)- ? Math.max(- coverageStartMs,- Math.min(cursorMs, currentHourStartMs) - rescanMs,- )- : coverageStartMs;-- // Bucket holes (evicted or unparsable entries older than the scan start)- // would otherwise silently drop usage from the rollup while the fresh- // heartbeat blocks the web fallback; extend the scan to repair them.- const existingBucketsByHourMs = new Map<- number,- V4LegacyApiHourBucket | null- >();- const rawExistingBuckets = await mgetInChunks(- coverageHourStartsMs.map(v4LegacyApiHourBucketKey),- );- coverageHourStartsMs.forEach((hourStartMs, index) => {- existingBucketsByHourMs.set(- hourStartMs,- parseHourBucket(rawExistingBuckets[index] ?? null),- );+ logger.info("v4 legacy API usage: lock acquired", {+ now: now.toISOString(),+ coverageStart: v4LegacyApiHourStartIso(coverageStartMs),+ currentHour: v4LegacyApiHourStartIso(currentHourStartMs),+ coverageHours: coverageHourStartsMs.length,+ clickhouseServices,});- const oldestHoleMs = coverageHourStartsMs.find(- (hourStartMs) =>- hourStartMs < cursorScanStartMs &&- existingBucketsByHourMs.get(hourStartMs) == null,- );- const scanStartMs = oldestHoleMs ?? cursorScanStartMs;- const scannedHourStartsMs = listV4LegacyApiHourStarts(- scanStartMs,- currentHourStartMs,- );- logger.info("Running v4 legacy API usage job", {- scanStart: v4LegacyApiHourStartIso(scanStartMs),- hours: scannedHourStartsMs.length,- coldStart: !Number.isFinite(cursorMs),- deepRescan: deepRescanDue,- repairedHole: oldestHoleMs !== undefined,- });+ try {+ const [cursorIso, deepRescanAtIso, rawPreviousRollupProjects] =+ await Promise.all([+ redis!.get(V4_LEGACY_API_USAGE_CURSOR_KEY),+ redis!.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY),+ redis!.get(V4_LEGACY_API_ROLLUP_PROJECTS_KEY),+ ]);+ const cursorMs = cursorIso ? Date.parse(cursorIso) : NaN;+ const deepRescanAtMs = deepRescanAtIso+ ? Date.parse(deepRescanAtIso)+ : NaN;+ // The deep re-scan is driven by elapsed time since the last one (not a+ // wall-clock hour), so a missed run only delays it instead of skipping a+ // whole day.+ const deepRescanDue =+ !Number.isFinite(deepRescanAtMs) ||+ nowMs - deepRescanAtMs >= V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS;+ const rescanHours = deepRescanDue+ ? V4_LEGACY_API_DEEP_RESCAN_HOURS+ : V4_LEGACY_API_RESCAN_HOURS;+ const rescanMs = rescanHours * HOUR_MS;+ // Resume from the cursor (self-healing backfill after missed runs) minus+ // the re-scan margin for late query_log flushes; never beyond the window.+ const cursorScanStartMs = Number.isFinite(cursorMs)+ ? Math.max(+ coverageStartMs,+ Math.min(cursorMs, currentHourStartMs) - rescanMs,+ )+ : coverageStartMs;- // All services must succeed; on failure the job throws, the cursor stays- // put, and the next run backfills the same hours.- const serviceRowSets = await Promise.all(- getQueryLogServices().map((preferredClickhouseService) =>- queryHourUsageRowsForService({- fromTimestamp: new Date(scanStartMs),- toTimestamp: now,- preferredClickhouseService,- }),- ),- );- const buckets = buildHourBuckets({- scannedHourStartsMs,- mergedRows: mergeServiceRows(serviceRowSets),- now,- });+ // Bucket holes (evicted or unparsable entries older than the scan start)+ // would otherwise silently drop usage from the rollup while the fresh+ // heartbeat blocks the web fallback; extend the scan to repair them.+ const existingBucketsByHourMs = new Map<+ number,+ V4LegacyApiHourBucket | null+ >();+ const redisReadStartedAt = performance.now();+ const rawExistingBuckets = await mgetInChunks(+ coverageHourStartsMs.map(v4LegacyApiHourBucketKey),+ );+ const unparsableHourStarts: string[] = [];+ let missingBucketCount = 0;+ coverageHourStartsMs.forEach((hourStartMs, index) => {+ const rawBucket = rawExistingBuckets[index] ?? null;+ if (!rawBucket) {+ missingBucketCount += 1;+ existingBucketsByHourMs.set(hourStartMs, null);+ return;+ }+ const parsed = parseHourBucket(rawBucket);+ if (!parsed) {+ unparsableHourStarts.push(v4LegacyApiHourStartIso(hourStartMs));+ existingBucketsByHourMs.set(hourStartMs, null);+ return;+ }+ existingBucketsByHourMs.set(hourStartMs, parsed);+ });+ if (unparsableHourStarts.length > 0) {+ logger.warn("v4 legacy API usage: unparsable hour buckets", {+ count: unparsableHourStarts.length,+ hourStarts: unparsableHourStarts.slice(0, 10),+ });+ }+ logger.info("v4 legacy API usage: loaded existing hour buckets", {+ coverageHours: coverageHourStartsMs.length,+ durationMs: elapsedMs(redisReadStartedAt),+ missingBucketCount,+ unparsableBucketCount: unparsableHourStarts.length,+ });+ const oldestHoleMs = coverageHourStartsMs.find(+ (hourStartMs) =>+ hourStartMs < cursorScanStartMs &&+ existingBucketsByHourMs.get(hourStartMs) == null,+ );+ const scanStartMs = oldestHoleMs ?? cursorScanStartMs;- await setexInChunks(- Array.from(buckets.entries()).map(([hourStartMs, bucket]) => ({- key: v4LegacyApiHourBucketKey(hourStartMs),- // Age-based TTL: backfilled/missing hours expire with the window,- // not a flat 15d from this write.- ttlSeconds: v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs),- value: JSON.stringify(bucket),- })),- );+ const scannedHourStartsMs = listV4LegacyApiHourStarts(+ scanStartMs,+ currentHourStartMs,+ );+ logger.info("Running v4 legacy API usage job", {+ scanStart: v4LegacyApiHourStartIso(scanStartMs),+ scanEnd: now.toISOString(),+ hours: scannedHourStartsMs.length,+ cursor: cursorIso,+ deepRescanAt: deepRescanAtIso,+ rescanHours,+ coldStart: !Number.isFinite(cursorMs),+ deepRescan: deepRescanDue,+ repairedHole: oldestHoleMs !== undefined,+ repairedHoleHour:+ oldestHoleMs !== undefined+ ? v4LegacyApiHourStartIso(oldestHoleMs)+ : null,+ missingBucketCount,+ unparsableBucketCount: unparsableHourStarts.length,+ clickhouseServices,+ });- // Rebuild the consumer-facing per-project entries from every bucket in- // the trailing window: freshly scanned hours plus the pre-read existing- // buckets outside the scan range (hole repair above guarantees each- // coverage hour is one of the two).- const rollup = aggregateV4LegacyApiHourBuckets(- coverageHourStartsMs.flatMap((hourStartMs) => {- const bucket =- buckets.get(hourStartMs) ?? existingBucketsByHourMs.get(hourStartMs);- return bucket ? [bucket] : [];- }),- );+ // All services must succeed; on failure the job throws, the cursor stays+ // put, and the next run backfills the same hours.+ const clickhouseStartedAt = performance.now();+ const serviceRowSets = await Promise.all(+ clickhouseServices.map((preferredClickhouseService) =>+ queryHourUsageRowsForService({+ fromTimestamp: new Date(scanStartMs),+ toTimestamp: now,+ preferredClickhouseService,+ }),+ ),+ );+ const clickhouseDurationMs = elapsedMs(clickhouseStartedAt);+ const mergedRows = mergeServiceRows(serviceRowSets);+ const buckets = buildHourBuckets({+ scannedHourStartsMs,+ mergedRows,+ now,+ });+ let nonEmptyBucketCount = 0;+ for (const bucket of buckets.values()) {+ if (bucket.apiRows.length > 0 || bucket.experimentPostRows.length > 0) {+ nonEmptyBucketCount += 1;+ }+ }- const computedAt = now.toISOString();- await setexInChunks([- ...Array.from(rollup.apiRowsByProjectId.entries()).map(- ([projectId, rows]) => ({- key: v4LegacyApiUsageProjectKey(projectId),- ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,- value: JSON.stringify({ version: 1, computedAt, rows }),+ const redisWriteStartedAt = performance.now();+ await setexInChunks(+ Array.from(buckets.entries()).map(([hourStartMs, bucket]) => ({+ key: v4LegacyApiHourBucketKey(hourStartMs),+ // Age-based TTL: backfilled/missing hours expire with the window,+ // not a flat 15d from this write.+ ttlSeconds: v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs),+ value: JSON.stringify(bucket),+ })),+ );+ logger.info("v4 legacy API usage: wrote hour buckets", {+ bucketsWritten: buckets.size,+ nonEmptyBuckets: nonEmptyBucketCount,+ emptyBuckets: buckets.size - nonEmptyBucketCount,+ durationMs: elapsedMs(redisWriteStartedAt),+ });++ // Rebuild the consumer-facing per-project entries from every bucket in+ // the trailing window: freshly scanned hours plus the pre-read existing+ // buckets outside the scan range (hole repair above guarantees each+ // coverage hour is one of the two).+ const rollup = aggregateV4LegacyApiHourBuckets(+ coverageHourStartsMs.flatMap((hourStartMs) => {+ const bucket =+ buckets.get(hourStartMs) ??+ existingBucketsByHourMs.get(hourStartMs);+ return bucket ? [bucket] : [];}),- ),- ...Array.from(rollup.experimentPostLastSeenByProjectId.entries()).map(- ([projectId, lastSeen]) => ({- key: v4ExperimentPostUsageProjectKey(projectId),- ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,- value: JSON.stringify({- version: 1,- computedAt,- used: true,- lastSeen,+ );++ const computedAt = now.toISOString();+ const projectWriteStartedAt = performance.now();+ await setexInChunks([+ ...Array.from(rollup.apiRowsByProjectId.entries()).map(+ ([projectId, rows]) => ({+ key: v4LegacyApiUsageProjectKey(projectId),+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,+ value: JSON.stringify({ version: 1, computedAt, rows }),+ }),+ ),+ ...Array.from(rollup.experimentPostLastSeenByProjectId.entries()).map(+ ([projectId, lastSeen]) => ({+ key: v4ExperimentPostUsageProjectKey(projectId),+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,+ value: JSON.stringify({+ version: 1,+ computedAt,+ used: true,+ lastSeen,+ }),}),+ ),+ ]);++ // Delete entries for projects that dropped out of the trailing window+ // since the previous run; otherwise their last written entry would keep+ // reporting usage for up to the entry TTL.+ const previousRollupProjects = (() => {+ try {+ const parsed = v4LegacyApiRollupProjectsSchema.safeParse(+ JSON.parse(rawPreviousRollupProjects ?? "null"),+ );+ return parsed.success ? parsed.data : null;+ } catch {+ return null;+ }+ })();+ const deletedApiProjects = (previousRollupProjects?.api ?? []).filter(+ (projectId) => !rollup.apiRowsByProjectId.has(projectId),+ );+ const deletedExperimentPostProjects = (+ previousRollupProjects?.experimentPost ?? []+ ).filter(+ (projectId) => !rollup.experimentPostLastSeenByProjectId.has(projectId),+ );+ await delInChunks([+ ...deletedApiProjects.map(v4LegacyApiUsageProjectKey),+ ...deletedExperimentPostProjects.map(v4ExperimentPostUsageProjectKey),+ ]);+ await redis!.setex(+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ JSON.stringify({+ version: 1,+ api: Array.from(rollup.apiRowsByProjectId.keys()),+ experimentPost: Array.from(+ rollup.experimentPostLastSeenByProjectId.keys(),+ ),}),- ),- ]);-- // Delete entries for projects that dropped out of the trailing window- // since the previous run; otherwise their last written entry would keep- // reporting usage for up to the entry TTL.- const previousRollupProjects = (() => {- try {- const parsed = v4LegacyApiRollupProjectsSchema.safeParse(- JSON.parse(rawPreviousRollupProjects ?? "null"),+ );++ // Cursor and heartbeat move only after a fully successful run. While the+ // heartbeat is fresh, web treats missing per-project entries as "no+ // usage" instead of falling back to ClickHouse.+ await redis!.setex(+ V4_LEGACY_API_USAGE_CURSOR_KEY,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ v4LegacyApiHourStartIso(currentHourStartMs),+ );+ if (deepRescanDue) {+ await redis!.setex(+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,+ computedAt,);- return parsed.success ? parsed.data : null;- } catch {- return null;}- })();- await delInChunks([- ...(previousRollupProjects?.api ?? [])- .filter((projectId) => !rollup.apiRowsByProjectId.has(projectId))- .map(v4LegacyApiUsageProjectKey),- ...(previousRollupProjects?.experimentPost ?? [])- .filter(- (projectId) =>- !rollup.experimentPostLastSeenByProjectId.has(projectId),- )- .map(v4ExperimentPostUsageProjectKey),- ]);- await redis!.setex(- V4_LEGACY_API_ROLLUP_PROJECTS_KEY,- V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,- JSON.stringify({- version: 1,- api: Array.from(rollup.apiRowsByProjectId.keys()),- experimentPost: Array.from(- rollup.experimentPostLastSeenByProjectId.keys(),- ),- }),- );-- // Cursor and heartbeat move only after a fully successful run. While the- // heartbeat is fresh, web treats missing per-project entries as "no- // usage" instead of falling back to ClickHouse.- await redis!.setex(- V4_LEGACY_API_USAGE_CURSOR_KEY,- V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,- v4LegacyApiHourStartIso(currentHourStartMs),- );- if (deepRescanDue) {await redis!.setex(- V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,computedAt,);- }- await redis!.setex(- V4_LEGACY_API_USAGE_HEARTBEAT_KEY,- V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,- computedAt,- );+ logger.info("v4 legacy API usage: wrote project rollup", {+ apiProjectsWritten: rollup.apiRowsByProjectId.size,+ experimentPostProjectsWritten:+ rollup.experimentPostLastSeenByProjectId.size,+ deletedApiProjects: deletedApiProjects.length,+ deletedExperimentPostProjects: deletedExperimentPostProjects.length,+ previousApiProjects: previousRollupProjects?.api.length ?? 0,+ previousExperimentPostProjects:+ previousRollupProjects?.experimentPost.length ?? 0,+ durationMs: elapsedMs(projectWriteStartedAt),+ });- logger.info("Completed v4 legacy API usage job", {- scannedHours: scannedHourStartsMs.length,- projectsWithLegacyApiUsage: rollup.apiRowsByProjectId.size,- projectsWithExperimentPostUsage:- rollup.experimentPostLastSeenByProjectId.size,- });- return "completed" as const;+ logger.info("Completed v4 legacy API usage job", {+ durationMs: elapsedMs(jobStartedAt),+ clickhouseDurationMs,+ scannedHours: scannedHourStartsMs.length,+ nonEmptyBuckets: nonEmptyBucketCount,+ emptyBuckets: buckets.size - nonEmptyBucketCount,+ projectsWithLegacyApiUsage: rollup.apiRowsByProjectId.size,+ projectsWithExperimentPostUsage:+ rollup.experimentPostLastSeenByProjectId.size,+ cursor: v4LegacyApiHourStartIso(currentHourStartMs),+ heartbeatAt: computedAt,+ deepRescanRecorded: deepRescanDue,+ });+ return "completed" as const;+ } catch (error) {+ logger.error("v4 legacy API usage: job failed", {+ durationMs: elapsedMs(jobStartedAt),+ ...serializeLogError(error),+ });+ throw error;+ }});if (result === null) {diff --git a/worker/src/queues/v4LegacyApiUsageQueue.ts b/worker/src/queues/v4LegacyApiUsageQueue.tsindex c88d5766d8..9e7980cd4d 100644--- a/worker/src/queues/v4LegacyApiUsageQueue.ts+++ b/worker/src/queues/v4LegacyApiUsageQueue.ts@@ -4,11 +4,26 @@ import { handleV4LegacyApiUsageJob } from "../features/v4/handleV4LegacyApiUsageexport const v4LegacyApiUsageProcessor: Processor = async (job) => {if (job.name === QueueJobs.V4LegacyApiUsageJob) {- logger.info("Executing V4 Legacy API Usage Job");+ const startedAt = performance.now();+ logger.info("Executing V4 Legacy API Usage Job", {+ jobId: job.id,+ attemptsMade: job.attemptsMade,+ });try {- return await handleV4LegacyApiUsageJob();+ const result = await handleV4LegacyApiUsageJob();+ logger.info("Finished V4 Legacy API Usage Job", {+ jobId: job.id,+ durationMs: Math.round(performance.now() - startedAt),+ });+ return result;} catch (error) {- logger.error("Error executing V4LegacyApiUsageJob", error);+ logger.error("Error executing V4LegacyApiUsageJob", {+ jobId: job.id,+ attemptsMade: job.attemptsMade,+ durationMs: Math.round(performance.now() - startedAt),+ errorMessage: error instanceof Error ? error.message : String(error),+ errorStack: error instanceof Error ? error.stack : undefined,+ });throw error;}}