Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): add structured logging to the v4 legacy API usage job (#16184)

Max Deichmann931c9755d936
diff --git a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
index 1e43a1ce88..2184187c70 100644
--- a/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
+++ b/worker/src/features/v4/__tests__/handleV4LegacyApiUsageJob.test.ts
@@ -62,6 +62,12 @@ const redisMock = vi.hoisted(() => ({
const mocks = vi.hoisted(() => ({
queryClickhouse: vi.fn(),
+ logger: {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ },
}));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
@@ -70,12 +76,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
return {
...original,
redis: redisMock,
- logger: {
- debug: vi.fn(),
- info: vi.fn(),
- warn: vi.fn(),
- error: vi.fn(),
- },
+ logger: mocks.logger,
queryClickhouse: mocks.queryClickhouse,
systemTableRef: (table: string) =>
`clusterAllReplicas('test-cluster', '${table}')`,
@@ -94,6 +95,21 @@ vi.mock("@langfuse/shared/src/env", () => ({
import { handleV4LegacyApiUsageJob } from "../handleV4LegacyApiUsageJob";
+const loggedInfo = (message: string) =>
+ mocks.logger.info.mock.calls.find(
+ ([loggedMessage]) => loggedMessage === message,
+ )?.[1];
+
+const loggedWarn = (message: string) =>
+ mocks.logger.warn.mock.calls.find(
+ ([loggedMessage]) => loggedMessage === message,
+ )?.[1];
+
+const loggedError = (message: string) =>
+ mocks.logger.error.mock.calls.find(
+ ([loggedMessage]) => loggedMessage === message,
+ )?.[1];
+
// 10:30 UTC: a regular run (the deep re-scan happens at 03:xx UTC only).
const TEST_NOW = new Date("2026-06-25T10:30:00Z");
const CURRENT_HOUR_ISO = "2026-06-25T10:00:00Z";
@@ -252,6 +268,33 @@ describe("handleV4LegacyApiUsageJob", () => {
);
// Lock released after the run.
expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);
+
+ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({
+ scanStart: "2026-06-11T10:00:00Z",
+ hours: 337,
+ coldStart: true,
+ repairedHole: false,
+ missingBucketCount: 337,
+ clickhouseServices: ["ReadWrite", "ReadOnly"],
+ });
+ expect(
+ loggedInfo("v4 legacy API usage: merged query_log rows"),
+ ).toMatchObject({
+ inputServiceCount: 2,
+ uniqueResultSetCount: 1,
+ mergedRowCount: 2,
+ uniqueProjects: 2,
+ apiRowCount: 1,
+ experimentPostRowCount: 1,
+ });
+ expect(loggedInfo("Completed v4 legacy API usage job")).toMatchObject({
+ scannedHours: 337,
+ nonEmptyBuckets: 2,
+ projectsWithLegacyApiUsage: 1,
+ projectsWithExperimentPostUsage: 1,
+ cursor: CURRENT_HOUR_ISO,
+ deepRescanRecorded: true,
+ });
});
it("incremental run: re-scans only the trailing margin and merges with existing buckets", async () => {
@@ -308,6 +351,21 @@ describe("handleV4LegacyApiUsageJob", () => {
api: ["project-a"],
experimentPost: [],
});
+ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({
+ scanStart: "2026-06-25T06:00:00Z",
+ hours: 5,
+ coldStart: false,
+ deepRescan: false,
+ repairedHole: false,
+ missingBucketCount: 0,
+ });
+ expect(
+ loggedInfo("v4 legacy API usage: merged query_log rows"),
+ ).toMatchObject({
+ uniqueResultSetCount: 2,
+ serviceRowCounts: [1, 1],
+ mergedRowCount: 1,
+ });
});
it("runs the deep re-scan when none happened within the last 24 hours", async () => {
@@ -353,6 +411,11 @@ describe("handleV4LegacyApiUsageJob", () => {
v4LegacyApiHourBucketKey(Date.parse("2026-06-21T05:00:00Z")),
),
).toBe(true);
+ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({
+ repairedHole: true,
+ repairedHoleHour: "2026-06-21T05:00:00Z",
+ missingBucketCount: 1,
+ });
});
it("deletes per-project entries for projects that dropped out of the window", async () => {
@@ -400,6 +463,16 @@ describe("handleV4LegacyApiUsageJob", () => {
api: [],
experimentPost: [],
});
+ expect(
+ loggedInfo("v4 legacy API usage: wrote project rollup"),
+ ).toMatchObject({
+ apiProjectsWritten: 0,
+ experimentPostProjectsWritten: 0,
+ deletedApiProjects: 1,
+ deletedExperimentPostProjects: 1,
+ previousApiProjects: 1,
+ previousExperimentPostProjects: 1,
+ });
});
it("skips the run when another worker holds the lock", async () => {
@@ -409,6 +482,9 @@ describe("handleV4LegacyApiUsageJob", () => {
expect(mocks.queryClickhouse).not.toHaveBeenCalled();
expect(redisState.store.has(V4_LEGACY_API_USAGE_CURSOR_KEY)).toBe(false);
+ expect(mocks.logger.info).toHaveBeenCalledWith(
+ "Skipping v4 legacy API usage job: another worker holds the lock",
+ );
});
it("throws and leaves the cursor untouched when a service scan fails", async () => {
@@ -430,5 +506,41 @@ describe("handleV4LegacyApiUsageJob", () => {
expect(redisState.store.has(V4_LEGACY_API_USAGE_HEARTBEAT_KEY)).toBe(false);
// Lock still released on failure.
expect(redisState.store.has(V4_LEGACY_API_USAGE_LOCK_KEY)).toBe(false);
+ expect(
+ loggedError("v4 legacy API usage: query_log scan failed"),
+ ).toMatchObject({
+ service: "ReadOnly",
+ errorMessage: "query_log unavailable",
+ });
+ expect(loggedError("v4 legacy API usage: job failed")).toMatchObject({
+ errorMessage: "query_log unavailable",
+ });
+ });
+
+ it("logs unparsable hour buckets and extends the scan to repair them", async () => {
+ redisState.store.set(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ "2026-06-25T09:00:00Z",
+ );
+ seedRecentDeepRescan();
+ seedCoverageBuckets();
+ redisState.store.set(
+ v4LegacyApiHourBucketKey(Date.parse("2026-06-21T05:00:00Z")),
+ "not-json",
+ );
+
+ await handleV4LegacyApiUsageJob();
+
+ expect(
+ loggedWarn("v4 legacy API usage: unparsable hour buckets"),
+ ).toMatchObject({
+ count: 1,
+ hourStarts: ["2026-06-21T05:00:00Z"],
+ });
+ expect(loggedInfo("Running v4 legacy API usage job")).toMatchObject({
+ repairedHole: true,
+ repairedHoleHour: "2026-06-21T05:00:00Z",
+ unparsableBucketCount: 1,
+ });
});
});
diff --git a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
index 8dbcdbcc6b..f11cc09817 100644
--- a/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
+++ b/worker/src/features/v4/handleV4LegacyApiUsageJob.ts
@@ -59,6 +59,18 @@ export const V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS = 10 * 60 * 1000;
const EXPERIMENT_POST_ROUTE = "POST /api/public/dataset-run-items";
+const elapsedMs = (startedAt: number): number =>
+ Math.round(performance.now() - startedAt);
+
+/** Enumerable error fields for winston JSON logs. Nesting an `Error` under a
+ * metadata key serializes to `{}` because message/stack are non-enumerable. */
+const serializeLogError = (
+ error: unknown,
+): { errorMessage: string; errorStack?: string } =>
+ error instanceof Error
+ ? { errorMessage: error.message, errorStack: error.stack }
+ : { errorMessage: String(error) };
+
type HourUsageRow = {
hourStart: string;
projectId: string;
@@ -104,9 +116,16 @@ const queryHourUsageRowsForService = async ({
fromTimestamp: Date;
toTimestamp: Date;
preferredClickhouseService: PreferredClickhouseService;
-}): Promise<HourUsageRow[]> =>
- queryClickhouse<HourUsageRow>({
- query: `
+}): Promise<HourUsageRow[]> => {
+ const startedAt = performance.now();
+ logger.info("v4 legacy API usage: scanning query_log", {
+ service: preferredClickhouseService,
+ fromTimestamp: fromTimestamp.toISOString(),
+ toTimestamp: toTimestamp.toISOString(),
+ });
+ try {
+ const rows = await queryClickhouse<HourUsageRow>({
+ query: `
WITH selected AS (
SELECT
JSONExtractString(log_comment, 'projectId') AS project_id,
@@ -192,19 +211,34 @@ GROUP BY hourStart, projectId, route
ORDER BY hourStart ASC, projectId ASC, route ASC
SETTINGS skip_unavailable_shards = 1
`,
- params: {
- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),
- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
- },
- tags: { route: "v4-legacy-api-usage-pipeline" },
- preferredClickhouseService,
- // Shared client derives max_execution_time = request_timeout/1000 + 5s
- // and enables HTTP progress headers when request_timeout > 30s default.
- clickhouseConfigs: {
- request_timeout: V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS,
- },
- clickhouseSettings: { skip_unavailable_shards: 1 },
- });
+ params: {
+ fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),
+ toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
+ },
+ tags: { route: "v4-legacy-api-usage-pipeline" },
+ preferredClickhouseService,
+ // Shared client derives max_execution_time = request_timeout/1000 + 5s
+ // and enables HTTP progress headers when request_timeout > 30s default.
+ clickhouseConfigs: {
+ request_timeout: V4_LEGACY_API_USAGE_QUERY_TIMEOUT_MS,
+ },
+ clickhouseSettings: { skip_unavailable_shards: 1 },
+ });
+ logger.info("v4 legacy API usage: query_log scan completed", {
+ service: preferredClickhouseService,
+ durationMs: elapsedMs(startedAt),
+ rowCount: rows.length,
+ });
+ return rows;
+ } catch (error) {
+ logger.error("v4 legacy API usage: query_log scan failed", {
+ service: preferredClickhouseService,
+ durationMs: elapsedMs(startedAt),
+ ...serializeLogError(error),
+ });
+ throw error;
+ }
+};
/**
* Merges per-service results. Services pointing at overlapping replicas can
@@ -251,7 +285,32 @@ const mergeServiceRows = (
);
}
}
- return Array.from(merged.values());
+ const mergedRows = Array.from(merged.values());
+ const projects = new Set<string>();
+ const routes = new Set<string>();
+ let apiRowCount = 0;
+ let experimentPostRowCount = 0;
+ for (const row of mergedRows) {
+ projects.add(row.projectId);
+ routes.add(row.route);
+ if (row.route === EXPERIMENT_POST_ROUTE) {
+ experimentPostRowCount += 1;
+ } else {
+ apiRowCount += 1;
+ }
+ }
+ logger.info("v4 legacy API usage: merged query_log rows", {
+ inputServiceCount: serviceRowSets.length,
+ uniqueResultSetCount: uniqueServiceResultSets.size,
+ serviceRowCounts: serviceRowSets.map((rows) => rows.length),
+ mergedRowCount: mergedRows.length,
+ uniqueProjects: projects.size,
+ uniqueRoutes: routes.size,
+ routes: [...routes].sort(),
+ apiRowCount,
+ experimentPostRowCount,
+ });
+ return mergedRows;
};
const buildHourBuckets = ({
@@ -366,6 +425,7 @@ export const handleV4LegacyApiUsageJob = async (
});
const result = await lock.withLock(async () => {
+ const jobStartedAt = performance.now();
const nowMs = now.getTime();
const currentHourStartMs = Math.floor(nowMs / HOUR_MS) * HOUR_MS;
const coverageStartMs =
@@ -374,195 +434,286 @@ export const handleV4LegacyApiUsageJob = async (
coverageStartMs,
currentHourStartMs,
);
+ const clickhouseServices = getQueryLogServices();
- const [cursorIso, deepRescanAtIso, rawPreviousRollupProjects] =
- await Promise.all([
- redis!.get(V4_LEGACY_API_USAGE_CURSOR_KEY),
- redis!.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY),
- redis!.get(V4_LEGACY_API_ROLLUP_PROJECTS_KEY),
- ]);
- const cursorMs = cursorIso ? Date.parse(cursorIso) : NaN;
- const deepRescanAtMs = deepRescanAtIso ? Date.parse(deepRescanAtIso) : NaN;
- // The deep re-scan is driven by elapsed time since the last one (not a
- // wall-clock hour), so a missed run only delays it instead of skipping a
- // whole day.
- const deepRescanDue =
- !Number.isFinite(deepRescanAtMs) ||
- nowMs - deepRescanAtMs >= V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS;
- const rescanMs =
- (deepRescanDue
- ? V4_LEGACY_API_DEEP_RESCAN_HOURS
- : V4_LEGACY_API_RESCAN_HOURS) * HOUR_MS;
- // Resume from the cursor (self-healing backfill after missed runs) minus
- // the re-scan margin for late query_log flushes; never beyond the window.
- const cursorScanStartMs = Number.isFinite(cursorMs)
- ? Math.max(
- coverageStartMs,
- Math.min(cursorMs, currentHourStartMs) - rescanMs,
- )
- : coverageStartMs;
-
- // Bucket holes (evicted or unparsable entries older than the scan start)
- // would otherwise silently drop usage from the rollup while the fresh
- // heartbeat blocks the web fallback; extend the scan to repair them.
- const existingBucketsByHourMs = new Map<
- number,
- V4LegacyApiHourBucket | null
- >();
- const rawExistingBuckets = await mgetInChunks(
- coverageHourStartsMs.map(v4LegacyApiHourBucketKey),
- );
- coverageHourStartsMs.forEach((hourStartMs, index) => {
- existingBucketsByHourMs.set(
- hourStartMs,
- parseHourBucket(rawExistingBuckets[index] ?? null),
- );
+ logger.info("v4 legacy API usage: lock acquired", {
+ now: now.toISOString(),
+ coverageStart: v4LegacyApiHourStartIso(coverageStartMs),
+ currentHour: v4LegacyApiHourStartIso(currentHourStartMs),
+ coverageHours: coverageHourStartsMs.length,
+ clickhouseServices,
});
- const oldestHoleMs = coverageHourStartsMs.find(
- (hourStartMs) =>
- hourStartMs < cursorScanStartMs &&
- existingBucketsByHourMs.get(hourStartMs) == null,
- );
- const scanStartMs = oldestHoleMs ?? cursorScanStartMs;
- const scannedHourStartsMs = listV4LegacyApiHourStarts(
- scanStartMs,
- currentHourStartMs,
- );
- logger.info("Running v4 legacy API usage job", {
- scanStart: v4LegacyApiHourStartIso(scanStartMs),
- hours: scannedHourStartsMs.length,
- coldStart: !Number.isFinite(cursorMs),
- deepRescan: deepRescanDue,
- repairedHole: oldestHoleMs !== undefined,
- });
+ try {
+ const [cursorIso, deepRescanAtIso, rawPreviousRollupProjects] =
+ await Promise.all([
+ redis!.get(V4_LEGACY_API_USAGE_CURSOR_KEY),
+ redis!.get(V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY),
+ redis!.get(V4_LEGACY_API_ROLLUP_PROJECTS_KEY),
+ ]);
+ const cursorMs = cursorIso ? Date.parse(cursorIso) : NaN;
+ const deepRescanAtMs = deepRescanAtIso
+ ? Date.parse(deepRescanAtIso)
+ : NaN;
+ // The deep re-scan is driven by elapsed time since the last one (not a
+ // wall-clock hour), so a missed run only delays it instead of skipping a
+ // whole day.
+ const deepRescanDue =
+ !Number.isFinite(deepRescanAtMs) ||
+ nowMs - deepRescanAtMs >= V4_LEGACY_API_DEEP_RESCAN_INTERVAL_MS;
+ const rescanHours = deepRescanDue
+ ? V4_LEGACY_API_DEEP_RESCAN_HOURS
+ : V4_LEGACY_API_RESCAN_HOURS;
+ const rescanMs = rescanHours * HOUR_MS;
+ // Resume from the cursor (self-healing backfill after missed runs) minus
+ // the re-scan margin for late query_log flushes; never beyond the window.
+ const cursorScanStartMs = Number.isFinite(cursorMs)
+ ? Math.max(
+ coverageStartMs,
+ Math.min(cursorMs, currentHourStartMs) - rescanMs,
+ )
+ : coverageStartMs;
- // All services must succeed; on failure the job throws, the cursor stays
- // put, and the next run backfills the same hours.
- const serviceRowSets = await Promise.all(
- getQueryLogServices().map((preferredClickhouseService) =>
- queryHourUsageRowsForService({
- fromTimestamp: new Date(scanStartMs),
- toTimestamp: now,
- preferredClickhouseService,
- }),
- ),
- );
- const buckets = buildHourBuckets({
- scannedHourStartsMs,
- mergedRows: mergeServiceRows(serviceRowSets),
- now,
- });
+ // Bucket holes (evicted or unparsable entries older than the scan start)
+ // would otherwise silently drop usage from the rollup while the fresh
+ // heartbeat blocks the web fallback; extend the scan to repair them.
+ const existingBucketsByHourMs = new Map<
+ number,
+ V4LegacyApiHourBucket | null
+ >();
+ const redisReadStartedAt = performance.now();
+ const rawExistingBuckets = await mgetInChunks(
+ coverageHourStartsMs.map(v4LegacyApiHourBucketKey),
+ );
+ const unparsableHourStarts: string[] = [];
+ let missingBucketCount = 0;
+ coverageHourStartsMs.forEach((hourStartMs, index) => {
+ const rawBucket = rawExistingBuckets[index] ?? null;
+ if (!rawBucket) {
+ missingBucketCount += 1;
+ existingBucketsByHourMs.set(hourStartMs, null);
+ return;
+ }
+ const parsed = parseHourBucket(rawBucket);
+ if (!parsed) {
+ unparsableHourStarts.push(v4LegacyApiHourStartIso(hourStartMs));
+ existingBucketsByHourMs.set(hourStartMs, null);
+ return;
+ }
+ existingBucketsByHourMs.set(hourStartMs, parsed);
+ });
+ if (unparsableHourStarts.length > 0) {
+ logger.warn("v4 legacy API usage: unparsable hour buckets", {
+ count: unparsableHourStarts.length,
+ hourStarts: unparsableHourStarts.slice(0, 10),
+ });
+ }
+ logger.info("v4 legacy API usage: loaded existing hour buckets", {
+ coverageHours: coverageHourStartsMs.length,
+ durationMs: elapsedMs(redisReadStartedAt),
+ missingBucketCount,
+ unparsableBucketCount: unparsableHourStarts.length,
+ });
+ const oldestHoleMs = coverageHourStartsMs.find(
+ (hourStartMs) =>
+ hourStartMs < cursorScanStartMs &&
+ existingBucketsByHourMs.get(hourStartMs) == null,
+ );
+ const scanStartMs = oldestHoleMs ?? cursorScanStartMs;
- await setexInChunks(
- Array.from(buckets.entries()).map(([hourStartMs, bucket]) => ({
- key: v4LegacyApiHourBucketKey(hourStartMs),
- // Age-based TTL: backfilled/missing hours expire with the window,
- // not a flat 15d from this write.
- ttlSeconds: v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs),
- value: JSON.stringify(bucket),
- })),
- );
+ const scannedHourStartsMs = listV4LegacyApiHourStarts(
+ scanStartMs,
+ currentHourStartMs,
+ );
+ logger.info("Running v4 legacy API usage job", {
+ scanStart: v4LegacyApiHourStartIso(scanStartMs),
+ scanEnd: now.toISOString(),
+ hours: scannedHourStartsMs.length,
+ cursor: cursorIso,
+ deepRescanAt: deepRescanAtIso,
+ rescanHours,
+ coldStart: !Number.isFinite(cursorMs),
+ deepRescan: deepRescanDue,
+ repairedHole: oldestHoleMs !== undefined,
+ repairedHoleHour:
+ oldestHoleMs !== undefined
+ ? v4LegacyApiHourStartIso(oldestHoleMs)
+ : null,
+ missingBucketCount,
+ unparsableBucketCount: unparsableHourStarts.length,
+ clickhouseServices,
+ });
- // Rebuild the consumer-facing per-project entries from every bucket in
- // the trailing window: freshly scanned hours plus the pre-read existing
- // buckets outside the scan range (hole repair above guarantees each
- // coverage hour is one of the two).
- const rollup = aggregateV4LegacyApiHourBuckets(
- coverageHourStartsMs.flatMap((hourStartMs) => {
- const bucket =
- buckets.get(hourStartMs) ?? existingBucketsByHourMs.get(hourStartMs);
- return bucket ? [bucket] : [];
- }),
- );
+ // All services must succeed; on failure the job throws, the cursor stays
+ // put, and the next run backfills the same hours.
+ const clickhouseStartedAt = performance.now();
+ const serviceRowSets = await Promise.all(
+ clickhouseServices.map((preferredClickhouseService) =>
+ queryHourUsageRowsForService({
+ fromTimestamp: new Date(scanStartMs),
+ toTimestamp: now,
+ preferredClickhouseService,
+ }),
+ ),
+ );
+ const clickhouseDurationMs = elapsedMs(clickhouseStartedAt);
+ const mergedRows = mergeServiceRows(serviceRowSets);
+ const buckets = buildHourBuckets({
+ scannedHourStartsMs,
+ mergedRows,
+ now,
+ });
+ let nonEmptyBucketCount = 0;
+ for (const bucket of buckets.values()) {
+ if (bucket.apiRows.length > 0 || bucket.experimentPostRows.length > 0) {
+ nonEmptyBucketCount += 1;
+ }
+ }
- const computedAt = now.toISOString();
- await setexInChunks([
- ...Array.from(rollup.apiRowsByProjectId.entries()).map(
- ([projectId, rows]) => ({
- key: v4LegacyApiUsageProjectKey(projectId),
- ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,
- value: JSON.stringify({ version: 1, computedAt, rows }),
+ const redisWriteStartedAt = performance.now();
+ await setexInChunks(
+ Array.from(buckets.entries()).map(([hourStartMs, bucket]) => ({
+ key: v4LegacyApiHourBucketKey(hourStartMs),
+ // Age-based TTL: backfilled/missing hours expire with the window,
+ // not a flat 15d from this write.
+ ttlSeconds: v4LegacyApiHourBucketTtlSeconds(hourStartMs, nowMs),
+ value: JSON.stringify(bucket),
+ })),
+ );
+ logger.info("v4 legacy API usage: wrote hour buckets", {
+ bucketsWritten: buckets.size,
+ nonEmptyBuckets: nonEmptyBucketCount,
+ emptyBuckets: buckets.size - nonEmptyBucketCount,
+ durationMs: elapsedMs(redisWriteStartedAt),
+ });
+
+ // Rebuild the consumer-facing per-project entries from every bucket in
+ // the trailing window: freshly scanned hours plus the pre-read existing
+ // buckets outside the scan range (hole repair above guarantees each
+ // coverage hour is one of the two).
+ const rollup = aggregateV4LegacyApiHourBuckets(
+ coverageHourStartsMs.flatMap((hourStartMs) => {
+ const bucket =
+ buckets.get(hourStartMs) ??
+ existingBucketsByHourMs.get(hourStartMs);
+ return bucket ? [bucket] : [];
}),
- ),
- ...Array.from(rollup.experimentPostLastSeenByProjectId.entries()).map(
- ([projectId, lastSeen]) => ({
- key: v4ExperimentPostUsageProjectKey(projectId),
- ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,
- value: JSON.stringify({
- version: 1,
- computedAt,
- used: true,
- lastSeen,
+ );
+
+ const computedAt = now.toISOString();
+ const projectWriteStartedAt = performance.now();
+ await setexInChunks([
+ ...Array.from(rollup.apiRowsByProjectId.entries()).map(
+ ([projectId, rows]) => ({
+ key: v4LegacyApiUsageProjectKey(projectId),
+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,
+ value: JSON.stringify({ version: 1, computedAt, rows }),
+ }),
+ ),
+ ...Array.from(rollup.experimentPostLastSeenByProjectId.entries()).map(
+ ([projectId, lastSeen]) => ({
+ key: v4ExperimentPostUsageProjectKey(projectId),
+ ttlSeconds: V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS,
+ value: JSON.stringify({
+ version: 1,
+ computedAt,
+ used: true,
+ lastSeen,
+ }),
}),
+ ),
+ ]);
+
+ // Delete entries for projects that dropped out of the trailing window
+ // since the previous run; otherwise their last written entry would keep
+ // reporting usage for up to the entry TTL.
+ const previousRollupProjects = (() => {
+ try {
+ const parsed = v4LegacyApiRollupProjectsSchema.safeParse(
+ JSON.parse(rawPreviousRollupProjects ?? "null"),
+ );
+ return parsed.success ? parsed.data : null;
+ } catch {
+ return null;
+ }
+ })();
+ const deletedApiProjects = (previousRollupProjects?.api ?? []).filter(
+ (projectId) => !rollup.apiRowsByProjectId.has(projectId),
+ );
+ const deletedExperimentPostProjects = (
+ previousRollupProjects?.experimentPost ?? []
+ ).filter(
+ (projectId) => !rollup.experimentPostLastSeenByProjectId.has(projectId),
+ );
+ await delInChunks([
+ ...deletedApiProjects.map(v4LegacyApiUsageProjectKey),
+ ...deletedExperimentPostProjects.map(v4ExperimentPostUsageProjectKey),
+ ]);
+ await redis!.setex(
+ V4_LEGACY_API_ROLLUP_PROJECTS_KEY,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ JSON.stringify({
+ version: 1,
+ api: Array.from(rollup.apiRowsByProjectId.keys()),
+ experimentPost: Array.from(
+ rollup.experimentPostLastSeenByProjectId.keys(),
+ ),
}),
- ),
- ]);
-
- // Delete entries for projects that dropped out of the trailing window
- // since the previous run; otherwise their last written entry would keep
- // reporting usage for up to the entry TTL.
- const previousRollupProjects = (() => {
- try {
- const parsed = v4LegacyApiRollupProjectsSchema.safeParse(
- JSON.parse(rawPreviousRollupProjects ?? "null"),
+ );
+
+ // Cursor and heartbeat move only after a fully successful run. While the
+ // heartbeat is fresh, web treats missing per-project entries as "no
+ // usage" instead of falling back to ClickHouse.
+ await redis!.setex(
+ V4_LEGACY_API_USAGE_CURSOR_KEY,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ v4LegacyApiHourStartIso(currentHourStartMs),
+ );
+ if (deepRescanDue) {
+ await redis!.setex(
+ V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,
+ V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
+ computedAt,
);
- return parsed.success ? parsed.data : null;
- } catch {
- return null;
}
- })();
- await delInChunks([
- ...(previousRollupProjects?.api ?? [])
- .filter((projectId) => !rollup.apiRowsByProjectId.has(projectId))
- .map(v4LegacyApiUsageProjectKey),
- ...(previousRollupProjects?.experimentPost ?? [])
- .filter(
- (projectId) =>
- !rollup.experimentPostLastSeenByProjectId.has(projectId),
- )
- .map(v4ExperimentPostUsageProjectKey),
- ]);
- await redis!.setex(
- V4_LEGACY_API_ROLLUP_PROJECTS_KEY,
- V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
- JSON.stringify({
- version: 1,
- api: Array.from(rollup.apiRowsByProjectId.keys()),
- experimentPost: Array.from(
- rollup.experimentPostLastSeenByProjectId.keys(),
- ),
- }),
- );
-
- // Cursor and heartbeat move only after a fully successful run. While the
- // heartbeat is fresh, web treats missing per-project entries as "no
- // usage" instead of falling back to ClickHouse.
- await redis!.setex(
- V4_LEGACY_API_USAGE_CURSOR_KEY,
- V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
- v4LegacyApiHourStartIso(currentHourStartMs),
- );
- if (deepRescanDue) {
await redis!.setex(
- V4_LEGACY_API_USAGE_DEEP_RESCAN_AT_KEY,
+ V4_LEGACY_API_USAGE_HEARTBEAT_KEY,
V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
computedAt,
);
- }
- await redis!.setex(
- V4_LEGACY_API_USAGE_HEARTBEAT_KEY,
- V4_LEGACY_API_HOUR_BUCKET_TTL_SECONDS,
- computedAt,
- );
+ logger.info("v4 legacy API usage: wrote project rollup", {
+ apiProjectsWritten: rollup.apiRowsByProjectId.size,
+ experimentPostProjectsWritten:
+ rollup.experimentPostLastSeenByProjectId.size,
+ deletedApiProjects: deletedApiProjects.length,
+ deletedExperimentPostProjects: deletedExperimentPostProjects.length,
+ previousApiProjects: previousRollupProjects?.api.length ?? 0,
+ previousExperimentPostProjects:
+ previousRollupProjects?.experimentPost.length ?? 0,
+ durationMs: elapsedMs(projectWriteStartedAt),
+ });
- logger.info("Completed v4 legacy API usage job", {
- scannedHours: scannedHourStartsMs.length,
- projectsWithLegacyApiUsage: rollup.apiRowsByProjectId.size,
- projectsWithExperimentPostUsage:
- rollup.experimentPostLastSeenByProjectId.size,
- });
- return "completed" as const;
+ logger.info("Completed v4 legacy API usage job", {
+ durationMs: elapsedMs(jobStartedAt),
+ clickhouseDurationMs,
+ scannedHours: scannedHourStartsMs.length,
+ nonEmptyBuckets: nonEmptyBucketCount,
+ emptyBuckets: buckets.size - nonEmptyBucketCount,
+ projectsWithLegacyApiUsage: rollup.apiRowsByProjectId.size,
+ projectsWithExperimentPostUsage:
+ rollup.experimentPostLastSeenByProjectId.size,
+ cursor: v4LegacyApiHourStartIso(currentHourStartMs),
+ heartbeatAt: computedAt,
+ deepRescanRecorded: deepRescanDue,
+ });
+ return "completed" as const;
+ } catch (error) {
+ logger.error("v4 legacy API usage: job failed", {
+ durationMs: elapsedMs(jobStartedAt),
+ ...serializeLogError(error),
+ });
+ throw error;
+ }
});
if (result === null) {
diff --git a/worker/src/queues/v4LegacyApiUsageQueue.ts b/worker/src/queues/v4LegacyApiUsageQueue.ts
index c88d5766d8..9e7980cd4d 100644
--- a/worker/src/queues/v4LegacyApiUsageQueue.ts
+++ b/worker/src/queues/v4LegacyApiUsageQueue.ts
@@ -4,11 +4,26 @@ import { handleV4LegacyApiUsageJob } from "../features/v4/handleV4LegacyApiUsage
export const v4LegacyApiUsageProcessor: Processor = async (job) => {
if (job.name === QueueJobs.V4LegacyApiUsageJob) {
- logger.info("Executing V4 Legacy API Usage Job");
+ const startedAt = performance.now();
+ logger.info("Executing V4 Legacy API Usage Job", {
+ jobId: job.id,
+ attemptsMade: job.attemptsMade,
+ });
try {
- return await handleV4LegacyApiUsageJob();
+ const result = await handleV4LegacyApiUsageJob();
+ logger.info("Finished V4 Legacy API Usage Job", {
+ jobId: job.id,
+ durationMs: Math.round(performance.now() - startedAt),
+ });
+ return result;
} catch (error) {
- logger.error("Error executing V4LegacyApiUsageJob", error);
+ logger.error("Error executing V4LegacyApiUsageJob", {
+ jobId: job.id,
+ attemptsMade: job.attemptsMade,
+ durationMs: Math.round(performance.now() - startedAt),
+ errorMessage: error instanceof Error ? error.message : String(error),
+ errorStack: error instanceof Error ? error.stack : undefined,
+ });
throw error;
}
}