🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(worker): run v4 legacy API usage job every 15 minutes (#16183)
Max Deichmann7c79bac2acf3
diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.tsnew file mode 100644index 0000000000..ea212184eb--- /dev/null+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts@@ -0,0 +1,62 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++describe("V4LegacyApiUsageQueue schedule", () => {+ beforeEach(() => {+ vi.resetModules();+ });++ afterEach(() => {+ vi.doUnmock("bullmq");+ vi.doUnmock("./redis");+ vi.doUnmock("../logger");+ vi.clearAllMocks();+ });++ it("removes the legacy hourly repeatable job before scheduling */15", async () => {+ const removeRepeatable = vi.fn().mockResolvedValue(undefined);+ const add = vi.fn().mockResolvedValue(undefined);+ const on = vi.fn();++ vi.doMock("bullmq", () => ({+ Queue: class {+ removeRepeatable = removeRepeatable;+ add = add;+ on = on;+ },+ }));++ vi.doMock("./redis", () => ({+ createBullMQQueueOptionsWithRedis: vi.fn(() => ({+ connection: {},+ prefix: "test",+ })),+ }));++ vi.doMock("../logger", () => ({+ logger: {+ debug: vi.fn(),+ error: vi.fn(),+ },+ }));++ const { V4LegacyApiUsageQueue, V4_LEGACY_API_USAGE_CRON_PATTERN } =+ await import("./v4LegacyApiUsageQueue.js");+ const { QueueJobs } = await import("../queues.js");++ V4LegacyApiUsageQueue.getInstance();++ expect(removeRepeatable).toHaveBeenCalledWith(+ QueueJobs.V4LegacyApiUsageJob,+ { pattern: "25 * * * *" },+ );+ expect(add).toHaveBeenCalledWith(+ QueueJobs.V4LegacyApiUsageJob,+ {},+ { repeat: { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN } },+ );+ expect(V4_LEGACY_API_USAGE_CRON_PATTERN).toBe("*/15 * * * *");+ expect(removeRepeatable.mock.invocationCallOrder[0]).toBeLessThan(+ add.mock.invocationCallOrder[0]!,+ );+ });+});diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.tsindex de43592d6d..106c990584 100644--- a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts@@ -3,10 +3,9 @@ import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";import { logger } from "../logger";-// Hourly, offset from other scheduled jobs (:5 metering, :30 posthog, :35-// free-tier). The worker re-scans a trailing margin each run, so the exact-// minute only affects freshness, not correctness.-export const V4_LEGACY_API_USAGE_CRON_PATTERN = "25 * * * *";+// Every 15 minutes. The worker re-scans a trailing margin each run, so the+// exact minute only affects freshness, not correctness.+export const V4_LEGACY_API_USAGE_CRON_PATTERN = "*/15 * * * *";export class V4LegacyApiUsageQueue {private static instance: Queue | null = null;@@ -40,6 +39,20 @@ export class V4LegacyApiUsageQueue {if (V4LegacyApiUsageQueue.instance) {logger.debug("Scheduling jobs for V4LegacyApiUsageQueue");+ // Remove the old hourly cron pattern - BullMQ keys repeatable jobs by+ // name + pattern, so changing the pattern creates a second schedule+ // while the old one keeps firing.+ V4LegacyApiUsageQueue.instance+ // eslint-disable-next-line @typescript-eslint/no-deprecated -- Existing repeatable-job cleanup; job scheduler migration should be handled separately.+ .removeRepeatable(QueueJobs.V4LegacyApiUsageJob, {+ pattern: "25 * * * *",+ })+ .catch((err) => {+ logger.error(+ "Error removing legacy V4LegacyApiUsageJob schedule",+ err,+ );+ });V4LegacyApiUsageQueue.instance.add(QueueJobs.V4LegacyApiUsageJob,diff --git a/packages/shared/src/server/v4/legacyApiUsage.ts b/packages/shared/src/server/v4/legacyApiUsage.tsindex 767e6ffc99..9ec1d1ac49 100644--- a/packages/shared/src/server/v4/legacyApiUsage.ts+++ b/packages/shared/src/server/v4/legacyApiUsage.ts@@ -56,7 +56,7 @@ export const v4LegacyApiHourBucketTtlSeconds = (/*** Per-project entries outlive several missed worker runs; the worker- * refreshes them hourly, so they only expire when the pipeline is down and+ * refreshes them every 15 minutes, so they only expire when the pipeline is down and* web falls back to its request-path cache behavior.*/export const V4_LEGACY_API_PROJECT_ENTRY_TTL_SECONDS = 12 * 60 * 60;@@ -69,7 +69,7 @@ export const V4_LEGACY_API_HEARTBEAT_FRESHNESS_MS = 3 * HOUR_MS;/** Trailing hours re-scanned every run to absorb late query_log flushes. */export const V4_LEGACY_API_RESCAN_HOURS = 3;-/** Deeper re-scan for stragglers beyond the hourly margin. */+/** Deeper re-scan for stragglers beyond the regular rescan margin. */export const V4_LEGACY_API_DEEP_RESCAN_HOURS = 24;/*** Minimum spacing between deep re-scans. Tracked via a persisted timestampdiff --git a/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx b/web/src/features/v4-migration/V4MigrationContent.clienttest.tsxindex cc73160f35..a1ed32e399 100644--- a/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx+++ b/web/src/features/v4-migration/V4MigrationContent.clienttest.tsx@@ -396,7 +396,7 @@ describe("V4MigrationDetailsContent", () => {expect(screen.getByText(- "SDK, instrumentation, experiment, and API checks cover activity from the last 14 days.",+ "SDK, instrumentation, experiment, and API checks cover activity from the last 14 days. API and experiment usage counts refresh about every 15 minutes, so recent calls may not appear yet.",),).toBeInTheDocument();expect(diff --git a/web/src/features/v4-migration/V4MigrationContent.tsx b/web/src/features/v4-migration/V4MigrationContent.tsxindex c11876d9b0..c9381e62f5 100644--- a/web/src/features/v4-migration/V4MigrationContent.tsx+++ b/web/src/features/v4-migration/V4MigrationContent.tsx@@ -896,7 +896,8 @@ export function V4MigrationApisSection({<><p className="text-muted-foreground mb-2 text-sm">You've called these deprecated endpoints in the last{" "}- {V4_MIGRATION_LOOKBACK_DAYS} days. They stop working soon; the{" "}+ {V4_MIGRATION_LOOKBACK_DAYS} days. Counts refresh about every 15+ minutes and can lag live traffic. They stop working soon; the{" "}<ExternalLinkhref={DEPRECATED_API_MIGRATION_URL}analytics={{ section: "apis", link: "deprecated_api_docs" }}@@ -940,7 +941,8 @@ export function V4MigrationApisSection({) : (<p className="text-muted-foreground text-sm">No deprecated public API usage detected in the last{" "}- {V4_MIGRATION_LOOKBACK_DAYS} days.+ {V4_MIGRATION_LOOKBACK_DAYS} days. This check refreshes about every 15+ minutes.</p>)}</Section>@@ -1437,7 +1439,9 @@ export function V4MigrationDetailsContent({</div><p className="text-muted-foreground text-sm">SDK, instrumentation, experiment, and API checks cover activity from- the last {V4_MIGRATION_LOOKBACK_DAYS} days.+ the last {V4_MIGRATION_LOOKBACK_DAYS} days. API and experiment usage+ counts refresh about every 15 minutes, so recent calls may not appear+ yet.</p><div><V4MigrationSdkSectiondiff --git a/worker/src/features/v4/v4LegacyApiUsageMetrics.ts b/worker/src/features/v4/v4LegacyApiUsageMetrics.tsindex 331a9c19e5..f9783ea7ff 100644--- a/worker/src/features/v4/v4LegacyApiUsageMetrics.ts+++ b/worker/src/features/v4/v4LegacyApiUsageMetrics.ts@@ -11,7 +11,7 @@ import {* A single gauge is enough to alert on staleness: the worker only refreshes* the heartbeat after a fully successful run, so a stuck, failing, or missing* job lets `heartbeat_age_seconds` climb past the freshness window. Emitted- * from QueueMetricsRunner so age keeps rising between the hourly job runs.+ * from QueueMetricsRunner so age keeps rising between the 15-minute job runs.** Alert: `avg:langfuse.v4.legacy_api_usage.heartbeat_age_seconds > 10800` (3h).*/