🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(monitors): stop pausing monitors on transient ClickHouse resource errors (#16217)
Mark Salpeter1311890b2289
diff --git a/packages/shared/src/features/monitors/processor/processor.ts b/packages/shared/src/features/monitors/processor/processor.tsindex a3ea69b44c..5dbb18460d 100644--- a/packages/shared/src/features/monitors/processor/processor.ts+++ b/packages/shared/src/features/monitors/processor/processor.ts@@ -9,6 +9,7 @@ import {instrumentSync,} from "../../../server/instrumentation";import { logger } from "../../../server/logger";+import { ClickHouseResourceError } from "../../../server/repositories/clickhouse";import {getTriggerConfigurations as defaultGetTriggerConfigurations,type TriggerDomainWithActions,@@ -162,6 +163,16 @@ export class MonitorProcessor {}metricMap["count_count"] = parseNumericValue(row["count_count"]);} catch (error) {+ // Resource pressure is transient, not a bad query; rethrow so the monitor stays ACTIVE and the scheduler retries.+ if (error instanceof ClickHouseResourceError) {+ logger.warn("queryMetrics hit a ClickHouse resource limit; retrying", {+ errorType: error.errorType,+ projectId: event.projectId,+ schedulerBatchId: event.schedulerBatchId.toString(),+ monitorIds: event.monitors.map((m) => m.monitorId),+ });+ throw error;+ }logger.error("queryMetrics failed; flipping affected monitors to ERROR_BAD_QUERY",{diff --git a/worker/package.json b/worker/package.jsonindex f4c936a19e..99aa1542cf 100644--- a/worker/package.json+++ b/worker/package.json@@ -26,7 +26,8 @@"lint:fix": "eslint . --cache --cache-location dist/.eslintcache --fix --concurrency 2","refill-ingestion-events": "dotenv -e ../.env -- tsx src/scripts/replayIngestionEvents/s3-ingestion-event-replay.ts","refill-billing-event": "dotenv -e ../.env -- tsx src/scripts/refill-billing-event.ts",- "refill-queue-event": "dotenv -e ../.env -- tsx src/scripts/refillQueueEvent/index.ts"+ "refill-queue-event": "dotenv -e ../.env -- tsx src/scripts/refillQueueEvent/index.ts",+ "retry-monitor-bad-query-errors": "dotenv -e ../.env -- tsx src/scripts/retryMonitorBadQueryErrors.ts"},"author": "[email protected]","dependencies": {diff --git a/worker/src/__tests__/monitorProcessor.test.ts b/worker/src/__tests__/monitorProcessor.test.tsindex fe130ddb46..1152fa244c 100644--- a/worker/src/__tests__/monitorProcessor.test.ts+++ b/worker/src/__tests__/monitorProcessor.test.ts@@ -1,7 +1,11 @@import { v4 } from "uuid";import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";-import { createOrgProjectAndApiKey, logger } from "@langfuse/shared/src/server";+import {+ ClickHouseResourceError,+ createOrgProjectAndApiKey,+ logger,+} from "@langfuse/shared/src/server";import { InvalidRequestError } from "@langfuse/shared";import {MonitorProcessor,@@ -96,7 +100,7 @@ type ProcessCase = {injectError?: {stage: InjectErrorStage;message: string;- errorClass?: "invalid-request";+ errorClass?: "invalid-request" | "resource";};preempt?: { newClaimedAt: Date };preemptPause?: { at: Date };@@ -700,6 +704,37 @@ const cases: ProcessCase[] = [],},},+ {+ name: "ClickHouseResourceError on executeQuery: rethrows, stays ACTIVE, not ERROR_BAD_QUERY",+ monitors: [+ {+ id: monitorAId,+ severity: MonitorSeveritySchema.enum.OK,+ severityChangedAt: tenMinutesAgo,+ lastPublishedAt: runAt,+ },+ ],+ injectError: {+ stage: "executeQuery",+ errorClass: "resource",+ message: "Timeout exceeded",+ },+ expect: {+ throws: "Timeout exceeded",+ publishCallCount: 0,+ rows: [+ {+ id: monitorAId,+ status: MonitorStatusSchema.enum.ACTIVE,+ severity: MonitorSeveritySchema.enum.OK,+ severityChangedAt: tenMinutesAgo,+ alertedAt: null,+ lastClaimedAt: justAfterRunAt,+ lastCompletedAt: null,+ },+ ],+ },+ },{name: "error on getTriggers: claim, no complete, no sev change, no emit",monitors: [@@ -1311,6 +1346,12 @@ describe("MonitorProcessor.process (integration)", () => {if (c.injectError.errorClass === "invalid-request") {throw new InvalidRequestError(c.injectError.message);}+ if (c.injectError.errorClass === "resource") {+ throw new ClickHouseResourceError(+ "TIMEOUT",+ new Error(c.injectError.message),+ );+ }throw new Error(c.injectError.message);}return c.ch ?? [{ count_count: 0 }];diff --git a/worker/src/__tests__/retryMonitorBadQueryErrors.test.ts b/worker/src/__tests__/retryMonitorBadQueryErrors.test.tsnew file mode 100644index 0000000000..9098af8f17--- /dev/null+++ b/worker/src/__tests__/retryMonitorBadQueryErrors.test.ts@@ -0,0 +1,28 @@+import { describe, expect, it } from "vitest";++import { __test } from "../scripts/retryMonitorBadQueryErrors";++const { parseArgs } = __test;++describe("retryMonitorBadQueryErrors parseArgs", () => {+ it("defaults to a dry run so an accidental invocation never mutates", () => {+ expect(parseArgs([])).toEqual({ apply: false, jitterMinutes: 15 });+ });++ it("enables the update only when --apply is present", () => {+ expect(parseArgs(["--apply"]).apply).toBe(true);+ });++ it("reads a custom --jitter-minutes", () => {+ expect(parseArgs(["--jitter-minutes=15"]).jitterMinutes).toBe(15);+ });++ it("allows a zero jitter for a big-bang reactivation", () => {+ expect(parseArgs(["--jitter-minutes=0"]).jitterMinutes).toBe(0);+ });++ it("falls back to the default on a negative or non-numeric jitter", () => {+ expect(parseArgs(["--jitter-minutes=-5"]).jitterMinutes).toBe(15);+ expect(parseArgs(["--jitter-minutes=abc"]).jitterMinutes).toBe(15);+ });+});diff --git a/worker/src/scripts/retryMonitorBadQueryErrors.ts b/worker/src/scripts/retryMonitorBadQueryErrors.tsnew file mode 100644index 0000000000..b829407e0d--- /dev/null+++ b/worker/src/scripts/retryMonitorBadQueryErrors.ts@@ -0,0 +1,97 @@+#!/usr/bin/env -S tsx+/**+ * One-off remediation that reactivates monitors paused at ERROR_BAD_QUERY,+ * for use after a fix stops a transient failure from flipping them there.+ *+ * Reactivating sets severity to UNKNOWN (rendered as "pending" in the UI) and+ * jitters each monitor's first run across --jitter-minutes so their queries+ * don't all re-fire in one scheduler tick; the default keeps the pending+ * window short. Deploy the flip fix first, or the next failure re-pauses them.+ *+ * Point DATABASE_URL at the target env (via tunnel), dry-run, then --apply:+ *+ * USAGE:+ * pnpm --filter=worker retry-monitor-bad-query-errors+ * pnpm --filter=worker retry-monitor-bad-query-errors --apply+ * pnpm --filter=worker retry-monitor-bad-query-errors --apply --jitter-minutes=30+ */++import { prisma } from "@langfuse/shared/src/db";+import { logger } from "@langfuse/shared/src/server";++/** defaultJitterMinutes spreads the reactivated monitors' first run over this window, kept short so the UNKNOWN "pending" state clears within ~15 minutes. */+const defaultJitterMinutes = 15;++/** retryMonitorBadQueryErrors reactivates every monitor stuck in ERROR_BAD_QUERY, mirroring the service resume transition and jittering the first run. */+async function retryMonitorBadQueryErrors(args: {+ apply: boolean;+ jitterMinutes: number;+}): Promise<void> {+ const preview = await countStuckMonitors();+ logger.info(+ `retryMonitorBadQueryErrors: ${preview.monitors} monitors in ERROR_BAD_QUERY across ${preview.queryShapes} query shapes (apply=${args.apply}, jitterMinutes=${args.jitterMinutes})`,+ );++ if (!args.apply) {+ logger.info("retryMonitorBadQueryErrors: dry run, pass --apply to execute");+ return;+ }++ const reactivated = await prisma.$executeRaw`+ UPDATE monitors+ SET status = 'ACTIVE',+ severity = 'UNKNOWN',+ severity_changed_at = now(),+ next_run_at = now() + random() * make_interval(mins => ${args.jitterMinutes}::int),+ last_published_at = NULL,+ last_completed_at = NULL,+ last_claimed_at = NULL,+ alerted_at = NULL+ WHERE status = 'ERROR_BAD_QUERY'+ `;+ logger.info(+ `retryMonitorBadQueryErrors: reactivated ${reactivated} monitors`,+ );+}++/** countStuckMonitors returns how many ERROR_BAD_QUERY monitors exist and how many distinct query shapes they collapse into. */+async function countStuckMonitors(): Promise<{+ monitors: number;+ queryShapes: number;+}> {+ const [row] = await prisma.$queryRaw<+ { monitors: bigint; query_shapes: bigint }[]+ >`+ SELECT count(*) AS monitors,+ count(DISTINCT (project_id, scheduler_batch_id)) AS query_shapes+ FROM monitors+ WHERE status = 'ERROR_BAD_QUERY'+ `;+ return {+ monitors: Number(row?.monitors ?? 0),+ queryShapes: Number(row?.query_shapes ?? 0),+ };+}++/** parseArgs reads --apply and --jitter-minutes from argv. */+function parseArgs(argv: string[]): { apply: boolean; jitterMinutes: number } {+ const apply = argv.includes("--apply");+ const jitterArg = argv+ .find((a) => a.startsWith("--jitter-minutes="))+ ?.split("=")[1];+ const parsed = jitterArg === undefined ? NaN : Number(jitterArg);+ const jitterMinutes =+ Number.isFinite(parsed) && parsed >= 0 ? parsed : defaultJitterMinutes;+ return { apply, jitterMinutes };+}++if (require.main === module) {+ retryMonitorBadQueryErrors(parseArgs(process.argv.slice(2)))+ .catch((err) => {+ logger.error("retryMonitorBadQueryErrors failed", { err });+ process.exit(1);+ })+ .finally(() => process.exit(0));+}++export const __test = { parseArgs };