Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(monitors): stop pausing monitors on transient ClickHouse resource errors (#16217)

Mark Salpeter1311890b2289
diff --git a/packages/shared/src/features/monitors/processor/processor.ts b/packages/shared/src/features/monitors/processor/processor.ts
index a3ea69b44c..5dbb18460d 100644
--- a/packages/shared/src/features/monitors/processor/processor.ts
+++ b/packages/shared/src/features/monitors/processor/processor.ts
@@ -9,6 +9,7 @@ import {
instrumentSync,
} from "../../../server/instrumentation";
import { logger } from "../../../server/logger";
+import { ClickHouseResourceError } from "../../../server/repositories/clickhouse";
import {
getTriggerConfigurations as defaultGetTriggerConfigurations,
type TriggerDomainWithActions,
@@ -162,6 +163,16 @@ export class MonitorProcessor {
}
metricMap["count_count"] = parseNumericValue(row["count_count"]);
} catch (error) {
+ // Resource pressure is transient, not a bad query; rethrow so the monitor stays ACTIVE and the scheduler retries.
+ if (error instanceof ClickHouseResourceError) {
+ logger.warn("queryMetrics hit a ClickHouse resource limit; retrying", {
+ errorType: error.errorType,
+ projectId: event.projectId,
+ schedulerBatchId: event.schedulerBatchId.toString(),
+ monitorIds: event.monitors.map((m) => m.monitorId),
+ });
+ throw error;
+ }
logger.error(
"queryMetrics failed; flipping affected monitors to ERROR_BAD_QUERY",
{
diff --git a/worker/package.json b/worker/package.json
index f4c936a19e..99aa1542cf 100644
--- a/worker/package.json
+++ b/worker/package.json
@@ -26,7 +26,8 @@
"lint:fix": "eslint . --cache --cache-location dist/.eslintcache --fix --concurrency 2",
"refill-ingestion-events": "dotenv -e ../.env -- tsx src/scripts/replayIngestionEvents/s3-ingestion-event-replay.ts",
"refill-billing-event": "dotenv -e ../.env -- tsx src/scripts/refill-billing-event.ts",
- "refill-queue-event": "dotenv -e ../.env -- tsx src/scripts/refillQueueEvent/index.ts"
+ "refill-queue-event": "dotenv -e ../.env -- tsx src/scripts/refillQueueEvent/index.ts",
+ "retry-monitor-bad-query-errors": "dotenv -e ../.env -- tsx src/scripts/retryMonitorBadQueryErrors.ts"
},
"author": "[email protected]",
"dependencies": {
diff --git a/worker/src/__tests__/monitorProcessor.test.ts b/worker/src/__tests__/monitorProcessor.test.ts
index fe130ddb46..1152fa244c 100644
--- a/worker/src/__tests__/monitorProcessor.test.ts
+++ b/worker/src/__tests__/monitorProcessor.test.ts
@@ -1,7 +1,11 @@
import { v4 } from "uuid";
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
-import { createOrgProjectAndApiKey, logger } from "@langfuse/shared/src/server";
+import {
+ ClickHouseResourceError,
+ createOrgProjectAndApiKey,
+ logger,
+} from "@langfuse/shared/src/server";
import { InvalidRequestError } from "@langfuse/shared";
import {
MonitorProcessor,
@@ -96,7 +100,7 @@ type ProcessCase = {
injectError?: {
stage: InjectErrorStage;
message: string;
- errorClass?: "invalid-request";
+ errorClass?: "invalid-request" | "resource";
};
preempt?: { newClaimedAt: Date };
preemptPause?: { at: Date };
@@ -700,6 +704,37 @@ const cases: ProcessCase[] = [
],
},
},
+ {
+ name: "ClickHouseResourceError on executeQuery: rethrows, stays ACTIVE, not ERROR_BAD_QUERY",
+ monitors: [
+ {
+ id: monitorAId,
+ severity: MonitorSeveritySchema.enum.OK,
+ severityChangedAt: tenMinutesAgo,
+ lastPublishedAt: runAt,
+ },
+ ],
+ injectError: {
+ stage: "executeQuery",
+ errorClass: "resource",
+ message: "Timeout exceeded",
+ },
+ expect: {
+ throws: "Timeout exceeded",
+ publishCallCount: 0,
+ rows: [
+ {
+ id: monitorAId,
+ status: MonitorStatusSchema.enum.ACTIVE,
+ severity: MonitorSeveritySchema.enum.OK,
+ severityChangedAt: tenMinutesAgo,
+ alertedAt: null,
+ lastClaimedAt: justAfterRunAt,
+ lastCompletedAt: null,
+ },
+ ],
+ },
+ },
{
name: "error on getTriggers: claim, no complete, no sev change, no emit",
monitors: [
@@ -1311,6 +1346,12 @@ describe("MonitorProcessor.process (integration)", () => {
if (c.injectError.errorClass === "invalid-request") {
throw new InvalidRequestError(c.injectError.message);
}
+ if (c.injectError.errorClass === "resource") {
+ throw new ClickHouseResourceError(
+ "TIMEOUT",
+ new Error(c.injectError.message),
+ );
+ }
throw new Error(c.injectError.message);
}
return c.ch ?? [{ count_count: 0 }];
diff --git a/worker/src/__tests__/retryMonitorBadQueryErrors.test.ts b/worker/src/__tests__/retryMonitorBadQueryErrors.test.ts
new file mode 100644
index 0000000000..9098af8f17
--- /dev/null
+++ b/worker/src/__tests__/retryMonitorBadQueryErrors.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+
+import { __test } from "../scripts/retryMonitorBadQueryErrors";
+
+const { parseArgs } = __test;
+
+describe("retryMonitorBadQueryErrors parseArgs", () => {
+ it("defaults to a dry run so an accidental invocation never mutates", () => {
+ expect(parseArgs([])).toEqual({ apply: false, jitterMinutes: 15 });
+ });
+
+ it("enables the update only when --apply is present", () => {
+ expect(parseArgs(["--apply"]).apply).toBe(true);
+ });
+
+ it("reads a custom --jitter-minutes", () => {
+ expect(parseArgs(["--jitter-minutes=15"]).jitterMinutes).toBe(15);
+ });
+
+ it("allows a zero jitter for a big-bang reactivation", () => {
+ expect(parseArgs(["--jitter-minutes=0"]).jitterMinutes).toBe(0);
+ });
+
+ it("falls back to the default on a negative or non-numeric jitter", () => {
+ expect(parseArgs(["--jitter-minutes=-5"]).jitterMinutes).toBe(15);
+ expect(parseArgs(["--jitter-minutes=abc"]).jitterMinutes).toBe(15);
+ });
+});
diff --git a/worker/src/scripts/retryMonitorBadQueryErrors.ts b/worker/src/scripts/retryMonitorBadQueryErrors.ts
new file mode 100644
index 0000000000..b829407e0d
--- /dev/null
+++ b/worker/src/scripts/retryMonitorBadQueryErrors.ts
@@ -0,0 +1,97 @@
+#!/usr/bin/env -S tsx
+/**
+ * One-off remediation that reactivates monitors paused at ERROR_BAD_QUERY,
+ * for use after a fix stops a transient failure from flipping them there.
+ *
+ * Reactivating sets severity to UNKNOWN (rendered as "pending" in the UI) and
+ * jitters each monitor's first run across --jitter-minutes so their queries
+ * don't all re-fire in one scheduler tick; the default keeps the pending
+ * window short. Deploy the flip fix first, or the next failure re-pauses them.
+ *
+ * Point DATABASE_URL at the target env (via tunnel), dry-run, then --apply:
+ *
+ * USAGE:
+ * pnpm --filter=worker retry-monitor-bad-query-errors
+ * pnpm --filter=worker retry-monitor-bad-query-errors --apply
+ * pnpm --filter=worker retry-monitor-bad-query-errors --apply --jitter-minutes=30
+ */
+
+import { prisma } from "@langfuse/shared/src/db";
+import { logger } from "@langfuse/shared/src/server";
+
+/** defaultJitterMinutes spreads the reactivated monitors' first run over this window, kept short so the UNKNOWN "pending" state clears within ~15 minutes. */
+const defaultJitterMinutes = 15;
+
+/** retryMonitorBadQueryErrors reactivates every monitor stuck in ERROR_BAD_QUERY, mirroring the service resume transition and jittering the first run. */
+async function retryMonitorBadQueryErrors(args: {
+ apply: boolean;
+ jitterMinutes: number;
+}): Promise<void> {
+ const preview = await countStuckMonitors();
+ logger.info(
+ `retryMonitorBadQueryErrors: ${preview.monitors} monitors in ERROR_BAD_QUERY across ${preview.queryShapes} query shapes (apply=${args.apply}, jitterMinutes=${args.jitterMinutes})`,
+ );
+
+ if (!args.apply) {
+ logger.info("retryMonitorBadQueryErrors: dry run, pass --apply to execute");
+ return;
+ }
+
+ const reactivated = await prisma.$executeRaw`
+ UPDATE monitors
+ SET status = 'ACTIVE',
+ severity = 'UNKNOWN',
+ severity_changed_at = now(),
+ next_run_at = now() + random() * make_interval(mins => ${args.jitterMinutes}::int),
+ last_published_at = NULL,
+ last_completed_at = NULL,
+ last_claimed_at = NULL,
+ alerted_at = NULL
+ WHERE status = 'ERROR_BAD_QUERY'
+ `;
+ logger.info(
+ `retryMonitorBadQueryErrors: reactivated ${reactivated} monitors`,
+ );
+}
+
+/** countStuckMonitors returns how many ERROR_BAD_QUERY monitors exist and how many distinct query shapes they collapse into. */
+async function countStuckMonitors(): Promise<{
+ monitors: number;
+ queryShapes: number;
+}> {
+ const [row] = await prisma.$queryRaw<
+ { monitors: bigint; query_shapes: bigint }[]
+ >`
+ SELECT count(*) AS monitors,
+ count(DISTINCT (project_id, scheduler_batch_id)) AS query_shapes
+ FROM monitors
+ WHERE status = 'ERROR_BAD_QUERY'
+ `;
+ return {
+ monitors: Number(row?.monitors ?? 0),
+ queryShapes: Number(row?.query_shapes ?? 0),
+ };
+}
+
+/** parseArgs reads --apply and --jitter-minutes from argv. */
+function parseArgs(argv: string[]): { apply: boolean; jitterMinutes: number } {
+ const apply = argv.includes("--apply");
+ const jitterArg = argv
+ .find((a) => a.startsWith("--jitter-minutes="))
+ ?.split("=")[1];
+ const parsed = jitterArg === undefined ? NaN : Number(jitterArg);
+ const jitterMinutes =
+ Number.isFinite(parsed) && parsed >= 0 ? parsed : defaultJitterMinutes;
+ return { apply, jitterMinutes };
+}
+
+if (require.main === module) {
+ retryMonitorBadQueryErrors(parseArgs(process.argv.slice(2)))
+ .catch((err) => {
+ logger.error("retryMonitorBadQueryErrors failed", { err });
+ process.exit(1);
+ })
+ .finally(() => process.exit(0));
+}
+
+export const __test = { parseArgs };