Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(clickhouse): classify memory limit variants as ClickHouseResourceError (#17194)

Niklas Semmlerce8aa1a9323b
diff --git a/packages/shared/src/features/monitors/processor/processor.ts b/packages/shared/src/features/monitors/processor/processor.ts
index 5dbb18460d..561e6811f2 100644
--- a/packages/shared/src/features/monitors/processor/processor.ts
+++ b/packages/shared/src/features/monitors/processor/processor.ts
@@ -164,7 +164,7 @@ export class MonitorProcessor {
metricMap["count_count"] = parseNumericValue(row["count_count"]);
} catch (error) {
// Resource pressure is transient, not a bad query; rethrow so the monitor stays ACTIVE and the scheduler retries.
- if (error instanceof ClickHouseResourceError) {
+ if (ClickHouseResourceError.is(error)) {
logger.warn("queryMetrics hit a ClickHouse resource limit; retrying", {
errorType: error.errorType,
projectId: event.projectId,
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index 97743be895..7d7e6f5d10 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -55,14 +55,23 @@ const ERROR_TYPE_CONFIG: Record<
discriminators: string[];
}
> = {
- MEMORY_LIMIT: {
- discriminators: ["memory limit exceeded"],
- },
+ // Order matters: matched top-to-bottom, first hit wins. OvercommitTracker
+ // kills also carry a "Memory limit … exceeded" phrase, so OVERCOMMIT must
+ // precede MEMORY_LIMIT to keep the more specific cause in the outcome metric.
OVERCOMMIT: {
- discriminators: ["OvercommitTracker"],
+ discriminators: ["overcommittracker"],
+ },
+ MEMORY_LIMIT: {
+ discriminators: [
+ "memory limit exceeded",
+ "memory limit (for query) exceeded",
+ "memory limit (total) exceeded",
+ "memory limit (for user) exceeded",
+ "memory limit",
+ ],
},
TIMEOUT: {
- discriminators: ["Timeout", "timeout", "timed out"],
+ discriminators: ["timeout", "timed out"],
},
};
@@ -89,11 +98,18 @@ export class ClickHouseResourceError extends Error {
}
}
+ static is(error: unknown): error is ClickHouseResourceError {
+ return (
+ error instanceof ClickHouseResourceError ||
+ (error instanceof Error && error.name === "ClickHouseResourceError")
+ );
+ }
+
static wrapIfResourceError(
originalError: Error,
tags?: NormalizedClickHouseQueryTags,
): Error {
- const errorMessage = originalError.message || "";
+ const errorMessage = (originalError.message || "").toLowerCase();
for (const [type, config] of Object.entries(ERROR_TYPE_CONFIG) as Array<
[
@@ -102,7 +118,7 @@ export class ClickHouseResourceError extends Error {
]
>) {
const hasDiscriminator = config.discriminators.some((discriminator) =>
- errorMessage.includes(discriminator),
+ errorMessage.includes(discriminator.toLowerCase()),
);
if (hasDiscriminator) {
diff --git a/packages/shared/src/server/repositories/clickhouseResourceError.test.ts b/packages/shared/src/server/repositories/clickhouseResourceError.test.ts
new file mode 100644
index 0000000000..b61ace9dba
--- /dev/null
+++ b/packages/shared/src/server/repositories/clickhouseResourceError.test.ts
@@ -0,0 +1,121 @@
+import { describe, it, expect } from "vitest";
+import { ClickHouseResourceError } from "./clickhouse";
+
+describe("ClickHouseResourceError", () => {
+ describe("wrapIfResourceError", () => {
+ const cases = [
+ {
+ name: "capitalized per-query memory limit",
+ message:
+ "Memory limit (for query) exceeded: would use 2.25 GiB (attempt to allocate chunk of 0.00 B), current RSS: 1.42 GiB, maximum: 2.25 GiB",
+ expectedType: "MEMORY_LIMIT",
+ },
+ {
+ name: "capitalized total memory limit",
+ message:
+ "Memory limit (total) exceeded: would use 2.25 GiB, current RSS: 1.42 GiB, maximum: 2.25 GiB",
+ expectedType: "MEMORY_LIMIT",
+ },
+ {
+ name: "capitalized user memory limit",
+ message: "Memory limit (for user) exceeded: would use 2.25 GiB",
+ expectedType: "MEMORY_LIMIT",
+ },
+ {
+ name: "lowercase total memory limit",
+ message:
+ "(total) memory limit exceeded: would use 2.25 GiB (attempt to allocate chunk of 0.00 B)",
+ expectedType: "MEMORY_LIMIT",
+ },
+ {
+ name: "simple memory limit exceeded",
+ message: "memory limit exceeded",
+ expectedType: "MEMORY_LIMIT",
+ },
+ {
+ name: "OvercommitTracker decision",
+ message:
+ "OvercommitTracker decision: Query was selected to stop by OvercommitTracker: While executing AggregatingTransform",
+ expectedType: "OVERCOMMIT",
+ },
+ {
+ // Overcommit kills also carry a "Memory limit … exceeded" phrase; the
+ // more specific OVERCOMMIT cause must win so the outcome metric stays accurate.
+ name: "overcommit kill carrying a memory limit phrase",
+ message:
+ "Memory limit (total) exceeded: would use 9.32 GiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker",
+ expectedType: "OVERCOMMIT",
+ },
+ {
+ name: "case-insensitive overcommittracker",
+ message: "query stopped by overcommittracker",
+ expectedType: "OVERCOMMIT",
+ },
+ {
+ name: "socket timeout",
+ message: "Timeout exceeded while reading from socket",
+ expectedType: "TIMEOUT",
+ },
+ {
+ name: "timed out error",
+ message: "Request timed out after 30000ms",
+ expectedType: "TIMEOUT",
+ },
+ ];
+
+ cases.forEach(({ name, message, expectedType }) => {
+ it(`wraps ${name} as ${expectedType}`, () => {
+ const error = new Error(message);
+ const wrapped = ClickHouseResourceError.wrapIfResourceError(error);
+
+ expect(wrapped).toBeInstanceOf(ClickHouseResourceError);
+ expect(ClickHouseResourceError.is(wrapped)).toBe(true);
+ expect((wrapped as ClickHouseResourceError).errorType).toBe(
+ expectedType,
+ );
+ });
+ });
+
+ const benignCases = [
+ "Table 'test.non_existent' doesn't exist",
+ "Unknown identifier 'foo' in scope SELECT foo FROM traces",
+ "Syntax error: failed at position 42",
+ "Cannot parse input: expected ')' before end of stream",
+ ];
+
+ benignCases.forEach((message) => {
+ it(`does not wrap benign error: ${message.slice(0, 32)}`, () => {
+ const error = new Error(message);
+ const wrapped = ClickHouseResourceError.wrapIfResourceError(error);
+
+ expect(wrapped).toBe(error);
+ expect(wrapped).not.toBeInstanceOf(ClickHouseResourceError);
+ expect(ClickHouseResourceError.is(wrapped)).toBe(false);
+ });
+ });
+ });
+
+ describe("is", () => {
+ it("returns true for ClickHouseResourceError instances", () => {
+ const error = new ClickHouseResourceError(
+ "MEMORY_LIMIT",
+ new Error("out of memory"),
+ );
+ expect(ClickHouseResourceError.is(error)).toBe(true);
+ });
+
+ it("returns true for errors matching name ClickHouseResourceError across boundaries", () => {
+ const fakeError = new Error("resource limit");
+ fakeError.name = "ClickHouseResourceError";
+ expect(ClickHouseResourceError.is(fakeError)).toBe(true);
+ });
+
+ it("returns false for standard errors and non-errors", () => {
+ expect(ClickHouseResourceError.is(new Error("generic"))).toBe(false);
+ expect(ClickHouseResourceError.is(null)).toBe(false);
+ expect(ClickHouseResourceError.is(undefined)).toBe(false);
+ expect(ClickHouseResourceError.is("ClickHouseResourceError")).toBe(false);
+ expect(ClickHouseResourceError.is({})).toBe(false);
+ });
+ });
+});
diff --git a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
index 21e4b3dfd0..9ff6c74a2c 100644
--- a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
+++ b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
@@ -248,6 +248,33 @@ describe("ClickHouse Resource Error Handling", () => {
shouldBeResourceError: true,
errorType: "OVERCOMMIT",
},
+ {
+ name: "Memory limit for query exceeded (capitalized)",
+ errorMessage:
+ "Memory limit (for query) exceeded: would use 2.25 GiB (attempt to allocate chunk of 0.00 B), current RSS: 1.42 GiB, maximum: 2.25 GiB",
+ shouldBeResourceError: true,
+ errorType: "MEMORY_LIMIT",
+ },
+ {
+ name: "Memory limit total exceeded (capitalized)",
+ errorMessage:
+ "Memory limit (total) exceeded: would use 2.25 GiB, current RSS: 1.42 GiB, maximum: 2.25 GiB",
+ shouldBeResourceError: true,
+ errorType: "MEMORY_LIMIT",
+ },
+ {
+ name: "Memory limit for user exceeded (capitalized)",
+ errorMessage: "Memory limit (for user) exceeded: would use 2.25 GiB",
+ shouldBeResourceError: true,
+ errorType: "MEMORY_LIMIT",
+ },
+ {
+ name: "Overcommit decision with memory limit message",
+ errorMessage:
+ "(total) memory limit exceeded: would use 2.25 GiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker: While executing AggregatingTransform",
+ shouldBeResourceError: true,
+ errorType: "OVERCOMMIT",
+ },
{
name: "Simple memory error",
errorMessage: "memory limit exceeded",
@@ -288,6 +315,9 @@ describe("ClickHouse Resource Error Handling", () => {
})(wrappedError);
expect(isResourceError).toBe(shouldBeResourceError);
+ expect(ClickHouseResourceError.is(wrappedError)).toBe(
+ shouldBeResourceError,
+ );
const resourceError = wrappedError as ClickHouseResourceError;
if (shouldBeResourceError && errorType) {
diff --git a/worker/src/__tests__/monitorProcessor.test.ts b/worker/src/__tests__/monitorProcessor.test.ts
index 1152fa244c..e2c9dead19 100644
--- a/worker/src/__tests__/monitorProcessor.test.ts
+++ b/worker/src/__tests__/monitorProcessor.test.ts
@@ -100,7 +100,7 @@ type ProcessCase = {
injectError?: {
stage: InjectErrorStage;
message: string;
- errorClass?: "invalid-request" | "resource";
+ errorClass?: "invalid-request" | "resource" | "resource-memory";
};
preempt?: { newClaimedAt: Date };
preemptPause?: { at: Date };
@@ -735,6 +735,37 @@ const cases: ProcessCase[] = [
],
},
},
+ {
+ name: "ClickHouse memory limit resource error on executeQuery: rethrows, stays ACTIVE, not ERROR_BAD_QUERY",
+ monitors: [
+ {
+ id: monitorAId,
+ severity: MonitorSeveritySchema.enum.OK,
+ severityChangedAt: tenMinutesAgo,
+ lastPublishedAt: runAt,
+ },
+ ],
+ injectError: {
+ stage: "executeQuery",
+ errorClass: "resource-memory",
+ message: "Memory limit (for query) exceeded: would use 2.25 GiB",
+ },
+ expect: {
+ throws: "Memory limit (for query) exceeded: would use 2.25 GiB",
+ publishCallCount: 0,
+ rows: [
+ {
+ id: monitorAId,
+ status: MonitorStatusSchema.enum.ACTIVE,
+ severity: MonitorSeveritySchema.enum.OK,
+ severityChangedAt: tenMinutesAgo,
+ alertedAt: null,
+ lastClaimedAt: justAfterRunAt,
+ lastCompletedAt: null,
+ },
+ ],
+ },
+ },
{
name: "error on getTriggers: claim, no complete, no sev change, no emit",
monitors: [
@@ -1352,6 +1383,11 @@ describe("MonitorProcessor.process (integration)", () => {
new Error(c.injectError.message),
);
}
+ if (c.injectError.errorClass === "resource-memory") {
+ throw ClickHouseResourceError.wrapIfResourceError(
+ new Error(c.injectError.message),
+ );
+ }
throw new Error(c.injectError.message);
}
return c.ch ?? [{ count_count: 0 }];