🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(clickhouse): classify memory limit variants as ClickHouseResourceError (#17194)
Niklas Semmlerce8aa1a9323b
diff --git a/packages/shared/src/features/monitors/processor/processor.ts b/packages/shared/src/features/monitors/processor/processor.tsindex 5dbb18460d..561e6811f2 100644--- a/packages/shared/src/features/monitors/processor/processor.ts+++ b/packages/shared/src/features/monitors/processor/processor.ts@@ -164,7 +164,7 @@ export class MonitorProcessor {metricMap["count_count"] = parseNumericValue(row["count_count"]);} catch (error) {// Resource pressure is transient, not a bad query; rethrow so the monitor stays ACTIVE and the scheduler retries.- if (error instanceof ClickHouseResourceError) {+ if (ClickHouseResourceError.is(error)) {logger.warn("queryMetrics hit a ClickHouse resource limit; retrying", {errorType: error.errorType,projectId: event.projectId,diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex 97743be895..7d7e6f5d10 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -55,14 +55,23 @@ const ERROR_TYPE_CONFIG: Record<discriminators: string[];}> = {- MEMORY_LIMIT: {- discriminators: ["memory limit exceeded"],- },+ // Order matters: matched top-to-bottom, first hit wins. OvercommitTracker+ // kills also carry a "Memory limit … exceeded" phrase, so OVERCOMMIT must+ // precede MEMORY_LIMIT to keep the more specific cause in the outcome metric.OVERCOMMIT: {- discriminators: ["OvercommitTracker"],+ discriminators: ["overcommittracker"],+ },+ MEMORY_LIMIT: {+ discriminators: [+ "memory limit exceeded",+ "memory limit (for query) exceeded",+ "memory limit (total) exceeded",+ "memory limit (for user) exceeded",+ "memory limit",+ ],},TIMEOUT: {- discriminators: ["Timeout", "timeout", "timed out"],+ discriminators: ["timeout", "timed out"],},};@@ -89,11 +98,18 @@ export class ClickHouseResourceError extends Error {}}+ static is(error: unknown): error is ClickHouseResourceError {+ return (+ error instanceof ClickHouseResourceError ||+ (error instanceof Error && error.name === "ClickHouseResourceError")+ );+ }+static wrapIfResourceError(originalError: Error,tags?: NormalizedClickHouseQueryTags,): Error {- const errorMessage = originalError.message || "";+ const errorMessage = (originalError.message || "").toLowerCase();for (const [type, config] of Object.entries(ERROR_TYPE_CONFIG) as Array<[@@ -102,7 +118,7 @@ export class ClickHouseResourceError extends Error {]>) {const hasDiscriminator = config.discriminators.some((discriminator) =>- errorMessage.includes(discriminator),+ errorMessage.includes(discriminator.toLowerCase()),);if (hasDiscriminator) {diff --git a/packages/shared/src/server/repositories/clickhouseResourceError.test.ts b/packages/shared/src/server/repositories/clickhouseResourceError.test.tsnew file mode 100644index 0000000000..b61ace9dba--- /dev/null+++ b/packages/shared/src/server/repositories/clickhouseResourceError.test.ts@@ -0,0 +1,121 @@+import { describe, it, expect } from "vitest";+import { ClickHouseResourceError } from "./clickhouse";++describe("ClickHouseResourceError", () => {+ describe("wrapIfResourceError", () => {+ const cases = [+ {+ name: "capitalized per-query memory limit",+ message:+ "Memory limit (for query) exceeded: would use 2.25 GiB (attempt to allocate chunk of 0.00 B), current RSS: 1.42 GiB, maximum: 2.25 GiB",+ expectedType: "MEMORY_LIMIT",+ },+ {+ name: "capitalized total memory limit",+ message:+ "Memory limit (total) exceeded: would use 2.25 GiB, current RSS: 1.42 GiB, maximum: 2.25 GiB",+ expectedType: "MEMORY_LIMIT",+ },+ {+ name: "capitalized user memory limit",+ message: "Memory limit (for user) exceeded: would use 2.25 GiB",+ expectedType: "MEMORY_LIMIT",+ },+ {+ name: "lowercase total memory limit",+ message:+ "(total) memory limit exceeded: would use 2.25 GiB (attempt to allocate chunk of 0.00 B)",+ expectedType: "MEMORY_LIMIT",+ },+ {+ name: "simple memory limit exceeded",+ message: "memory limit exceeded",+ expectedType: "MEMORY_LIMIT",+ },+ {+ name: "OvercommitTracker decision",+ message:+ "OvercommitTracker decision: Query was selected to stop by OvercommitTracker: While executing AggregatingTransform",+ expectedType: "OVERCOMMIT",+ },+ {+ // Overcommit kills also carry a "Memory limit … exceeded" phrase; the+ // more specific OVERCOMMIT cause must win so the outcome metric stays accurate.+ name: "overcommit kill carrying a memory limit phrase",+ message:+ "Memory limit (total) exceeded: would use 9.32 GiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker",+ expectedType: "OVERCOMMIT",+ },+ {+ name: "case-insensitive overcommittracker",+ message: "query stopped by overcommittracker",+ expectedType: "OVERCOMMIT",+ },+ {+ name: "socket timeout",+ message: "Timeout exceeded while reading from socket",+ expectedType: "TIMEOUT",+ },+ {+ name: "timed out error",+ message: "Request timed out after 30000ms",+ expectedType: "TIMEOUT",+ },+ ];++ cases.forEach(({ name, message, expectedType }) => {+ it(`wraps ${name} as ${expectedType}`, () => {+ const error = new Error(message);+ const wrapped = ClickHouseResourceError.wrapIfResourceError(error);++ expect(wrapped).toBeInstanceOf(ClickHouseResourceError);+ expect(ClickHouseResourceError.is(wrapped)).toBe(true);+ expect((wrapped as ClickHouseResourceError).errorType).toBe(+ expectedType,+ );+ });+ });++ const benignCases = [+ "Table 'test.non_existent' doesn't exist",+ "Unknown identifier 'foo' in scope SELECT foo FROM traces",+ "Syntax error: failed at position 42",+ "Cannot parse input: expected ')' before end of stream",+ ];++ benignCases.forEach((message) => {+ it(`does not wrap benign error: ${message.slice(0, 32)}`, () => {+ const error = new Error(message);+ const wrapped = ClickHouseResourceError.wrapIfResourceError(error);++ expect(wrapped).toBe(error);+ expect(wrapped).not.toBeInstanceOf(ClickHouseResourceError);+ expect(ClickHouseResourceError.is(wrapped)).toBe(false);+ });+ });+ });++ describe("is", () => {+ it("returns true for ClickHouseResourceError instances", () => {+ const error = new ClickHouseResourceError(+ "MEMORY_LIMIT",+ new Error("out of memory"),+ );+ expect(ClickHouseResourceError.is(error)).toBe(true);+ });++ it("returns true for errors matching name ClickHouseResourceError across boundaries", () => {+ const fakeError = new Error("resource limit");+ fakeError.name = "ClickHouseResourceError";+ expect(ClickHouseResourceError.is(fakeError)).toBe(true);+ });++ it("returns false for standard errors and non-errors", () => {+ expect(ClickHouseResourceError.is(new Error("generic"))).toBe(false);+ expect(ClickHouseResourceError.is(null)).toBe(false);+ expect(ClickHouseResourceError.is(undefined)).toBe(false);+ expect(ClickHouseResourceError.is("ClickHouseResourceError")).toBe(false);+ expect(ClickHouseResourceError.is({})).toBe(false);+ });+ });+});diff --git a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.tsindex 21e4b3dfd0..9ff6c74a2c 100644--- a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts+++ b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts@@ -248,6 +248,33 @@ describe("ClickHouse Resource Error Handling", () => {shouldBeResourceError: true,errorType: "OVERCOMMIT",},+ {+ name: "Memory limit for query exceeded (capitalized)",+ errorMessage:+ "Memory limit (for query) exceeded: would use 2.25 GiB (attempt to allocate chunk of 0.00 B), current RSS: 1.42 GiB, maximum: 2.25 GiB",+ shouldBeResourceError: true,+ errorType: "MEMORY_LIMIT",+ },+ {+ name: "Memory limit total exceeded (capitalized)",+ errorMessage:+ "Memory limit (total) exceeded: would use 2.25 GiB, current RSS: 1.42 GiB, maximum: 2.25 GiB",+ shouldBeResourceError: true,+ errorType: "MEMORY_LIMIT",+ },+ {+ name: "Memory limit for user exceeded (capitalized)",+ errorMessage: "Memory limit (for user) exceeded: would use 2.25 GiB",+ shouldBeResourceError: true,+ errorType: "MEMORY_LIMIT",+ },+ {+ name: "Overcommit decision with memory limit message",+ errorMessage:+ "(total) memory limit exceeded: would use 2.25 GiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker: While executing AggregatingTransform",+ shouldBeResourceError: true,+ errorType: "OVERCOMMIT",+ },{name: "Simple memory error",errorMessage: "memory limit exceeded",@@ -288,6 +315,9 @@ describe("ClickHouse Resource Error Handling", () => {})(wrappedError);expect(isResourceError).toBe(shouldBeResourceError);+ expect(ClickHouseResourceError.is(wrappedError)).toBe(+ shouldBeResourceError,+ );const resourceError = wrappedError as ClickHouseResourceError;if (shouldBeResourceError && errorType) {diff --git a/worker/src/__tests__/monitorProcessor.test.ts b/worker/src/__tests__/monitorProcessor.test.tsindex 1152fa244c..e2c9dead19 100644--- a/worker/src/__tests__/monitorProcessor.test.ts+++ b/worker/src/__tests__/monitorProcessor.test.ts@@ -100,7 +100,7 @@ type ProcessCase = {injectError?: {stage: InjectErrorStage;message: string;- errorClass?: "invalid-request" | "resource";+ errorClass?: "invalid-request" | "resource" | "resource-memory";};preempt?: { newClaimedAt: Date };preemptPause?: { at: Date };@@ -735,6 +735,37 @@ const cases: ProcessCase[] = [],},},+ {+ name: "ClickHouse memory limit resource error on executeQuery: rethrows, stays ACTIVE, not ERROR_BAD_QUERY",+ monitors: [+ {+ id: monitorAId,+ severity: MonitorSeveritySchema.enum.OK,+ severityChangedAt: tenMinutesAgo,+ lastPublishedAt: runAt,+ },+ ],+ injectError: {+ stage: "executeQuery",+ errorClass: "resource-memory",+ message: "Memory limit (for query) exceeded: would use 2.25 GiB",+ },+ expect: {+ throws: "Memory limit (for query) exceeded: would use 2.25 GiB",+ publishCallCount: 0,+ rows: [+ {+ id: monitorAId,+ status: MonitorStatusSchema.enum.ACTIVE,+ severity: MonitorSeveritySchema.enum.OK,+ severityChangedAt: tenMinutesAgo,+ alertedAt: null,+ lastClaimedAt: justAfterRunAt,+ lastCompletedAt: null,+ },+ ],+ },+ },{name: "error on getTriggers: claim, no complete, no sev change, no emit",monitors: [@@ -1352,6 +1383,11 @@ describe("MonitorProcessor.process (integration)", () => {new Error(c.injectError.message),);}+ if (c.injectError.errorClass === "resource-memory") {+ throw ClickHouseResourceError.wrapIfResourceError(+ new Error(c.injectError.message),+ );+ }throw new Error(c.injectError.message);}return c.ch ?? [{ count_count: 0 }];