🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(web): drop expected remote-experiment URL validation from Sentry (#16885)
cursor[bot]5f9f64550a98
diff --git a/web/src/utils/api.clienttest.ts b/web/src/utils/api.clienttest.tsindex 17e3a6d3a7..c251dc3068 100644--- a/web/src/utils/api.clienttest.ts+++ b/web/src/utils/api.clienttest.ts@@ -3,6 +3,7 @@import { TRPCClientError } from "@trpc/client";import { vi } from "vitest";import {+ EXPECTED_TRPC_BAD_REQUEST_PATHS,EXPECTED_TRPC_CONFLICT_PATHS,EXPECTED_TRPC_ERROR_CODES,captureBuildId,@@ -491,6 +492,50 @@ describe("isExpectedTrpcClientError", () => {).toBe(false);});+ it("treats a rejected remote-experiment URL as expected", () => {+ // triggerRemoteExperiment / upsertRemoteExperiment throw BAD_REQUEST+ // only when validateWebhookURL rejects a user-configured URL (DNS+ // lookup failed, private IP, …). The UI already toasts the message.+ for (const path of EXPECTED_TRPC_BAD_REQUEST_PATHS) {+ expect(+ isExpectedTrpcClientError(+ trpcServerError({+ code: "BAD_REQUEST",+ httpStatus: 400,+ path,+ message:+ "Invalid remote run URL: DNS lookup failed for example.invalid",+ }),+ ),+ ).toBe(true);+ }+ });++ it("does not treat BAD_REQUEST on other procedures as expected", () => {+ // Negative fixture: a missing-projectId / invariant BAD_REQUEST is a+ // client bug and must still reach Sentry. Widening the allowlist+ // would hide those.+ expect(+ isExpectedTrpcClientError(+ trpcServerError({+ code: "BAD_REQUEST",+ httpStatus: 400,+ path: "traces.byId",+ message: "Invalid input, projectId is required",+ }),+ ),+ ).toBe(false);+ expect(+ isExpectedTrpcClientError(+ trpcServerError({+ code: "INTERNAL_SERVER_ERROR",+ httpStatus: 500,+ path: EXPECTED_TRPC_BAD_REQUEST_PATHS[0],+ }),+ ),+ ).toBe(false);+ });+it("suppresses Zod input validation (empty/too-short fields) as expected user input", () => {expect(isExpectedTrpcClientError(@@ -664,6 +709,26 @@ describe("reportTrpcErrorWithoutToast", () => {warnSpy.mockRestore();});+ it("suppresses a rejected remote-experiment URL (breadcrumb, no capture)", () => {+ reportTrpcErrorWithoutToast(+ trpcServerError({+ code: "BAD_REQUEST",+ httpStatus: 400,+ path: EXPECTED_TRPC_BAD_REQUEST_PATHS[0],+ message:+ "Invalid remote run URL: DNS lookup failed for example.invalid",+ }),+ "experiments",+ );++ expect(captureExceptionMock).not.toHaveBeenCalled();+ expect(addBreadcrumbMock).toHaveBeenCalledTimes(1);+ expect(addBreadcrumbMock.mock.calls[0]![0].data).toMatchObject({+ code: "BAD_REQUEST",+ path: "datasets.triggerRemoteExperiment",+ });+ });+it("suppresses a stale in-app-agent tool approval (breadcrumb, no capture)", () => {reportTrpcErrorWithoutToast(trpcServerError({@@ -764,6 +829,27 @@ describe("reportTrpcErrorWithoutToast", () => {});});+ it("still captures a 5xx on an allowlisted remote-experiment path", () => {+ // Negative fixture: the BAD_REQUEST allowlist must not swallow a real+ // server failure on the same procedure.+ reportTrpcErrorWithoutToast(+ trpcServerError({+ code: "INTERNAL_SERVER_ERROR",+ httpStatus: 500,+ path: EXPECTED_TRPC_BAD_REQUEST_PATHS[0],+ }),+ "experiments",+ );++ expect(captureExceptionMock).toHaveBeenCalledTimes(1);+ const [, options] = captureExceptionMock.mock.calls[0]!;+ expect(options.tags).toMatchObject({+ area: "trpc",+ "trpc.code": "INTERNAL_SERVER_ERROR",+ "trpc.path": "datasets.triggerRemoteExperiment",+ });+ });+it("captures non-tRPC errors with the caller's area (not the seam's `trpc`)", () => {reportTrpcErrorWithoutToast(new Error("post-success work failed"), "evals");diff --git a/web/src/utils/api.ts b/web/src/utils/api.tsindex 748aef7447..930148de73 100644--- a/web/src/utils/api.ts+++ b/web/src/utils/api.ts@@ -128,12 +128,14 @@ export const isNetworkConnectivityError = (error: unknown): boolean => {* Deliberately narrow: only these codes on an actual `TRPCClientError`, plus* Zod input validation (`BAD_REQUEST` whose message is a Zod 4 issue list or* whose `data.zodError` is populated), plus CONFLICT on- * {@link EXPECTED_TRPC_CONFLICT_PATHS}. Empty/too-short fields and stale- * in-app-agent approvals are the product working as designed — the toast is- * the UX; Sentry must not log them.- * A 5xx (`INTERNAL_SERVER_ERROR`), a non-Zod `BAD_REQUEST`, a CONFLICT- * outside the allowlist, an unrecognized code, or any non-tRPC error is not- * expected and keeps flowing to Sentry.+ * {@link EXPECTED_TRPC_CONFLICT_PATHS}, plus BAD_REQUEST on+ * {@link EXPECTED_TRPC_BAD_REQUEST_PATHS}. Empty/too-short fields, stale+ * in-app-agent approvals, and a rejected user-configured remote-experiment+ * URL are the product working as designed — the toast is the UX; Sentry+ * must not log them.+ * A 5xx (`INTERNAL_SERVER_ERROR`), a non-Zod `BAD_REQUEST` outside the+ * allowlist, a CONFLICT outside the allowlist, an unrecognized code, or+ * any non-tRPC error is not expected and keeps flowing to Sentry.*/export const EXPECTED_TRPC_ERROR_CODES = ["NOT_FOUND",@@ -156,6 +158,22 @@ export const EXPECTED_TRPC_CONFLICT_PATHS = ["inAppAgent.decideToolApproval",] as const;+/**+ * BAD_REQUEST is usually a client bug (a missing hydrated route param,+ * an invariant the UI should not have sent). These procedures throw 400+ * only as user-configured-URL / secret-header validation the product+ * already toasts — expected user-facing state, not a regression.+ *+ * `datasets.triggerRemoteExperiment` and `datasets.upsertRemoteExperiment`+ * wrap `validateWebhookURL` (and related header checks) as BAD_REQUEST.+ * A DNS miss, private IP, or missing remote URL is the user fixing their+ * webhook, not an app failure.+ */+export const EXPECTED_TRPC_BAD_REQUEST_PATHS = [+ "datasets.triggerRemoteExperiment",+ "datasets.upsertRemoteExperiment",+] as const;+const getTrpcErrorData = (error: unknown,): { code?: unknown; path?: unknown } | undefined =>@@ -196,7 +214,8 @@ export const getTrpcErrorFingerprint = (error: unknown): string[] => [/*** True when `error` is a TRPCClientError whose code is an EXPECTED, user-facing* state that should not be captured to Sentry.- * See {@link EXPECTED_TRPC_ERROR_CODES} and {@link EXPECTED_TRPC_CONFLICT_PATHS}.+ * See {@link EXPECTED_TRPC_ERROR_CODES}, {@link EXPECTED_TRPC_CONFLICT_PATHS},+ * and {@link EXPECTED_TRPC_BAD_REQUEST_PATHS}.*/export const isExpectedTrpcClientError = (error: unknown): boolean => {const code = getTrpcErrorCode(error);@@ -214,6 +233,13 @@ export const isExpectedTrpcClientError = (error: unknown): boolean => {) {return true;}+ if (+ code === "BAD_REQUEST" &&+ path !== undefined &&+ (EXPECTED_TRPC_BAD_REQUEST_PATHS as readonly string[]).includes(path)+ ) {+ return true;+ }return isTrpcZodValidationError(error);};