AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(frontend): make Sentry reports readable and stop instrumentation noise (#15391)
Nicholas Tindlee0b9f614b498
diff --git a/autogpt_platform/frontend/.env.default b/autogpt_platform/frontend/.env.defaultindex 77c0e0f92b..c181433307 100644--- a/autogpt_platform/frontend/.env.default+++ b/autogpt_platform/frontend/.env.default@@ -54,6 +54,10 @@ NEXT_PUBLIC_FRONTEND_BASE_URL=http://localhost:3000NEXT_PUBLIC_APP_ENV=localNEXT_PUBLIC_BEHAVE_AS=LOCAL+# Error reporting is off unless you set your own Sentry DSN here. A self-hosted+# build never reports to AutoGPT's Sentry. Inlined at build time.+# NEXT_PUBLIC_SENTRY_DSN=+# Feature flagsNEXT_PUBLIC_LAUNCHDARKLY_ENABLED=falseNEXT_PUBLIC_LAUNCHDARKLY_CLIENT_ID=687ab1372f497809b131e06ediff --git a/autogpt_platform/frontend/instrumentation-client.ts b/autogpt_platform/frontend/instrumentation-client.tsindex 80da7f5d92..9894b29449 100644--- a/autogpt_platform/frontend/instrumentation-client.ts+++ b/autogpt_platform/frontend/instrumentation-client.ts@@ -2,34 +2,39 @@// The config you add here will be used whenever a users loads a page in their browser.// https://docs.sentry.io/platforms/javascript/guides/nextjs/-import { isNextRSCNavigationFallback } from "@/lib/sentry-filters";+import {+ CLIENT_IGNORE_ERRORS,+ isFromBot,+ isNextRSCNavigationFallback,+} from "@/lib/sentry-filters";import { setupSessionReplay } from "@/lib/session-replay";import { environment } from "@/services/environment";import * as Sentry from "@sentry/nextjs";-const shouldEnable = environment.isSentryEnabled();+const dsn = environment.getSentryDSN();Sentry.init({- dsn: "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288",+ dsn,environment: environment.getEnvironmentStr(),- enabled: shouldEnable,+ enabled: Boolean(dsn),// Suppress cross-origin stylesheet errors from Sentry Replay (rrweb)// serializing DOM snapshots with cross-origin stylesheets// (e.g., from browser extensions or CDN-loaded CSS)- ignoreErrors: [- /Not allowed to access cross-origin stylesheet/,- // Sentry SDK internal issue on some mobile browsers- /Error invoking postEvent: Method not found/,- ],+ ignoreErrors: CLIENT_IGNORE_ERRORS,// Next's handled RSC fetch fallback reaches us only through console capture// and falls back to a full navigation, so it is noise (BUILDER-3QB).+ // Crawlers render our pages too; what breaks for them is not a user's bug.beforeSend(event) {+ if (isFromBot(navigator.userAgent)) return null;return isNextRSCNavigationFallback(event) ? null : event;},+ beforeSendTransaction(event) {+ return isFromBot(navigator.userAgent) ? null : event;+ },// Add optional integrations for additional featuresintegrations: [diff --git a/autogpt_platform/frontend/next.config.mjs b/autogpt_platform/frontend/next.config.mjsindex ad1463686c..5997683976 100644--- a/autogpt_platform/frontend/next.config.mjs+++ b/autogpt_platform/frontend/next.config.mjs@@ -14,6 +14,8 @@ const nextConfig = {// tag preview deployments as previews rather than as production.env: {NEXT_PUBLIC_VERCEL_ENV: process.env.VERCEL_ENV ?? "",+ // Sentry initialises in the browser too, so its kill switch must reach it.+ DISABLE_SENTRY: process.env.DISABLE_SENTRY ?? "",},// Suppress the "X-Powered-By: Next.js" header (framework fingerprinting).poweredByHeader: false,diff --git a/autogpt_platform/frontend/sentry.edge.config.ts b/autogpt_platform/frontend/sentry.edge.config.tsindex b2f762e745..b299544f93 100644--- a/autogpt_platform/frontend/sentry.edge.config.ts+++ b/autogpt_platform/frontend/sentry.edge.config.ts@@ -6,14 +6,14 @@import { environment } from "@/services/environment";import * as Sentry from "@sentry/nextjs";-const shouldEnable = environment.isSentryEnabled();+const dsn = environment.getSentryDSN();Sentry.init({- dsn: "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288",+ dsn,environment: environment.getEnvironmentStr(),- enabled: shouldEnable,+ enabled: Boolean(dsn),// Define how likely traces are sampled. Adjust this value in production, or use tracesSampler for greater control.tracesSampleRate: 1,diff --git a/autogpt_platform/frontend/sentry.server.config.ts b/autogpt_platform/frontend/sentry.server.config.tsindex 269e2c9098..2e2218b71e 100644--- a/autogpt_platform/frontend/sentry.server.config.ts+++ b/autogpt_platform/frontend/sentry.server.config.ts@@ -6,14 +6,14 @@ import { environment } from "@/services/environment";import * as Sentry from "@sentry/nextjs";// import { NodeProfilingIntegration } from "@sentry/profiling-node";-const shouldEnable = environment.isSentryEnabled();+const dsn = environment.getSentryDSN();Sentry.init({- dsn: "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288",+ dsn,environment: environment.getEnvironmentStr(),- enabled: shouldEnable,+ enabled: Boolean(dsn),// Define how likely traces are sampled. Adjust this value in production, or use tracesSampler for greater control.tracesSampleRate: 1,@@ -29,7 +29,10 @@ Sentry.init({// Integrationsintegrations: [- Sentry.anrIntegration(),+ // Vercel freezes an idle function between requests, and the ANR watchdog+ // reports every thaw as a blocked event loop with no stack (BUILDER-9).+ // Only a long-lived self-hosted Node server can really hang.+ ...(environment.isVercel() ? [] : [Sentry.anrIntegration()]),// NodeProfilingIntegration,Sentry.extraErrorDataIntegration(),// Sentry.fsIntegration(),diff --git a/autogpt_platform/frontend/src/app/api/mutators/__tests__/custom-mutator.test.ts b/autogpt_platform/frontend/src/app/api/mutators/__tests__/custom-mutator.test.tsindex cb1578476c..bee3de4250 100644--- a/autogpt_platform/frontend/src/app/api/mutators/__tests__/custom-mutator.test.ts+++ b/autogpt_platform/frontend/src/app/api/mutators/__tests__/custom-mutator.test.ts@@ -28,6 +28,8 @@ vi.mock("@/lib/autogpt-server-api/helpers", () => ({vi.mock("@sentry/nextjs", () => ({getTraceData: vi.fn(() => ({})),+ captureMessage: vi.fn(),+ addBreadcrumb: vi.fn(),}));import { customMutator } from "../custom-mutator";diff --git a/autogpt_platform/frontend/src/app/api/mutators/__tests__/request-failure-log.test.ts b/autogpt_platform/frontend/src/app/api/mutators/__tests__/request-failure-log.test.tsindex 693f26d56b..54eb2dea5c 100644--- a/autogpt_platform/frontend/src/app/api/mutators/__tests__/request-failure-log.test.ts+++ b/autogpt_platform/frontend/src/app/api/mutators/__tests__/request-failure-log.test.ts@@ -1,106 +1,210 @@-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import * as Sentry from "@sentry/nextjs";+import {+ afterAll,+ afterEach,+ beforeAll,+ beforeEach,+ describe,+ expect,+ it,+ vi,+} from "vitest";import {logClientRequestFailure,resetClientRequestFailureLog,+ routeTemplate,} from "../request-failure-log";-const base = {- method: "GET",- url: "/api/credits/subscription",- errorMessage: "Backend sent no response within 30000ms",- responseData: null,-};--describe("logClientRequestFailure", () => {- beforeEach(() => {- vi.useFakeTimers();- vi.spyOn(console, "error").mockImplementation(() => {});- vi.spyOn(console, "warn").mockImplementation(() => {});- });-- afterEach(() => {- resetClientRequestFailureLog();- vi.useRealTimers();- vi.restoreAllMocks();+// A real Sentry client with the browser's console capture, so the events and+// their grouping are built exactly as they are in production.+const sent: Sentry.ErrorEvent[] = [];+const breadcrumbs: Sentry.Breadcrumb[] = [];+// Spied before init, so Sentry's console patch wraps the spy.+const consoleInfo = vi+ .spyOn(console, "info")+ .mockImplementation(() => undefined);++beforeAll(() => {+ vi.spyOn(console, "warn").mockImplementation(() => undefined);+ vi.spyOn(console, "error").mockImplementation(() => undefined);+ Sentry.init({+ dsn: "https://[email protected]/0",+ transport: () => ({+ send: () => Promise.resolve({}),+ flush: () => Promise.resolve(true),+ }),+ integrations: [+ Sentry.captureConsoleIntegration({ levels: ["fatal", "error", "warn"] }),+ ],+ beforeSend(event) {+ sent.push(event);+ return null;+ },+ beforeBreadcrumb(breadcrumb) {+ if (breadcrumb.category === "http.client.failure") {+ breadcrumbs.push(breadcrumb);+ }+ return breadcrumb;+ },});+});- it("logs the first failure and suppresses repeats of the same method, url and status", () => {- for (let i = 0; i < 12; i++) {- logClientRequestFailure({ ...base, status: 504 });- }+beforeEach(() => {+ sent.length = 0;+ breadcrumbs.length = 0;+ consoleInfo.mockClear();+});- expect(console.error).toHaveBeenCalledTimes(1);- expect(console.error).toHaveBeenCalledWith(- "Request failed on client",- expect.objectContaining({ status: 504, url: base.url }),- );- });+afterEach(() => {+ resetClientRequestFailureLog();+ vi.useRealTimers();+});- it("reports how many were suppressed once the window closes", () => {- logClientRequestFailure({ ...base, status: 504 });- logClientRequestFailure({ ...base, status: 504 });- logClientRequestFailure({ ...base, status: 504 });+afterAll(() => {+ vi.restoreAllMocks();+});- expect(console.error).toHaveBeenCalledTimes(1);- vi.advanceTimersByTime(10_000);+const SESSION_A = "9c1f6a52-3c0e-4b7e-9d0a-6f1e2b3c4d5e";+const SESSION_B = "0d4b2e61-8a7f-4c3d-b2e1-7a6f5e4d3c2b";- expect(console.error).toHaveBeenCalledTimes(2);- expect(console.error).toHaveBeenLastCalledWith(- "Request failed on client ×3 in the last 10s",- expect.objectContaining({ status: 504, url: base.url }),- );- expect(console.warn).not.toHaveBeenCalled();+function fail(status: number, url: string, method = "GET") {+ logClientRequestFailure({+ status,+ method,+ url,+ errorMessage: "Backend sent no response within 30000ms",});-- it("reports suppressed expected failures at warn", () => {- logClientRequestFailure({ ...base, status: 404 });- logClientRequestFailure({ ...base, status: 404 });- vi.advanceTimersByTime(10_000);-- expect(console.warn).toHaveBeenLastCalledWith(- "Request failed on client ×2 in the last 10s",- expect.objectContaining({ status: 404, url: base.url }),+}++// Sentry's flush waits on real timers, so close the dedupe window on fake ones+// and hand back the real clock before flushing.+async function closeWindowAndFlush() {+ vi.advanceTimersByTime(10_000);+ vi.useRealTimers();+ await Sentry.flush(2000);+}++describe("client request failures in Sentry", () => {+ it("titles a server failure with its method, route and status", async () => {+ fail(504, `/api/chat/sessions/${SESSION_A}/stream?resume=1`);+ await Sentry.flush(2000);++ expect(sent).toHaveLength(1);+ expect(sent[0].message).toBe(+ "Request failed on client: GET /api/chat/sessions/{id}/stream 504",);- expect(console.error).not.toHaveBeenCalled();+ expect(sent[0].level).toBe("error");+ expect(JSON.stringify(sent[0])).not.toContain("[object Object]");+ expect(sent[0].contexts?.request_failure).toMatchObject({+ route: "/api/chat/sessions/{id}/stream",+ path: `/api/chat/sessions/${SESSION_A}/stream`,+ status: 504,+ });});- it("says nothing extra when the failure happened once", () => {- logClientRequestFailure({ ...base, status: 504 });- vi.advanceTimersByTime(10_000);-- expect(console.warn).not.toHaveBeenCalled();+ it("groups one route's server failures together, and apart from other routes", async () => {+ fail(504, `/api/chat/sessions/${SESSION_A}`);+ fail(502, `/api/chat/sessions/${SESSION_B}`);+ fail(504, "/api/credits/trial/cancel", "POST");+ await Sentry.flush(2000);++ const [first, second, other] = sent.map((event) => event.fingerprint);+ expect(first).toEqual([+ "client-request-failure",+ "GET /api/chat/sessions/{id}",+ "5xx",+ ]);+ expect(second).toEqual(first);+ expect(other).toEqual([+ "client-request-failure",+ "POST /api/credits/trial/cancel",+ "5xx",+ ]);});- it("logs again once the window has passed", () => {- logClientRequestFailure({ ...base, status: 504 });- vi.advanceTimersByTime(10_001);- logClientRequestFailure({ ...base, status: 504 });+ it("reports a request the app got wrong as a warning in its own 4xx issue", async () => {+ fail(400, `/api/graphs/${SESSION_A}/execute/7`, "POST");+ fail(422, `/api/graphs/${SESSION_B}/execute/9`, "POST");+ await Sentry.flush(2000);++ expect(sent.map((event) => event.level)).toEqual(["warning", "warning"]);+ expect(sent[0].fingerprint).toEqual([+ "client-request-failure",+ "POST /api/graphs/{id}/execute/{id}",+ "4xx",+ ]);+ expect(sent[1].fingerprint).toEqual(sent[0].fingerprint);+ });- expect(console.error).toHaveBeenCalledTimes(2);+ it.each([+ [401, "signed out"],+ [402, "paywall"],+ [403, "permission the user lacks"],+ [404, "deleted resource"],+ [499, "user abandoned the request"],+ ])("leaves only a breadcrumb for an expected %i (%s)", async (status) => {+ fail(status, "/api/experts/identities");+ await Sentry.flush(2000);++ expect(sent).toHaveLength(0);+ expect(breadcrumbs).toEqual([+ expect.objectContaining({+ level: "info",+ message: `Request failed on client: GET /api/experts/identities ${status}`,+ }),+ ]);});- it("keeps a different status, url or method visible", () => {- logClientRequestFailure({ ...base, status: 504 });- logClientRequestFailure({ ...base, status: 500 });- logClientRequestFailure({ ...base, status: 504, url: "/api/other" });- logClientRequestFailure({ ...base, status: 504, method: "POST" });+ it("sends one event for a burst of the same failure and counts the rest in the console", async () => {+ vi.useFakeTimers();+ for (let i = 0; i < 12; i++) fail(504, "/api/chat/connections");+ await closeWindowAndFlush();- expect(console.error).toHaveBeenCalledTimes(4);+ expect(sent).toHaveLength(1);+ expect(consoleInfo).toHaveBeenLastCalledWith(+ "Request failed on client: GET /api/chat/connections 504 ×12 in the last 10s",+ );});- it("warns rather than errors on the statuses the app asks for and handles", () => {- logClientRequestFailure({ ...base, status: 401 });- logClientRequestFailure({ ...base, status: 403 });- logClientRequestFailure({ ...base, status: 404 });+ it("reports the same failure again once the window has passed", async () => {+ vi.useFakeTimers();+ fail(504, "/api/chat/connections");+ await closeWindowAndFlush();+ fail(504, "/api/chat/connections");+ await Sentry.flush(2000);- expect(console.error).not.toHaveBeenCalled();- expect(console.warn).toHaveBeenCalledTimes(3);+ expect(sent).toHaveLength(2);});- it("keeps a malformed request an error", () => {- logClientRequestFailure({ ...base, status: 422 });+ it("does not ship a proxy's HTML error page", async () => {+ logClientRequestFailure({+ status: 502,+ method: "POST",+ url: "/api/credits/trial/cancel",+ errorMessage: `<!DOCTYPE html>${"<div>cloudflare</div>".repeat(200)}`,+ });+ await Sentry.flush(2000);++ const detail = sent[0].contexts?.request_failure?.detail;+ expect(typeof detail === "string" && detail.length).toBe(300);+ });+});- expect(console.error).toHaveBeenCalledTimes(1);+describe("routeTemplate", () => {+ it.each([+ ["/api/library/agents", "/api/library/agents"],+ [`/api/library/agents/${SESSION_A}`, "/api/library/agents/{id}"],+ ["/api/executions/12345?page=2", "/api/executions/{id}"],+ ["/api/files/3f9a8b7c6d5e4f30", "/api/files/{id}"],+ ["/api/store/agents/V1StGXR8_Z5jdHi6B-myT", "/api/store/agents/{id}"],+ [+ "/api/store/agents/alice/email-digest",+ "/api/store/agents/alice/email-digest",+ ],+ ["/api/users/someone%40example.com", "/api/users/{id}"],+ ["/api/integrations/codex/login", "/api/integrations/codex/login"],+ ])("%s → %s", (url, expected) => {+ expect(routeTemplate(url)).toBe(expected);});});diff --git a/autogpt_platform/frontend/src/app/api/mutators/request-failure-log.ts b/autogpt_platform/frontend/src/app/api/mutators/request-failure-log.tsindex 2c58bed92a..e787f57586 100644--- a/autogpt_platform/frontend/src/app/api/mutators/request-failure-log.ts+++ b/autogpt_platform/frontend/src/app/api/mutators/request-failure-log.ts@@ -1,9 +1,10 @@+import * as Sentry from "@sentry/nextjs";+interface RequestFailure {status: number;method: string;url: string;errorMessage: string;- responseData: unknown;}// A failing endpoint is usually polled or retried, so the same failure lands in@@ -30,9 +31,8 @@ export function logClientRequestFailure(failure: RequestFailure) {const closed = openWindows.get(key);openWindows.delete(key);if (closed?.suppressed) {- logAtSeverityOf(failure.status)(- `Request failed on client ×${closed.suppressed + 1} in the last ${DEDUPE_WINDOW_MS / 1000}s`,- { status: failure.status, method: failure.method, url: failure.url },+ console.info(+ `${summarize(failure)} ×${closed.suppressed + 1} in the last ${DEDUPE_WINDOW_MS / 1000}s`,);}}, DEDUPE_WINDOW_MS),@@ -52,28 +52,84 @@ interface OpenWindow {const openWindows = new Map<string, OpenWindow>();-// Statuses the app asks for and handles: an unauthenticated read of a gated-// resource, a permission the user does not have, an optional resource that is-// absent. A 4xx that means the request itself was wrong (400, 422) and every-// 5xx stay errors.-const EXPECTED_STATUSES = new Set([401, 403, 404]);--function emit({- status,- method,- url,- errorMessage,- responseData,-}: RequestFailure) {- logAtSeverityOf(status)("Request failed on client", {- status,- method,- url,- errorMessage,- responseData: responseData || "No response data",+// Outcomes the app asks for and handles: a signed-out read of a gated resource,+// a permission the user lacks (re-auth, email verification, waitlist), the+// paywall, an absent or deleted resource, and a request the user abandoned+// (the proxy answers 499 when the browser hangs up). They leave a breadcrumb+// for whatever error follows, but are not issues of their own.+const EXPECTED_STATUSES = new Set([401, 402, 403, 404, 499]);++// Long enough for a validation detail, short enough that an HTML error page+// from a proxy does not ride along.+const MAX_DETAIL_LENGTH = 300;++function emit(failure: RequestFailure) {+ const route = routeTemplate(failure.url);+ const message = summarize(failure);++ if (EXPECTED_STATUSES.has(failure.status)) {+ console.info(message);+ Sentry.addBreadcrumb({+ category: "http.client.failure",+ level: "info",+ message,+ data: { status: failure.status, method: failure.method, route },+ });+ return;+ }++ console.info(message, failure.errorMessage);+ Sentry.captureMessage(message, {+ level: failure.status >= 500 ? "error" : "warning",+ fingerprint: [+ "client-request-failure",+ `${failure.method} ${route}`,+ statusClass(failure.status),+ ],+ tags: {+ "http.method": failure.method,+ "http.route": route,+ "http.status_code": String(failure.status),+ },+ contexts: {+ request_failure: {+ method: failure.method,+ route,+ path: failure.url.split("?")[0],+ status: failure.status,+ detail: failure.errorMessage.slice(0, MAX_DETAIL_LENGTH),+ },+ },});}-function logAtSeverityOf(status: number) {- return EXPECTED_STATUSES.has(status) ? console.warn : console.error;+function summarize({ method, url, status }: RequestFailure) {+ return `Request failed on client: ${method} ${routeTemplate(url)} ${status}`;+}++function statusClass(status: number) {+ return status >= 100 && status < 600+ ? `${Math.floor(status / 100)}xx`+ : "0xx";+}++const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+const HEX_ID = /^[0-9a-f]{8,}$/i;++// An opaque ID (UUID, number, hash, nanoid) becomes {id} so every agent, run+// and session shares one route. A word or a short slug stays as it is.+function isOpaqueID(segment: string) {+ if (UUID.test(segment) || /^\d+$/.test(segment) || HEX_ID.test(segment)) {+ return true;+ }+ if (segment.includes("@") || segment.includes("%")) return true;+ return segment.length >= 12 && /\d/.test(segment);+}++export function routeTemplate(url: string) {+ const path = url.split(/[?#]/)[0] || "/";+ return path+ .split("/")+ .map((segment) => (segment && isOpaqueID(segment) ? "{id}" : segment))+ .join("/");}diff --git a/autogpt_platform/frontend/src/lib/__tests__/sentry-filters.test.ts b/autogpt_platform/frontend/src/lib/__tests__/sentry-filters.test.tsindex f0a78a4c26..02b6162cdc 100644--- a/autogpt_platform/frontend/src/lib/__tests__/sentry-filters.test.ts+++ b/autogpt_platform/frontend/src/lib/__tests__/sentry-filters.test.ts@@ -15,7 +15,11 @@ import {resetClientRequestFailureLog,} from "@/app/api/mutators/request-failure-log";-import { isNextRSCNavigationFallback } from "../sentry-filters";+import {+ CLIENT_IGNORE_ERRORS,+ isFromBot,+ isNextRSCNavigationFallback,+} from "../sentry-filters";// Drives a real Sentry client with the same console capture as// instrumentation-client.ts, so the events are built exactly as in production.@@ -27,6 +31,7 @@ beforeAll(() => {vi.spyOn(console, "warn").mockImplementation(() => undefined);Sentry.init({dsn: "https://[email protected]/0",+ ignoreErrors: CLIENT_IGNORE_ERRORS,// No network: a failed send is itself logged, and would be captured too.transport: () => ({send: () => Promise.resolve({}),@@ -94,12 +99,13 @@ describe("Sentry filter for Next's RSC navigation fallback", () => {method: "GET",url: "/api/library/agents",errorMessage: "Internal Server Error",- responseData: null,});await Sentry.flush(2000);expect(sent).toHaveLength(1);- expect(sent[0].message).toContain("Request failed on client");+ expect(sent[0].message).toBe(+ "Request failed on client: GET /api/library/agents 500",+ );});it("still reports a message that only borrows the RSC wording", async () => {@@ -112,3 +118,42 @@ describe("Sentry filter for Next's RSC navigation fallback", () => {expect(sent).toHaveLength(1);});});++describe("Sentry filter for crawler noise", () => {+ it("drops Cookiebot's 'included twice' warning", async () => {+ console.warn(+ "WARNING: Cookiebot script is included twice - please remove one instance to avoid unexpected results.",+ );+ await Sentry.flush(2000);++ expect(sent).toHaveLength(0);+ });++ it("still reports our own warnings", async () => {+ console.warn("Cookie consent banner failed to load");+ await Sentry.flush(2000);++ expect(sent).toHaveLength(1);+ });++ it.each([+ "",+ "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",+ "Mozilla/5.0 (compatible; GoogleOther)",+ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)",+ "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/131.0.0.0 Safari/537.36",+ "Mozilla/5.0 (Linux; Android 11; moto g power (2022)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36 Chrome-Lighthouse",+ ])("treats %j as a bot", (userAgent) => {+ expect(isFromBot(userAgent)).toBe(true);+ });++ it.each([+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",+ "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Mobile/15E148 Safari/604.1",+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0",+ "Mozilla/5.0 (Linux; Android 10; Cubot P40) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Mobile Safari/537.36",+ ])("treats %j as a person", (userAgent) => {+ expect(isFromBot(userAgent)).toBe(false);+ });+});diff --git a/autogpt_platform/frontend/src/lib/sentry-filters.ts b/autogpt_platform/frontend/src/lib/sentry-filters.tsindex b20f082921..01453e2bb9 100644--- a/autogpt_platform/frontend/src/lib/sentry-filters.ts+++ b/autogpt_platform/frontend/src/lib/sentry-filters.ts@@ -20,3 +20,25 @@ export function isNextRSCNavigationFallback(event: Event) {const [message] = args;return typeof message === "string" && NEXT_RSC_FALLBACK_MESSAGE.test(message);}++export const CLIENT_IGNORE_ERRORS = [+ // Sentry Replay (rrweb) serializing DOM snapshots with cross-origin+ // stylesheets (e.g. from browser extensions or CDN-loaded CSS)+ /Not allowed to access cross-origin stylesheet/,+ // Sentry SDK internal issue on some mobile browsers+ /Error invoking postEvent: Method not found/,+ // Cookiebot's own console warning when a crawler re-runs the page's scripts;+ // our layout includes it once (BUILDER-90X).+ /Cookiebot script is included twice/,+];++// Crawler user agents name themselves ("Googlebot/2.1", "GPTBot/1.0") and link+// to their docs ("+http://…"), which no real browser does. Headless and audit+// browsers identify themselves too. An empty user agent is never a person.+const BOT_USER_AGENT =+ /bot\/|bot;|\+https?:\/\/|crawler|spider|slurp|GoogleOther|Google-InspectionTool|HeadlessChrome|Lighthouse|PhantomJS|Prerender|python-requests|curl\/|wget\//i;++export function isFromBot(userAgent: string | undefined) {+ if (!userAgent?.trim()) return true;+ return BOT_USER_AGENT.test(userAgent);+}diff --git a/autogpt_platform/frontend/src/services/environment/__tests__/index.test.ts b/autogpt_platform/frontend/src/services/environment/__tests__/index.test.tsindex b1114f5db6..8c755d6de3 100644--- a/autogpt_platform/frontend/src/services/environment/__tests__/index.test.ts+++ b/autogpt_platform/frontend/src/services/environment/__tests__/index.test.ts@@ -113,13 +113,25 @@ describe("Sentry environment tagging", () => {});describe("Sentry enablement", () => {+ const HOSTED_DSN_PROJECT = "/4507946746380288";+it("stays off under the test runner, where no cloud deployment env is set", () => {expect(environment.isSentryEnabled()).toBe(false);});- it("is on for a cloud deployment", () => {+ it("reports to AutoGPT's Sentry from a hosted deployment on Vercel", () => {vi.stubEnv("NEXT_PUBLIC_APP_ENV", "prod");vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "CLOUD");+ vi.stubEnv("NEXT_PUBLIC_VERCEL_ENV", "production");++ expect(environment.isSentryEnabled()).toBe(true);+ expect(environment.getSentryDSN()).toContain(HOSTED_DSN_PROJECT);+ });++ it("reports from a hosted preview deployment", () => {+ vi.stubEnv("NEXT_PUBLIC_APP_ENV", "dev");+ vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "CLOUD");+ vi.stubEnv("NEXT_PUBLIC_VERCEL_ENV", "preview");expect(environment.isSentryEnabled()).toBe(true);});@@ -131,9 +143,25 @@ describe("Sentry enablement", () => {expect(environment.isSentryEnabled()).toBe(false);});+ it("does not report to AutoGPT's Sentry from a self-hosted build configured like the cloud", () => {+ vi.stubEnv("NEXT_PUBLIC_APP_ENV", "prod");+ vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "CLOUD");+ vi.stubEnv("NEXT_PUBLIC_VERCEL_ENV", "");+ vi.stubEnv("VERCEL", "");++ expect(environment.isSentryEnabled()).toBe(false);+ });++ it("reports to a self-hoster's own Sentry once they opt in with a DSN", () => {+ vi.stubEnv("NEXT_PUBLIC_SENTRY_DSN", "https://[email protected]/7");++ expect(environment.getSentryDSN()).toBe("https://[email protected]/7");+ });+it("honours the DISABLE_SENTRY kill switch on a cloud deployment", () => {vi.stubEnv("NEXT_PUBLIC_APP_ENV", "prod");vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "CLOUD");+ vi.stubEnv("NEXT_PUBLIC_VERCEL_ENV", "production");vi.stubEnv("DISABLE_SENTRY", "true");expect(environment.isSentryEnabled()).toBe(false);diff --git a/autogpt_platform/frontend/src/services/environment/index.ts b/autogpt_platform/frontend/src/services/environment/index.tsindex 4e44e8bb1e..308bb2a334 100644--- a/autogpt_platform/frontend/src/services/environment/index.ts+++ b/autogpt_platform/frontend/src/services/environment/index.ts@@ -148,8 +148,25 @@ function isVercelPreview() {);}+function isVercel() {+ return Boolean(process.env.NEXT_PUBLIC_VERCEL_ENV || process.env.VERCEL);+}++// AutoGPT's own Sentry project. Only our hosted deployments (on Vercel) report+// to it; a self-hosted build reports nowhere unless its operator sets their own+// NEXT_PUBLIC_SENTRY_DSN, so no self-hoster sends us errors without opting in.+const HOSTED_SENTRY_DSN =+ "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288";++function getSentryDSN() {+ if (process.env.DISABLE_SENTRY === "true") return undefined;+ const ownDSN = process.env.NEXT_PUBLIC_SENTRY_DSN;+ if (ownDSN) return ownDSN;+ return isVercel() && (isProd() || isDev()) ? HOSTED_SENTRY_DSN : undefined;+}+function isSentryEnabled() {- return process.env.DISABLE_SENTRY !== "true" && (isProd() || isDev());+ return Boolean(getSentryDSN());}function areFeatureFlagsEnabled() {@@ -182,6 +199,7 @@ export const environment = {getGoogleAdsConversionLabels,getCookiebotCBID,getCookiebotGeoRegions,+ getSentryDSN,// AssertionsisServerSide,isClientSide,@@ -191,6 +209,7 @@ export const environment = {isProd,isCloud,isLocal,+ isVercel,isVercelPreview,isSentryEnabled,isPostHogEnabled,diff --git a/autogpt_platform/frontend/src/services/feature-flags/__tests__/flag-backends.test.tsx b/autogpt_platform/frontend/src/services/feature-flags/__tests__/flag-backends.test.tsxindex a04a987367..111f99d992 100644--- a/autogpt_platform/frontend/src/services/feature-flags/__tests__/flag-backends.test.tsx+++ b/autogpt_platform/frontend/src/services/feature-flags/__tests__/flag-backends.test.tsx@@ -44,9 +44,13 @@ vi.mock("@posthog/react", () => ({usePostHog: () => postHogClient,}));-const sentry = vi.hoisted(() => ({ addFeatureFlag: vi.fn() }));+const sentry = vi.hoisted(() => ({+ addFeatureFlag: vi.fn(),+ captureMessage: vi.fn(),+}));vi.mock("@sentry/nextjs", () => ({+ captureMessage: sentry.captureMessage,getClient: () => ({getIntegrationByName: (name: string) =>name === "FeatureFlags"@@ -220,9 +224,9 @@ describe("dual backend", () => {expect(result.current).toBe(true);});- it("reports the disagreement to PostHog and the console", async () => {+ it("reports the disagreement to PostHog and Sentry", async () => {const { Flag, useGetFlag } = await loadWithBackend("dual");- const warn = vi.spyOn(console, "warn").mockImplementation(() => {});+ vi.spyOn(console, "info").mockImplementation(() => {});launchDarkly.flags = { [HIRE_EXPERTS]: true };postHog.enabled.mockReturnValue(false);@@ -233,7 +237,12 @@ describe("dual backend", () => {launchdarkly: { value: true, resolved: true },posthog: { value: false, resolved: true },});- expect(warn).toHaveBeenCalled();+ expect(sentry.captureMessage).toHaveBeenCalledExactlyOnceWith(+ `Feature flag mismatch: ${HIRE_EXPERTS} (LaunchDarkly=true, PostHog=false)`,+ expect.objectContaining({+ fingerprint: ["feature-flag-mismatch", HIRE_EXPERTS],+ }),+ );});it("stays quiet when the two agree", async () => {@@ -261,7 +270,7 @@ describe("dual backend", () => {it("reports a JSON flag whose values really differ", async () => {const { Flag, useGetFlag } = await loadWithBackend("dual");- vi.spyOn(console, "warn").mockImplementation(() => {});+ vi.spyOn(console, "info").mockImplementation(() => {});launchDarkly.flags = {[Flag.COPILOT_BOT_PLATFORMS]: { slack: true, discord: false },};@@ -275,7 +284,7 @@ describe("dual backend", () => {it("reports a flag LaunchDarkly has and PostHog has never heard of", async () => {const { Flag, useGetFlag } = await loadWithBackend("dual");- vi.spyOn(console, "warn").mockImplementation(() => {});+ vi.spyOn(console, "info").mockImplementation(() => {});launchDarkly.flags = { [HIRE_EXPERTS]: true };postHog.enabled.mockReturnValue(undefined);@@ -292,7 +301,7 @@ describe("dual backend", () => {// The two never resolve on the same render, so comparing before both// have answered reports the load order rather than a disagreement.const { Flag, useGetFlag } = await loadWithBackend("dual");- vi.spyOn(console, "warn").mockImplementation(() => {});+ vi.spyOn(console, "info").mockImplementation(() => {});launchDarkly.flags = {};postHog.enabled.mockReturnValue(true);@@ -310,7 +319,7 @@ describe("dual backend", () => {it("reports once LaunchDarkly catches up and still disagrees", async () => {const { Flag, useGetFlag } = await loadWithBackend("dual");- vi.spyOn(console, "warn").mockImplementation(() => {});+ vi.spyOn(console, "info").mockImplementation(() => {});postHog.enabled.mockReturnValue(true);const { rerender } = renderHook(() => useGetFlag(Flag.HIRE_EXPERTS));expect(postHog.capture).not.toHaveBeenCalled();@@ -471,6 +480,7 @@ beforeEach(() => {postHog.payload.mockReturnValue(undefined);postHog.capture.mockClear();sentry.addFeatureFlag.mockReset();+ sentry.captureMessage.mockClear();postHog.loaded = true;postHog.errorsLoading = false;Object.keys(process.env)diff --git a/autogpt_platform/frontend/src/services/feature-flags/backends/__tests__/dual.test.ts b/autogpt_platform/frontend/src/services/feature-flags/backends/__tests__/dual.test.tsnew file mode 100644index 0000000000..c06b139c79--- /dev/null+++ b/autogpt_platform/frontend/src/services/feature-flags/backends/__tests__/dual.test.ts@@ -0,0 +1,92 @@+import * as Sentry from "@sentry/nextjs";+import {+ afterAll,+ beforeAll,+ beforeEach,+ describe,+ expect,+ it,+ vi,+} from "vitest";++import { reportMismatchToSentry } from "../dual";++const sent: Sentry.ErrorEvent[] = [];++beforeAll(() => {+ vi.spyOn(console, "info").mockImplementation(() => undefined);+ Sentry.init({+ dsn: "https://[email protected]/0",+ transport: () => ({+ send: () => Promise.resolve({}),+ flush: () => Promise.resolve(true),+ }),+ beforeSend(event) {+ sent.push(event);+ return null;+ },+ });+});++beforeEach(() => {+ sent.length = 0;+});++afterAll(() => {+ vi.restoreAllMocks();+});++describe("feature-flag mismatch report", () => {+ it("names the flag and which way the vendors disagree", async () => {+ reportMismatchToSentry({+ flag: "enable-platform-payment",+ launchdarkly: { value: true },+ posthog: { value: false },+ });+ await Sentry.flush(2000);++ expect(sent).toHaveLength(1);+ expect(sent[0].message).toBe(+ "Feature flag mismatch: enable-platform-payment (LaunchDarkly=true, PostHog=false)",+ );+ expect(sent[0].level).toBe("warning");+ expect(sent[0].tags).toMatchObject({+ "feature_flag.key": "enable-platform-payment",+ "feature_flag.launchdarkly": "true",+ "feature_flag.posthog": "false",+ });+ });++ it("files each flag as its own issue, whichever way it disagrees", async () => {+ reportMismatchToSentry({+ flag: "enable-platform-payment",+ launchdarkly: { value: true },+ posthog: { value: false },+ });+ reportMismatchToSentry({+ flag: "enable-platform-payment",+ launchdarkly: { value: false },+ posthog: { value: true },+ });+ reportMismatchToSentry({+ flag: "copilot-models",+ launchdarkly: { value: { default: "a" } },+ posthog: { value: null },+ });+ await Sentry.flush(2000);++ const [first, flipped, other] = sent;+ expect(first.fingerprint).toEqual([+ "feature-flag-mismatch",+ "enable-platform-payment",+ ]);+ expect(flipped.fingerprint).toEqual(first.fingerprint);+ expect(other.fingerprint).toEqual([+ "feature-flag-mismatch",+ "copilot-models",+ ]);+ expect(other.message).toBe(+ 'Feature flag mismatch: copilot-models (LaunchDarkly={"default":"a"}, PostHog=null)',+ );+ });+});diff --git a/autogpt_platform/frontend/src/services/feature-flags/backends/dual.ts b/autogpt_platform/frontend/src/services/feature-flags/backends/dual.tsindex af9ab41c76..e51c757fb9 100644--- a/autogpt_platform/frontend/src/services/feature-flags/backends/dual.ts+++ b/autogpt_platform/frontend/src/services/feature-flags/backends/dual.ts@@ -3,6 +3,7 @@import { FeatureFlagEvent } from "@/services/analytics/posthog-events";import { environment } from "@/services/environment";import { usePostHog } from "@posthog/react";+import * as Sentry from "@sentry/nextjs";import isEqual from "lodash/isEqual";import { useEffect } from "react";import { isPostHogFlagsEnabled } from "../flag-backend";@@ -54,7 +55,49 @@ function useReportMismatch(useEffect(() => {if (!comparable || agree) return;const mismatch = JSON.parse(record);- console.warn("feature-flag mismatch", mismatch);+ reportMismatchToSentry(mismatch);posthog?.capture(FeatureFlagEvent.FEATURE_FLAG_MISMATCHED, mismatch);}, [comparable, agree, record, posthog]);}++interface Mismatch {+ flag: string;+ launchdarkly: { value: unknown };+ posthog: { value: unknown };+}++// Sentry tag values are capped at 200 characters.+const MAX_TAG_LENGTH = 200;++// One issue per flag, so the patrol sees which flags disagree; the tags carry+// each vendor's answer, so the issue shows the direction too (BUILDER-94P).+export function reportMismatchToSentry({+ flag,+ launchdarkly,+ posthog,+}: Mismatch) {+ const launchDarklyValue = describeValue(launchdarkly.value);+ const postHogValue = describeValue(posthog.value);+ const message = `Feature flag mismatch: ${flag} (LaunchDarkly=${launchDarklyValue}, PostHog=${postHogValue})`;+ console.info(message);+ Sentry.captureMessage(message, {+ level: "warning",+ fingerprint: ["feature-flag-mismatch", flag],+ tags: {+ "feature_flag.key": flag,+ "feature_flag.launchdarkly": launchDarklyValue.slice(0, MAX_TAG_LENGTH),+ "feature_flag.posthog": postHogValue.slice(0, MAX_TAG_LENGTH),+ },+ contexts: {+ feature_flag_mismatch: {+ flag,+ launchdarkly: launchDarklyValue,+ posthog: postHogValue,+ },+ },+ });+}++function describeValue(value: unknown) {+ return value === undefined ? "undefined" : JSON.stringify(value);+}