AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
Merge commit from fork
Reinier van der Leer0edeafc5902f
diff --git a/autogpt_platform/frontend/instrumentation-client.ts b/autogpt_platform/frontend/instrumentation-client.tsindex 80da7f5d92..b2c2e65053 100644--- a/autogpt_platform/frontend/instrumentation-client.ts+++ b/autogpt_platform/frontend/instrumentation-client.ts@@ -3,6 +3,7 @@// https://docs.sentry.io/platforms/javascript/guides/nextjs/import { isNextRSCNavigationFallback } from "@/lib/sentry-filters";+import { getSentryTunnelURL } from "@/lib/sentry-tunnel";import { setupSessionReplay } from "@/lib/session-replay";import { environment } from "@/services/environment";import * as Sentry from "@sentry/nextjs";@@ -16,6 +17,10 @@ Sentry.init({enabled: shouldEnable,+ // The tunnel is same-origin, so fetch would attach our cookies to every envelope.+ tunnel: getSentryTunnelURL(environment.getSentryDSN()),+ transportOptions: { fetchOptions: { credentials: "omit" } },+// Suppress cross-origin stylesheet errors from Sentry Replay (rrweb)// serializing DOM snapshots with cross-origin stylesheets// (e.g., from browser extensions or CDN-loaded CSS)diff --git a/autogpt_platform/frontend/next.config.mjs b/autogpt_platform/frontend/next.config.mjsindex ad1463686c..f785f73d47 100644--- a/autogpt_platform/frontend/next.config.mjs+++ b/autogpt_platform/frontend/next.config.mjs@@ -200,11 +200,7 @@ export default skipSentryPluginenabled: true,},- // Route browser requests to Sentry through a Next.js rewrite to circumvent ad-blockers.- // This can increase your server load as well as your hosting bill.- // Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client-- // side errors will fail.- tunnelRoute: "/store",+ // Browser Sentry envelopes use the tunnel in src/lib/sentry-tunnel.ts, relayed by src/middleware.ts.// No need to hide source maps from generated client bundles// since the source is public anyway :)diff --git a/autogpt_platform/frontend/sentry.edge.config.ts b/autogpt_platform/frontend/sentry.edge.config.tsindex b2f762e745..63791a0cde 100644--- a/autogpt_platform/frontend/sentry.edge.config.ts+++ b/autogpt_platform/frontend/sentry.edge.config.ts@@ -3,6 +3,8 @@// Note that this config is unrelated to the Vercel Edge Runtime and is also required when running locally.// https://docs.sentry.io/platforms/javascript/guides/nextjs/+import { stripRequestCredentials } from "@/lib/sentry-filters";+import { isSentryTunnelTransaction } from "@/lib/sentry-tunnel";import { environment } from "@/services/environment";import * as Sentry from "@sentry/nextjs";@@ -24,11 +26,19 @@ Sentry.init({/^https:\/\/.*\.agpt\.co\/api/,],+ beforeSend: stripRequestCredentials,+ beforeSendTransaction(event) {+ return isSentryTunnelTransaction(event)+ ? null+ : stripRequestCredentials(event);+ },+// Setting this option to true will print useful information to the console while you're setting up Sentry.debug: false,enableLogs: true,integrations: [+ Sentry.requestDataIntegration({ include: { cookies: false, data: false } }),Sentry.captureConsoleIntegration({ levels: ["fatal", "error", "warn"] }),Sentry.extraErrorDataIntegration(),],diff --git a/autogpt_platform/frontend/sentry.server.config.ts b/autogpt_platform/frontend/sentry.server.config.tsindex 269e2c9098..527e645609 100644--- a/autogpt_platform/frontend/sentry.server.config.ts+++ b/autogpt_platform/frontend/sentry.server.config.ts@@ -2,6 +2,8 @@// The config you add here will be used whenever the server handles a request.// https://docs.sentry.io/platforms/javascript/guides/nextjs/+import { stripRequestCredentials } from "@/lib/sentry-filters";+import { isSentryTunnelTransaction } from "@/lib/sentry-tunnel";import { environment } from "@/services/environment";import * as Sentry from "@sentry/nextjs";// import { NodeProfilingIntegration } from "@sentry/profiling-node";@@ -24,11 +26,19 @@ Sentry.init({/^https:\/\/.*\.agpt\.co\/api/,],+ beforeSend: stripRequestCredentials,+ beforeSendTransaction(event) {+ return isSentryTunnelTransaction(event)+ ? null+ : stripRequestCredentials(event);+ },+// Setting this option to true will print useful information to the console while you're setting up Sentry.debug: false,// Integrationsintegrations: [+ Sentry.requestDataIntegration({ include: { cookies: false, data: false } }),Sentry.anrIntegration(),// NodeProfilingIntegration,Sentry.extraErrorDataIntegration(),diff --git a/autogpt_platform/frontend/src/__tests__/instrumentation-client.test.ts b/autogpt_platform/frontend/src/__tests__/instrumentation-client.test.tsindex 721728ebd9..017712aba0 100644--- a/autogpt_platform/frontend/src/__tests__/instrumentation-client.test.ts+++ b/autogpt_platform/frontend/src/__tests__/instrumentation-client.test.ts@@ -41,6 +41,35 @@ describe("Sentry client instrumentation", () => {expect(types).toEqual(expect.arrayContaining(["crash", "intervention"]));});+ it("sends envelopes to the same-origin tunnel without cookies", async () => {+ vi.resetModules();+ initMock.mockClear();+ vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "CLOUD");+ vi.stubEnv("NEXT_PUBLIC_APP_ENV", "dev");+ await import("../../instrumentation-client");+ vi.unstubAllEnvs();++ const { tunnel, transportOptions } = initMock.mock.calls[0][0];++ expect(tunnel).toBe(+ "/relay/snentry?o=4505260022104064&p=4507946746380288&r=us",+ );+ expect(transportOptions?.fetchOptions?.credentials).toBe("omit");+ });++ it("starts with no tunnel while Sentry is disabled", async () => {+ vi.resetModules();+ initMock.mockClear();+ vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "LOCAL");+ await import("../../instrumentation-client");+ vi.unstubAllEnvs();++ const { enabled, tunnel } = initMock.mock.calls[0][0];++ expect(enabled).toBe(false);+ expect(tunnel).toBeUndefined();+ });+it("drops Next's handled RSC fetch fallback but keeps other fetch errors", async () => {await import("../../instrumentation-client");diff --git a/autogpt_platform/frontend/src/__tests__/sentry-runtime-configs.test.ts b/autogpt_platform/frontend/src/__tests__/sentry-runtime-configs.test.tsnew file mode 100644index 0000000000..4efa5c1328--- /dev/null+++ b/autogpt_platform/frontend/src/__tests__/sentry-runtime-configs.test.ts@@ -0,0 +1,139 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const { initMock } = vi.hoisted(() => ({ initMock: vi.fn() }));++const BODY_MARKER = "marker-4f1c";++vi.mock("@sentry/nextjs", async (importOriginal) => {+ const actual = await importOriginal<typeof import("@sentry/nextjs")>();+ function stubIntegration(name: string) {+ return vi.fn(() => ({ name }));+ }++ return {+ init: initMock,+ requestDataIntegration: actual.requestDataIntegration,+ anrIntegration: stubIntegration("Anr"),+ captureConsoleIntegration: stubIntegration("CaptureConsole"),+ extraErrorDataIntegration: stubIntegration("ExtraErrorData"),+ };+});++beforeEach(() => {+ initMock.mockReset();+ vi.resetModules();+});++describe.each([+ ["edge", () => import("../../sentry.edge.config")],+ ["server", () => import("../../sentry.server.config")],+])("Sentry %s config", (_, load) => {+ it("drops the transactions of relaying a browser envelope", async () => {+ await load();++ const { beforeSendTransaction } = initMock.mock.calls[0][0];+ const serverRelay = {+ transaction: "POST /relay/snentry",+ request: { url: "https://platform.agpt.co/relay/snentry?o=1&p=2" },+ };+ const middlewareRelay = {+ transaction: "middleware POST",+ contexts: { trace: { data: { "http.target": "/relay/snentry" } } },+ };+ const page = {+ transaction: "middleware POST",+ request: { url: "https://platform.agpt.co/marketplace" },+ contexts: { trace: { data: { "http.target": "/marketplace" } } },+ };+ const malformed = { request: { url: "http://[" } };++ expect(beforeSendTransaction(serverRelay)).toBeNull();+ expect(beforeSendTransaction(middlewareRelay)).toBeNull();+ expect(beforeSendTransaction(page)).toBe(page);+ expect(beforeSendTransaction(malformed)).toBe(malformed);+ });++ it("records no request cookies", async () => {+ await load();++ const { integrations } = initMock.mock.calls[0][0];+ const requestData = integrations.find(+ (integration: { name: string }) => integration.name === "RequestData",+ );+ const event = requestData.processEvent(+ {+ sdkProcessingMetadata: {+ normalizedRequest: {+ method: "GET",+ url: "https://platform.agpt.co/library",+ headers: { cookie: "a=b", "user-agent": "ua" },+ cookies: { a: "b" },+ },+ },+ },+ {},+ { getOptions: () => ({}) },+ );++ expect(event.request.url).toBe("https://platform.agpt.co/library");+ expect(event.request.cookies).toBeUndefined();+ expect(event.request.headers).toEqual({ "user-agent": "ua" });+ });++ it("records no request body", async () => {+ await load();++ const { integrations } = initMock.mock.calls[0][0];+ const requestData = integrations.find(+ (integration: { name: string }) => integration.name === "RequestData",+ );+ const event = requestData.processEvent(+ {+ sdkProcessingMetadata: {+ normalizedRequest: {+ method: "POST",+ url: "https://platform.agpt.co/api/auth/sign-in/email",+ headers: { "content-type": "application/json" },+ data: JSON.stringify({ email: "[email protected]", password: BODY_MARKER }),+ },+ },+ },+ {},+ { getOptions: () => ({}) },+ );++ expect(event.request.url).toBe(+ "https://platform.agpt.co/api/auth/sign-in/email",+ );+ // The SDK's input, which createEventEnvelope deletes before sending.+ delete event.sdkProcessingMetadata;+ expect(JSON.stringify(event)).not.toContain(BODY_MARKER);+ });++ it("strips the body, cookies and credential headers from errors and transactions", async () => {+ await load();++ const { beforeSend, beforeSendTransaction } = initMock.mock.calls[0][0];+ function withCredentials() {+ return {+ request: {+ url: "https://platform.agpt.co/api/auth/sign-in/email",+ data: { password: BODY_MARKER },+ cookies: { a: "b" },+ headers: {+ Cookie: "a=b",+ authorization: "Basic x",+ "user-agent": "ua",+ },+ },+ };+ }++ const stripped = {+ url: "https://platform.agpt.co/api/auth/sign-in/email",+ headers: { "user-agent": "ua" },+ };+ expect(beforeSend(withCredentials(), {}).request).toEqual(stripped);+ expect(beforeSendTransaction(withCredentials()).request).toEqual(stripped);+ });+});diff --git a/autogpt_platform/frontend/src/__tests__/sentry-tunnel.test.ts b/autogpt_platform/frontend/src/__tests__/sentry-tunnel.test.tsnew file mode 100644index 0000000000..af16305538--- /dev/null+++ b/autogpt_platform/frontend/src/__tests__/sentry-tunnel.test.ts@@ -0,0 +1,150 @@+// @vitest-environment node+// happy-dom hides a Request's cookie header from get(), so a cookie test there cannot fail.+import { getSentryTunnelURL, SENTRY_TUNNEL_PATH } from "@/lib/sentry-tunnel";+import { readFileSync } from "node:fs";+import { unstable_doesMiddlewareMatch } from "next/experimental/testing/server";+import { NextRequest } from "next/server";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++const authMiddlewareMock = vi.fn();++vi.mock("@/lib/auth/middleware", () => ({+ authMiddleware: (...args: unknown[]) => authMiddlewareMock(...args),+}));++import { config, middleware } from "../middleware";++const ORIGIN = "https://platform.agpt.co";+const DSN =+ "https://[email protected]/4507946746380288";++beforeEach(() => {+ authMiddlewareMock.mockReset();+ vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "CLOUD");+ vi.stubEnv("NEXT_PUBLIC_APP_ENV", "dev");+});++afterEach(() => {+ vi.unstubAllEnvs();+});++describe("Sentry tunnel", () => {+ it("runs through our middleware, not Sentry's own tunnel rewrite", () => {+ const nextConfig = readFileSync(+ new URL("../../next.config.mjs", import.meta.url),+ "utf8",+ );++ expect(nextConfig).not.toMatch(/^\s*tunnelRoute:/m);+ expect(+ unstable_doesMiddlewareMatch({+ config,+ url: `${ORIGIN}${getSentryTunnelURL(DSN)}`,+ }),+ ).toBe(true);+ });++ it("forwards our envelopes to our ingest", async () => {+ const response = await middleware(envelopePost(getSentryTunnelURL(DSN)));++ expect(response.headers.get("x-middleware-rewrite")).toBe(+ "https://o4505260022104064.ingest.us.sentry.io/api/4507946746380288/envelope/?hsts=0",+ );+ expect(authMiddlewareMock).not.toHaveBeenCalled();+ });++ it("forwards only the headers Sentry reads", async () => {+ const response = await middleware(envelopePost(getSentryTunnelURL(DSN)));++ expect(forwardedHeaders(response)).toEqual({+ host: "o4505260022104064.ingest.us.sentry.io",+ "content-type": "text/plain;charset=UTF-8",+ "content-length": "3",+ "user-agent": "Mozilla/5.0",+ "x-forwarded-for": "203.0.113.7",+ });+ });++ it.each([+ "o=1&p=2",+ "o=1&p=2&r=evil.example",+ "o=evil.example&p=2",+ "o=1",+ "p=2",+ ])("refuses ?%s without forwarding it", async (query) => {+ const response = await middleware(+ envelopePost(`${SENTRY_TUNNEL_PATH}?${query}`),+ );++ expect(response.status).toBe(404);+ expect(response.headers.get("x-middleware-rewrite")).toBeNull();+ expect(authMiddlewareMock).not.toHaveBeenCalled();+ });++ it("relays nothing while Sentry is disabled", async () => {+ vi.stubEnv("NEXT_PUBLIC_BEHAVE_AS", "LOCAL");+ const response = await middleware(envelopePost(getSentryTunnelURL(DSN)));++ expect(response.status).toBe(404);+ expect(response.headers.get("x-middleware-rewrite")).toBeNull();+ });++ it("relays POST only", async () => {+ const response = await middleware(+ new NextRequest(`${ORIGIN}${getSentryTunnelURL(DSN)}`, { method: "GET" }),+ );++ expect(response.status).toBe(405);+ expect(response.headers.get("allow")).toBe("POST");+ expect(response.headers.get("x-middleware-rewrite")).toBeNull();+ });++ it.each([null, "0", "abc", String(21 * 1024 * 1024)])(+ "refuses a content-length of %s",+ async (contentLength) => {+ const response = await middleware(+ envelopePost(getSentryTunnelURL(DSN), contentLength),+ );++ expect(response.status).toBe(413);+ expect(response.headers.get("x-middleware-rewrite")).toBeNull();+ },+ );++ it("leaves a self-hosted DSN untunnelled", () => {+ expect(getSentryTunnelURL("https://[email protected]/2")).toBe(+ undefined,+ );+ });+});++function envelopePost(+ tunnelURL: string | undefined,+ contentLength: string | null = "3",+) {+ return new NextRequest(`${ORIGIN}${tunnelURL}`, {+ method: "POST",+ body: "{}\n",+ headers: {+ cookie: "better-auth.session_token=secret; sb-proj-auth-token=legacy",+ authorization: "Bearer secret",+ "content-type": "text/plain;charset=UTF-8",+ "user-agent": "Mozilla/5.0",+ "x-forwarded-for": "203.0.113.7",+ "x-unlisted": "secret",+ ...(contentLength === null ? {} : { "content-length": contentLength }),+ },+ });+}++function forwardedHeaders(response: Response) {+ const names = (response.headers.get("x-middleware-override-headers") ?? "")+ .split(",")+ .filter(Boolean);+ return Object.fromEntries(+ names.map((name) => [+ name,+ response.headers.get(`x-middleware-request-${name}`),+ ]),+ );+}diff --git a/autogpt_platform/frontend/src/lib/sentry-filters.ts b/autogpt_platform/frontend/src/lib/sentry-filters.tsindex b20f082921..e7b27e7641 100644--- a/autogpt_platform/frontend/src/lib/sentry-filters.ts+++ b/autogpt_platform/frontend/src/lib/sentry-filters.ts@@ -20,3 +20,21 @@ export function isNextRSCNavigationFallback(event: Event) {const [message] = args;return typeof message === "string" && NEXT_RSC_FALLBACK_MESSAGE.test(message);}++const CREDENTIAL_HEADERS = new Set(["cookie", "authorization"]);++// Keeps the request's body, cookies and credential headers out of server and+// edge events.+export function stripRequestCredentials<T extends Event>(event: T): T {+ const { request } = event;+ if (!request) return event;+ delete request.data;+ delete request.cookies;+ if (!request.headers) return event;+ request.headers = Object.fromEntries(+ Object.entries(request.headers).filter(+ ([name]) => !CREDENTIAL_HEADERS.has(name.toLowerCase()),+ ),+ );+ return event;+}diff --git a/autogpt_platform/frontend/src/lib/sentry-tunnel-middleware.ts b/autogpt_platform/frontend/src/lib/sentry-tunnel-middleware.tsnew file mode 100644index 0000000000..92a2ab9df7--- /dev/null+++ b/autogpt_platform/frontend/src/lib/sentry-tunnel-middleware.ts@@ -0,0 +1,63 @@+import { getSentryTunnelURL } from "@/lib/sentry-tunnel";+import { environment } from "@/services/environment";+import { NextResponse, type NextRequest } from "next/server";++// What Sentry's ingest reads. The forwarded-for headers keep the user's IP,+// which Sentry records; cookies and Authorization never leave.+const FORWARDED_HEADERS = [+ "content-type",+ "content-length",+ "content-encoding",+ "user-agent",+ "sec-ch-ua",+ "sec-ch-ua-model",+ "sec-ch-ua-platform",+ "sec-ch-ua-platform-version",+ "origin",+ "referer",+ "x-forwarded-for",+ "x-vercel-forwarded-for",+];++// Above Replay's ~20 MB event buffer, the largest envelope the browser SDK sends.+const MAX_ENVELOPE_BYTES = 20 * 1024 * 1024;++export function relaySentryEnvelope(request: NextRequest) {+ const destination = getIngestURL(request);+ if (!destination) return new NextResponse(null, { status: 404 });+ if (request.method !== "POST") {+ return new NextResponse(null, { status: 405, headers: { allow: "POST" } });+ }+ const length = Number(request.headers.get("content-length"));+ if (!length || length > MAX_ENVELOPE_BYTES) {+ return new NextResponse(null, { status: 413 });+ }++ const headers = new Headers({ host: destination.host });+ for (const name of FORWARDED_HEADERS) {+ const value = request.headers.get(name);+ if (value !== null) headers.set(name, value);+ }++ return NextResponse.rewrite(destination, { request: { headers } });+}++// Only our own DSN's tunnel URL is relayed, so the path carries nothing to+// other Sentry orgs or projects.+function getIngestURL(request: NextRequest) {+ const { pathname, search, searchParams: query } = request.nextUrl;+ if (pathname + search !== getSentryTunnelURL(environment.getSentryDSN())) {+ return null;+ }++ const org = query.get("o") ?? "";+ const project = query.get("p") ?? "";+ const region = query.get("r");+ if (!/^\d+$/.test(org) || !/^\d+$/.test(project)) return null;+ if (region !== null && !/^[a-z]{2}$/.test(region)) return null;++ const host = region+ ? `o${org}.ingest.${region}.sentry.io`+ : `o${org}.ingest.sentry.io`;+ return new URL(`https://${host}/api/${project}/envelope/?hsts=0`);+}diff --git a/autogpt_platform/frontend/src/lib/sentry-tunnel.ts b/autogpt_platform/frontend/src/lib/sentry-tunnel.tsnew file mode 100644index 0000000000..a2c76aa43d--- /dev/null+++ b/autogpt_platform/frontend/src/lib/sentry-tunnel.ts@@ -0,0 +1,38 @@+import type { Event } from "@sentry/nextjs";++// Browser envelopes go to Sentry through our own domain so ad blockers+// don't drop them. Misspelt on purpose: blocklist rules match the literal word.+export const SENTRY_TUNNEL_PATH = "/relay/snentry";++const SENTRY_INGEST_HOST = /^o(\d+)\.ingest(?:\.([a-z]{2}))?\.sentry\.io$/;++// Carries the DSN's org, project and region as `o`, `p` and `r`, the query+// @sentry/nextjs's own tunnel uses; middleware rebuilds the ingest URL from it.+export function getSentryTunnelURL(dsn: string | undefined) {+ if (!dsn) return undefined;+ const { hostname, pathname } = new URL(dsn);+ const [, org, region] = hostname.match(SENTRY_INGEST_HOST) ?? [];+ if (!org) return undefined;++ const query = new URLSearchParams({ o: org, p: pathname.slice(1) });+ if (region) query.set("r", region);+ return `${SENTRY_TUNNEL_PATH}?${query}`;+}++// Sentry dropped these itself while the tunnel was its `tunnelRoute`. The server+// transaction carries the request URL; the edge middleware's only its target.+export function isSentryTunnelTransaction(event: Event) {+ const target = event.contexts?.trace?.data?.["http.target"];+ return [event.request?.url, target].some(+ (url) => typeof url === "string" && getPathname(url) === SENTRY_TUNNEL_PATH,+ );+}++// A throwing beforeSendTransaction would drop the event, so a malformed URL is no match.+function getPathname(url: string) {+ try {+ return new URL(url, "http://localhost").pathname;+ } catch {+ return null;+ }+}diff --git a/autogpt_platform/frontend/src/middleware.ts b/autogpt_platform/frontend/src/middleware.tsindex 595680a018..c2a059c7a6 100644--- a/autogpt_platform/frontend/src/middleware.ts+++ b/autogpt_platform/frontend/src/middleware.ts@@ -1,7 +1,15 @@import { authMiddleware } from "@/lib/auth/middleware";+import { SENTRY_TUNNEL_PATH } from "@/lib/sentry-tunnel";+import { relaySentryEnvelope } from "@/lib/sentry-tunnel-middleware";import { NextResponse, type NextRequest } from "next/server";export async function middleware(request: NextRequest) {+ // First, ahead of the redirects, auth checks and any broader /relay handler:+ // each of those would drop or misroute the browser's envelopes.+ if (request.nextUrl.pathname === SENTRY_TUNNEL_PATH) {+ return relaySentryEnvelope(request);+ }+// Redirect www to non-www so auth cookies are issued against a single,// canonical host and avoid the auth/cookie domain mismatch (#9188).// Use url.hostname (already lowercase-normalized by the URL parser) insteaddiff --git a/autogpt_platform/frontend/src/services/environment/index.ts b/autogpt_platform/frontend/src/services/environment/index.tsindex 4e44e8bb1e..e81419180a 100644--- a/autogpt_platform/frontend/src/services/environment/index.ts+++ b/autogpt_platform/frontend/src/services/environment/index.ts@@ -152,6 +152,14 @@ function isSentryEnabled() {return process.env.DISABLE_SENTRY !== "true" && (isProd() || isDev());}+// AutoGPT's own Sentry project, which only our hosted deployments report to.+const HOSTED_SENTRY_DSN =+ "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288";++function getSentryDSN() {+ return isSentryEnabled() ? HOSTED_SENTRY_DSN : undefined;+}+function areFeatureFlagsEnabled() {return (process.env.NEXT_PUBLIC_LAUNCHDARKLY_ENABLED === "true" &&@@ -193,6 +201,7 @@ export const environment = {isLocal,isVercelPreview,isSentryEnabled,+ getSentryDSN,isPostHogEnabled,areFeatureFlagsEnabled,};