Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): redact headers from logged request errors (#5301)
Abimael Martell0052e59e483d
diff --git a/apps/api/src/scraper/scrapeURL/lib/__tests__/fetch.test.ts b/apps/api/src/scraper/scrapeURL/lib/__tests__/fetch.test.tsnew file mode 100644index 000000000..8528b0c4d--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/__tests__/fetch.test.ts@@ -0,0 +1,123 @@+import { redactHeaders, robustFetch } from "../fetch";+import type { MockState } from "../mock";++const secretHeaders = {+ Authorization: "Bearer secret-value",+ "Proxy-Authorization": "Basic secret-value",+ Cookie: "session=secret-value",+ "Set-Cookie": "session=secret-value",+ "x-api-key": "secret-value",+ "api-key": "secret-value",+ "X-Auth-Token": "secret-value",+ "X-Client-Secret": "secret-value",+ "Content-Type": "application/json",+ "User-Agent": "firecrawl",+};++function makeLogger() {+ const logger: any = {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ };+ logger.child = vi.fn(() => logger);+ return logger;+}++function mockWith(status: number, body: string): MockState {+ return {+ requests: [+ {+ time: 0,+ options: { url: "https://upstream.test/run", method: "POST" },+ result: { status, headers: {}, body },+ },+ ],+ tracker: {},+ } as unknown as MockState;+}++async function captureCause(promise: Promise<unknown>): Promise<any> {+ try {+ await promise;+ } catch (error) {+ return (error as Error).cause;+ }+ throw new Error("expected robustFetch to throw");+}++describe("redactHeaders", () => {+ it("redacts credential headers case-insensitively and keeps the rest", () => {+ expect(redactHeaders(secretHeaders)).toEqual({+ Authorization: "[REDACTED]",+ "Proxy-Authorization": "[REDACTED]",+ Cookie: "[REDACTED]",+ "Set-Cookie": "[REDACTED]",+ "x-api-key": "[REDACTED]",+ "api-key": "[REDACTED]",+ "X-Auth-Token": "[REDACTED]",+ "X-Client-Secret": "[REDACTED]",+ "Content-Type": "application/json",+ "User-Agent": "firecrawl",+ });+ });++ it("passes undefined through", () => {+ expect(redactHeaders(undefined)).toBeUndefined();+ });+});++describe("robustFetch error logging", () => {+ it.each([+ ["a failure status", mockWith(500, "{}"), undefined],+ ["malformed JSON", mockWith(200, "not json"), undefined],+ [+ "a schema mismatch",+ mockWith(200, JSON.stringify({ other: 1 })),+ (z: typeof import("zod")) => z.object({ id: z.string() }),+ ],+ ])(+ "never logs or throws credential headers on %s",+ async (_, mock, schemaOf) => {+ const z = await import("zod");+ const logger = makeLogger();++ const cause = await captureCause(+ robustFetch({+ url: "https://upstream.test/run",+ method: "POST",+ headers: secretHeaders,+ body: { input: { file_content: "BASE64" } },+ logger,+ mock,+ schema: schemaOf?.(z),+ }),+ );++ const serialized = JSON.stringify([cause, logger.debug.mock.calls]);+ expect(serialized).not.toContain("secret-value");+ expect(cause.params.headers.Authorization).toBe("[REDACTED]");+ expect(cause.params.headers["User-Agent"]).toBe("firecrawl");+ },+ );++ it("redacts headers forwarded inside the request body", async () => {+ const logger = makeLogger();++ const cause = await captureCause(+ robustFetch({+ url: "https://upstream.test/run",+ method: "POST",+ body: { url: "https://example.com", headers: secretHeaders },+ logger,+ mock: mockWith(500, "{}"),+ }),+ );++ const serialized = JSON.stringify([cause, logger.debug.mock.calls]);+ expect(serialized).not.toContain("secret-value");+ expect(cause.params.body.url).toBe("https://example.com");+ expect(cause.params.body.headers["User-Agent"]).toBe("firecrawl");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 498a9ed1a..524dc4b71 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -26,6 +26,21 @@ type RobustFetchParams<Schema extends z.Schema<any>> = {useCacheableLookup?: boolean;};+const SENSITIVE_HEADER =+ /^(authorization|proxy-authorization|cookie|set-cookie|x-api-key|api-key)$|token|secret/i;++export function redactHeaders(+ headers: Record<string, string> | undefined,+): Record<string, string> | undefined {+ if (headers === undefined) return undefined;+ return Object.fromEntries(+ Object.entries(headers).map(([name, value]) => [+ name,+ SENSITIVE_HEADER.test(name) ? "[REDACTED]" : value,+ ]),+ );+}+const robustAgent = new Agent({headersTimeout: 0,bodyTimeout: 0,@@ -79,23 +94,30 @@ export async function robustFetch<abort,};- // omit pdf file content from logs- const logParams = {- ...params,- body: body?.input+ // omit pdf file content and credentials from logs+ const logBody = body?.input+ ? {+ ...body,+ input: {+ ...body.input,+ file_content: undefined,+ },+ }+ : body?.pdf? {...body,- input: {- ...body.input,- file_content: undefined,- },+ pdf: undefined,}- : body?.pdf- ? {- ...body,- pdf: undefined,- }- : body,+ : body;+ const logParams = {+ ...params,+ headers: redactHeaders(headers),+ body:+ logBody?.headers !== null &&+ typeof logBody?.headers === "object" &&+ !(logBody instanceof FormData)+ ? { ...logBody, headers: redactHeaders(logBody.headers) }+ : logBody,logger: undefined,};