Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): redact request headers and internal fields from job log rows (#5480)
Abimael Martellf582d2c2708f
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex a1bfbe0c0..d9210d8c7 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -314,6 +314,48 @@ describe("operational job state logging", () => {publishMessage.mockResolvedValue("message-id");});+ it("publishes scrape options without credentials or internal fields", async () => {+ const id = "019e6f45-7778-727d-adf0-0abe9d5062c1";+ const options = {+ formats: ["markdown"],+ headers: { Authorization: "Bearer secret", cookie: "session=abc" },+ actions: [+ { type: "click", selector: "#login" },+ { type: "write", text: "hunter2", selector: "#password" },+ ],+ __searchPreviewToken: "preview-token",+ __agentInterop: { auth: "s2s-secret", requestId: "r1", shouldBill: true },+ };+ await logScrape({+ id,+ request_id: id,+ url: "https://example.com",+ is_successful: true,+ time_taken: 1,+ team_id: "team-id",+ options: options as any,+ credits_cost: 1,+ skipNuq: false,+ zeroDataRetention: false,+ });++ const published = JSON.parse(+ publishMessage.mock.calls[0][0].data.toString(),+ );+ expect(published.options).toEqual({+ formats: ["markdown"],+ headers: { Authorization: "<redacted>", cookie: "<redacted>" },+ actions: [+ { type: "click", selector: "#login" },+ { type: "write", text: "<redacted>", selector: "#password" },+ ],+ });+ expect(JSON.stringify(published)).not.toContain("secret");+ expect(JSON.stringify(published)).not.toContain("hunter2");+ // The caller's object is left as it was.+ expect(options.headers.Authorization).toBe("Bearer secret");+ });+it("writes terminal standalone scrape state", async () => {const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";await logScrape({diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex d4a4513a4..805f9c86b 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -27,6 +27,7 @@ import { trackFirstSurfaceUse } from "../posthog";import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";import { sanitizeLogData, sanitizeText } from "./sanitize";+import { redactLogData } from "./redact";import {JOB_ACCESS_TTL_MS,isApiJobKind,@@ -391,11 +392,14 @@ async function robustInsert(table: string, data: any, _logger: Logger) {return;}- // Clean the row once so the log receives accepted values.- data = sanitizeLogData({- ...data,- created_at: data.created_at ?? new Date(),- });+ // Redact credentials and internal fields, then clean the row once so the+ // log receives accepted values.+ data = sanitizeLogData(+ redactLogData({+ ...data,+ created_at: data.created_at ?? new Date(),+ }),+ );// Pub/Sub is the store of record: publish and wait for it. A failed// publish fails the log call.diff --git a/apps/api/src/services/logging/redact.test.ts b/apps/api/src/services/logging/redact.test.tsnew file mode 100644index 000000000..b09876d21--- /dev/null+++ b/apps/api/src/services/logging/redact.test.ts@@ -0,0 +1,108 @@+import { REDACTED, redactLogData } from "./redact";++describe("redactLogData", () => {+ it("replaces header values under any spelling and keeps the names", () => {+ const row = redactLogData({+ options: {+ headers: {+ Authorization: "Bearer secret",+ cookie: "session=abc",+ "X-Api-Key": "key",+ "Accept-Language": "en",+ },+ },+ });++ expect(row.options.headers).toEqual({+ Authorization: REDACTED,+ cookie: REDACTED,+ "X-Api-Key": REDACTED,+ "Accept-Language": REDACTED,+ });+ });++ it("redacts webhook headers nested inside crawl options", () => {+ const row = redactLogData({+ options: {+ scrapeOptions: { formats: ["markdown"] },+ webhook: {+ url: "https://hooks.example/firecrawl",+ headers: { Authorization: "Bearer hook-secret" },+ metadata: { env: "prod" },+ },+ },+ });++ expect(row.options.webhook.url).toBe("https://hooks.example/firecrawl");+ expect(row.options.webhook.headers).toEqual({ Authorization: REDACTED });+ expect(row.options.webhook.metadata).toEqual({ env: "prod" });+ });++ it("drops internal double-underscore fields at every depth", () => {+ const row = redactLogData({+ options: {+ formats: ["markdown"],+ __searchPreviewToken: "preview-token",+ __agentInterop: { auth: "s2s-secret", requestId: "r1" },+ scrapeOptions: { __forceFirePDF: true, onlyMainContent: true },+ },+ });++ expect(row.options).toEqual({+ formats: ["markdown"],+ scrapeOptions: { onlyMainContent: true },+ });+ });++ it("redacts the text of write actions and nothing else about them", () => {+ const row = redactLogData({+ options: {+ actions: [+ { type: "click", selector: "#login" },+ {+ type: "write",+ text: "hunter2",+ selector: "#password",+ __internal: "flag",+ headers: { Authorization: "Bearer x" },+ },+ { type: "wait", milliseconds: 500 },+ ],+ },+ });++ expect(row.options.actions).toEqual([+ { type: "click", selector: "#login" },+ {+ type: "write",+ text: REDACTED,+ selector: "#password",+ headers: { Authorization: REDACTED },+ },+ { type: "wait", milliseconds: 500 },+ ]);+ });++ it("leaves non-header objects named headers alone", () => {+ const row = redactLogData({ headers: { count: 3 } });+ expect(row.headers).toEqual({ count: 3 });+ });++ it("returns class instances by reference and keeps other values intact", () => {+ const createdAt = new Date("2026-01-01T00:00:00Z");+ const buffer = Buffer.from("x");+ const row = redactLogData({+ created_at: createdAt,+ buffer,+ url: "https://example.com",+ credits_cost: 1,+ options: null,+ });++ expect(row.created_at).toBe(createdAt);+ expect(row.buffer).toBe(buffer);+ expect(row.url).toBe("https://example.com");+ expect(row.credits_cost).toBe(1);+ expect(row.options).toBeNull();+ });+});diff --git a/apps/api/src/services/logging/redact.ts b/apps/api/src/services/logging/redact.tsnew file mode 100644index 000000000..06e419bec--- /dev/null+++ b/apps/api/src/services/logging/redact.ts@@ -0,0 +1,75 @@+/**+ * Redaction for every row that leaves the logging boundary.+ *+ * Request options are stored on log rows as the caller sent them. Three+ * parts of them are credentials or internal plumbing and have no business in+ * a warehouse:+ *+ * - `headers`: the caller's headers for the target site (Authorization,+ * Cookie, API keys under any name) and webhook delivery headers. Names are+ * kept, since they describe how the request was made; values are replaced.+ * - `write` actions: the text typed into a page, which is where passwords go.+ * - `__`-prefixed fields: internal request plumbing (service-to-service auth,+ * preview tokens, engine flags) that the public schema never exposes.+ *+ * Runs on the row before sanitizing, so both stores receive the same values.+ */++export const REDACTED = "<redacted>";++function isPlainObject(value: unknown): value is Record<string, unknown> {+ if (value === null || typeof value !== "object" || Array.isArray(value)) {+ return false;+ }+ const proto = Object.getPrototypeOf(value);+ return proto === Object.prototype || proto === null;+}++function isHeaderMap(value: unknown): value is Record<string, string> {+ return (+ isPlainObject(value) &&+ Object.values(value).every(entry => typeof entry === "string")+ );+}++function redactHeaderMap(headers: Record<string, string>) {+ return Object.fromEntries(+ Object.keys(headers).map(name => [name, REDACTED] as const),+ );+}++function redactAction(action: unknown): unknown {+ if (+ isPlainObject(action) &&+ action.type === "write" &&+ typeof action.text === "string"+ ) {+ return redactLogData({ ...action, text: REDACTED });+ }+ return redactLogData(action);+}++/**+ * Deep-redacts a log row. Plain objects and arrays are walked; Dates, Buffers+ * and other class instances are returned by reference.+ */+export function redactLogData<T>(value: T): T {+ if (Array.isArray(value)) {+ return value.map(item => redactLogData(item)) as T;+ }+ if (!isPlainObject(value)) {+ return value;+ }+ const out: Record<string, unknown> = {};+ for (const [key, entry] of Object.entries(value)) {+ if (key.startsWith("__")) continue;+ if (key === "headers" && isHeaderMap(entry)) {+ out[key] = redactHeaderMap(entry);+ } else if (key === "actions" && Array.isArray(entry)) {+ out[key] = entry.map(redactAction);+ } else {+ out[key] = redactLogData(entry);+ }+ }+ return out as T;+}