Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): redact request headers and internal fields from job log rows (#5480)

Abimael Martellf582d2c2708f
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index a1bfbe0c0..d9210d8c7 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -314,6 +314,48 @@ describe("operational job state logging", () => {
publishMessage.mockResolvedValue("message-id");
});
+ it("publishes scrape options without credentials or internal fields", async () => {
+ const id = "019e6f45-7778-727d-adf0-0abe9d5062c1";
+ const options = {
+ formats: ["markdown"],
+ headers: { Authorization: "Bearer secret", cookie: "session=abc" },
+ actions: [
+ { type: "click", selector: "#login" },
+ { type: "write", text: "hunter2", selector: "#password" },
+ ],
+ __searchPreviewToken: "preview-token",
+ __agentInterop: { auth: "s2s-secret", requestId: "r1", shouldBill: true },
+ };
+ await logScrape({
+ id,
+ request_id: id,
+ url: "https://example.com",
+ is_successful: true,
+ time_taken: 1,
+ team_id: "team-id",
+ options: options as any,
+ credits_cost: 1,
+ skipNuq: false,
+ zeroDataRetention: false,
+ });
+
+ const published = JSON.parse(
+ publishMessage.mock.calls[0][0].data.toString(),
+ );
+ expect(published.options).toEqual({
+ formats: ["markdown"],
+ headers: { Authorization: "<redacted>", cookie: "<redacted>" },
+ actions: [
+ { type: "click", selector: "#login" },
+ { type: "write", text: "<redacted>", selector: "#password" },
+ ],
+ });
+ expect(JSON.stringify(published)).not.toContain("secret");
+ expect(JSON.stringify(published)).not.toContain("hunter2");
+ // The caller's object is left as it was.
+ expect(options.headers.Authorization).toBe("Bearer secret");
+ });
+
it("writes terminal standalone scrape state", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
await logScrape({
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index d4a4513a4..805f9c86b 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -27,6 +27,7 @@ import { trackFirstSurfaceUse } from "../posthog";
import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";
import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";
import { sanitizeLogData, sanitizeText } from "./sanitize";
+import { redactLogData } from "./redact";
import {
JOB_ACCESS_TTL_MS,
isApiJobKind,
@@ -391,11 +392,14 @@ async function robustInsert(table: string, data: any, _logger: Logger) {
return;
}
- // Clean the row once so the log receives accepted values.
- data = sanitizeLogData({
- ...data,
- created_at: data.created_at ?? new Date(),
- });
+ // Redact credentials and internal fields, then clean the row once so the
+ // log receives accepted values.
+ data = sanitizeLogData(
+ redactLogData({
+ ...data,
+ created_at: data.created_at ?? new Date(),
+ }),
+ );
// Pub/Sub is the store of record: publish and wait for it. A failed
// publish fails the log call.
diff --git a/apps/api/src/services/logging/redact.test.ts b/apps/api/src/services/logging/redact.test.ts
new file mode 100644
index 000000000..b09876d21
--- /dev/null
+++ b/apps/api/src/services/logging/redact.test.ts
@@ -0,0 +1,108 @@
+import { REDACTED, redactLogData } from "./redact";
+
+describe("redactLogData", () => {
+ it("replaces header values under any spelling and keeps the names", () => {
+ const row = redactLogData({
+ options: {
+ headers: {
+ Authorization: "Bearer secret",
+ cookie: "session=abc",
+ "X-Api-Key": "key",
+ "Accept-Language": "en",
+ },
+ },
+ });
+
+ expect(row.options.headers).toEqual({
+ Authorization: REDACTED,
+ cookie: REDACTED,
+ "X-Api-Key": REDACTED,
+ "Accept-Language": REDACTED,
+ });
+ });
+
+ it("redacts webhook headers nested inside crawl options", () => {
+ const row = redactLogData({
+ options: {
+ scrapeOptions: { formats: ["markdown"] },
+ webhook: {
+ url: "https://hooks.example/firecrawl",
+ headers: { Authorization: "Bearer hook-secret" },
+ metadata: { env: "prod" },
+ },
+ },
+ });
+
+ expect(row.options.webhook.url).toBe("https://hooks.example/firecrawl");
+ expect(row.options.webhook.headers).toEqual({ Authorization: REDACTED });
+ expect(row.options.webhook.metadata).toEqual({ env: "prod" });
+ });
+
+ it("drops internal double-underscore fields at every depth", () => {
+ const row = redactLogData({
+ options: {
+ formats: ["markdown"],
+ __searchPreviewToken: "preview-token",
+ __agentInterop: { auth: "s2s-secret", requestId: "r1" },
+ scrapeOptions: { __forceFirePDF: true, onlyMainContent: true },
+ },
+ });
+
+ expect(row.options).toEqual({
+ formats: ["markdown"],
+ scrapeOptions: { onlyMainContent: true },
+ });
+ });
+
+ it("redacts the text of write actions and nothing else about them", () => {
+ const row = redactLogData({
+ options: {
+ actions: [
+ { type: "click", selector: "#login" },
+ {
+ type: "write",
+ text: "hunter2",
+ selector: "#password",
+ __internal: "flag",
+ headers: { Authorization: "Bearer x" },
+ },
+ { type: "wait", milliseconds: 500 },
+ ],
+ },
+ });
+
+ expect(row.options.actions).toEqual([
+ { type: "click", selector: "#login" },
+ {
+ type: "write",
+ text: REDACTED,
+ selector: "#password",
+ headers: { Authorization: REDACTED },
+ },
+ { type: "wait", milliseconds: 500 },
+ ]);
+ });
+
+ it("leaves non-header objects named headers alone", () => {
+ const row = redactLogData({ headers: { count: 3 } });
+ expect(row.headers).toEqual({ count: 3 });
+ });
+
+ it("returns class instances by reference and keeps other values intact", () => {
+ const createdAt = new Date("2026-01-01T00:00:00Z");
+ const buffer = Buffer.from("x");
+ const row = redactLogData({
+ created_at: createdAt,
+ buffer,
+ url: "https://example.com",
+ credits_cost: 1,
+ options: null,
+ });
+
+ expect(row.created_at).toBe(createdAt);
+ expect(row.buffer).toBe(buffer);
+ expect(row.url).toBe("https://example.com");
+ expect(row.credits_cost).toBe(1);
+ expect(row.options).toBeNull();
+ });
+});
diff --git a/apps/api/src/services/logging/redact.ts b/apps/api/src/services/logging/redact.ts
new file mode 100644
index 000000000..06e419bec
--- /dev/null
+++ b/apps/api/src/services/logging/redact.ts
@@ -0,0 +1,75 @@
+/**
+ * Redaction for every row that leaves the logging boundary.
+ *
+ * Request options are stored on log rows as the caller sent them. Three
+ * parts of them are credentials or internal plumbing and have no business in
+ * a warehouse:
+ *
+ * - `headers`: the caller's headers for the target site (Authorization,
+ * Cookie, API keys under any name) and webhook delivery headers. Names are
+ * kept, since they describe how the request was made; values are replaced.
+ * - `write` actions: the text typed into a page, which is where passwords go.
+ * - `__`-prefixed fields: internal request plumbing (service-to-service auth,
+ * preview tokens, engine flags) that the public schema never exposes.
+ *
+ * Runs on the row before sanitizing, so both stores receive the same values.
+ */
+
+export const REDACTED = "<redacted>";
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
+ return false;
+ }
+ const proto = Object.getPrototypeOf(value);
+ return proto === Object.prototype || proto === null;
+}
+
+function isHeaderMap(value: unknown): value is Record<string, string> {
+ return (
+ isPlainObject(value) &&
+ Object.values(value).every(entry => typeof entry === "string")
+ );
+}
+
+function redactHeaderMap(headers: Record<string, string>) {
+ return Object.fromEntries(
+ Object.keys(headers).map(name => [name, REDACTED] as const),
+ );
+}
+
+function redactAction(action: unknown): unknown {
+ if (
+ isPlainObject(action) &&
+ action.type === "write" &&
+ typeof action.text === "string"
+ ) {
+ return redactLogData({ ...action, text: REDACTED });
+ }
+ return redactLogData(action);
+}
+
+/**
+ * Deep-redacts a log row. Plain objects and arrays are walked; Dates, Buffers
+ * and other class instances are returned by reference.
+ */
+export function redactLogData<T>(value: T): T {
+ if (Array.isArray(value)) {
+ return value.map(item => redactLogData(item)) as T;
+ }
+ if (!isPlainObject(value)) {
+ return value;
+ }
+ const out: Record<string, unknown> = {};
+ for (const [key, entry] of Object.entries(value)) {
+ if (key.startsWith("__")) continue;
+ if (key === "headers" && isHeaderMap(entry)) {
+ out[key] = redactHeaderMap(entry);
+ } else if (key === "actions" && Array.isArray(entry)) {
+ out[key] = entry.map(redactAction);
+ } else {
+ out[key] = redactLogData(entry);
+ }
+ }
+ return out as T;
+}