Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): logging sanitization and retries (#4532)

rahul-firecrawl43db5b698e16
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 568c8380d..a12e1c2fc 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -250,6 +250,20 @@ const configSchema = z.object({
// Google Cloud Pub/Sub
PUBSUB_CREDENTIALS: z.string().optional(),
+ // Publisher backlog cap, per process. Log publishing is fire-and-forget and
+ // retries for up to five minutes, so during a stall the backlog is what
+ // grows; rows beyond the cap are dropped and counted rather than letting a
+ // hung channel take the process down.
+ PUBSUB_MAX_OUTSTANDING_MESSAGES: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(10_000),
+ PUBSUB_MAX_OUTSTANDING_BYTES: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(64 * 1024 * 1024),
// Cloud Bigtable (change tracking bookkeeping store). The client
// auto-detects BIGTABLE_EMULATOR_HOST, so local dev only needs the
diff --git a/apps/api/src/lib/pubsub-log-metrics.ts b/apps/api/src/lib/pubsub-log-metrics.ts
new file mode 100644
index 000000000..38313b9b8
--- /dev/null
+++ b/apps/api/src/lib/pubsub-log-metrics.ts
@@ -0,0 +1,22 @@
+import { Counter, register } from "prom-client";
+
+const NAME = "pubsub_log_publish_total";
+
+/**
+ * Log rows handed to the Pub/Sub publisher, by table and outcome:
+ * `published`, `failed` (every retry exhausted), or `dropped` (backlog cap).
+ * A channel stall shows up here as `failed` and `dropped` rising together,
+ * without waiting for the daily reconciliation against PostgreSQL.
+ *
+ * Looked up before creation so a re-evaluated module (test isolation) does
+ * not register the same series twice in the shared default registry.
+ */
+export const pubsubLogPublishTotal =
+ (register.getSingleMetric(NAME) as
+ | Counter<"table" | "outcome">
+ | undefined) ??
+ new Counter({
+ name: NAME,
+ help: "Log rows handed to the Pub/Sub publisher, by table and outcome",
+ labelNames: ["table", "outcome"] as const,
+ });
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index d553ccd2a..e69b0e11e 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -13,6 +13,7 @@ const {
publishMessage,
flush,
close,
+ metricInc,
} = vi.hoisted(() => {
const logger: any = {
info: vi.fn(),
@@ -46,6 +47,7 @@ const {
publishMessage,
flush,
close,
+ metricInc: vi.fn(),
};
});
@@ -67,6 +69,8 @@ vi.mock("../../config", () => ({
JSON.stringify({ project_id: "firecrawl" }),
).toString("base64"),
USE_DB_AUTHENTICATION: true,
+ PUBSUB_MAX_OUTSTANDING_MESSAGES: 10_000,
+ PUBSUB_MAX_OUTSTANDING_BYTES: 64 * 1024 * 1024,
},
}));
@@ -95,6 +99,10 @@ vi.mock("../posthog", () => ({
trackFirstSurfaceUse: vi.fn(),
}));
+vi.mock("../../lib/pubsub-log-metrics", () => ({
+ pubsubLogPublishTotal: { inc: metricInc },
+}));
+
import {
logRequest,
logSearch,
@@ -102,6 +110,7 @@ import {
type LoggedSearch,
} from "./log_job";
import * as schema from "../../db/schema";
+import { config } from "../../config";
function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
return {
@@ -146,7 +155,7 @@ describe("logSearch", () => {
const inserted = values.mock.calls[0][0];
expect(inserted.query).toBe("helloworld");
expect(inserted.options.query).toBe("nestedquery");
- expect(inserted.options.sources[0].location).toBe("New\u0000York");
+ expect(inserted.options.sources[0].location).toBe("NewYork");
expect(search.options.query).toBe("nested\u0000query");
});
@@ -208,10 +217,16 @@ describe("logRequest", () => {
await logRequest(makeRequest("op_integration_42"));
expect(insert).toHaveBeenCalledWith(schema.requests);
- expect(publishes).toContainEqual({
- name: "requests",
- options: { gaxOpts: { timeout: 60_000 } },
+ expect(publishes[0].name).toBe("requests");
+ const gaxOpts = publishes[0].options.gaxOpts;
+ // A bare `timeout` would collapse the retry budget to one attempt.
+ expect(gaxOpts.timeout).toBeUndefined();
+ expect(gaxOpts.retry.backoffSettings).toMatchObject({
+ initialRpcTimeoutMillis: 15_000,
+ maxRpcTimeoutMillis: 15_000,
+ totalTimeoutMillis: 300_000,
});
+ expect(gaxOpts.retry.retryCodes).toBeUndefined();
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString("utf8"),
@@ -270,6 +285,76 @@ describe("logRequest", () => {
expect(values.mock.calls[1][0].external_request_id).toBe("é".repeat(1024));
});
+ it("cleans NUL bytes and unpaired surrogates for both stores", async () => {
+ // "Łódź" mis-decoded by a client arrives as a lone low surrogate, which
+ // JSON.stringify would emit as "\udc81" and ClickPipes would reject as
+ // invalid JSON; PostgreSQL's driver stores it as U+FFFD. The row must
+ // reach both stores already cleaned, and identical.
+ const replacement = String.fromCharCode(0xfffd);
+ await logRequest({
+ ...makeRequest(null),
+ target_hint: "wyciek Å\udc81ódź" + String.fromCharCode(0) + "!",
+ origin: "api" + String.fromCharCode(0),
+ });
+
+ const inserted = values.mock.calls[0][0];
+ expect(inserted.target_hint).toBe("wyciek Å" + replacement + "ódź!");
+ expect(inserted.origin).toBe("api");
+
+ const raw = publishMessage.mock.calls[0][0].data.toString("utf8");
+ expect(raw).not.toMatch(/\\u[dD][89a-fA-F]/);
+ expect(raw).not.toMatch(/\\u0{4}/);
+ const published = JSON.parse(raw);
+ expect(published.target_hint).toBe(inserted.target_hint);
+ expect(published.origin).toBe("api");
+ });
+
+ it("drops publishes beyond the outstanding cap instead of queueing them", async () => {
+ // A fresh module instance: the outstanding counters are process-wide and
+ // an earlier test leaves a publish that never settles.
+ vi.resetModules();
+ const fresh = await import("./log_job.js");
+ // Hung channel: publishes never settle, so each one stays outstanding.
+ publishMessage.mockReturnValue(new Promise(() => {}));
+ config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 2;
+ try {
+ await fresh.logRequest(makeRequest(null));
+ await fresh.logRequest(makeRequest(null));
+ await fresh.logRequest(makeRequest(null));
+ } finally {
+ config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 10_000;
+ }
+
+ // The database write is never held back by the publisher.
+ expect(values).toHaveBeenCalledTimes(3);
+ expect(publishMessage).toHaveBeenCalledTimes(2);
+ expect(metricInc).toHaveBeenCalledWith({
+ table: "requests",
+ outcome: "dropped",
+ });
+ expect(logger.warn).toHaveBeenCalledWith(
+ "Dropping Pub/Sub log: publisher backlog is full",
+ expect.objectContaining({ outstandingMessages: 2, droppedTotal: 1 }),
+ );
+ });
+
+ it("counts a published row and a failed row separately", async () => {
+ await logRequest(makeRequest(null));
+ await new Promise(resolve => setImmediate(resolve));
+ expect(metricInc).toHaveBeenCalledWith({
+ table: "requests",
+ outcome: "published",
+ });
+
+ publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
+ await logRequest(makeRequest(null));
+ await new Promise(resolve => setImmediate(resolve));
+ expect(metricInc).toHaveBeenCalledWith({
+ table: "requests",
+ outcome: "failed",
+ });
+ });
+
it("flushes Pub/Sub messages during shutdown", async () => {
await logRequest(makeRequest(null));
@@ -279,3 +364,45 @@ describe("logRequest", () => {
expect(close).toHaveBeenCalledOnce();
});
});
+
+describe("shutdownPubSubLogging deadline", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ values.mockResolvedValue(undefined);
+ publishMessage.mockResolvedValue("message-id");
+ publishes.length = 0;
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("closes the client when a flush outlives the shutdown deadline", async () => {
+ // A fresh module instance: shutdown is memoized per process.
+ vi.resetModules();
+ const fresh = await import("./log_job.js");
+ await fresh.logRequest({
+ id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
+ kind: "scrape",
+ api_version: "v2",
+ team_id: "team-id",
+ origin: "api",
+ target_hint: "https://example.com",
+ zeroDataRetention: false,
+ api_key_id: null,
+ external_request_id: null,
+ });
+
+ vi.useFakeTimers();
+ flush.mockReturnValueOnce(new Promise(() => {}));
+ const shutdown = fresh.shutdownPubSubLogging();
+ await vi.advanceTimersByTimeAsync(40_000);
+ await shutdown;
+
+ expect(close).toHaveBeenCalledOnce();
+ expect(logger.warn).toHaveBeenCalledWith(
+ "Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
+ expect.objectContaining({ timeoutMs: 40_000 }),
+ );
+ });
+});
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 8d3169e2f..38236e438 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -23,21 +23,24 @@ import type { CostTracking } from "../../lib/cost-tracking";
import type { Logger } from "winston";
import { saveExtractResult } from "../../lib/extract/extract-redis";
import { trackFirstSurfaceUse } from "../posthog";
-import { PubSub, type Topic } from "@google-cloud/pubsub";
+import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";
+import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";
+import { sanitizeLogData, sanitizeText } from "./sanitize";
configDotenv();
const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
-const nullByteRegex = /\u0000/g;
/**
- * Sanitize string fields by removing null bytes (\u0000)
- * PostgreSQL doesn't allow null bytes in text fields
- * This can come from user-provided data like URLs, origin, integration fields
+ * Null-aware wrapper around the shared text sanitizer, kept where a cleaned
+ * value feeds a later decision (the external_request_id byte cap). Every row
+ * is deep-sanitized again in robustInsert before it reaches either store, so
+ * nothing else depends on this being called; see ./sanitize.ts for what gets
+ * cleaned and why.
*/
function sanitizeString(value: string | null | undefined): string | null {
if (value === null || value === undefined) return null;
- return value.replace(nullByteRegex, "");
+ return sanitizeText(value);
}
const tableMap: Record<string, PgTable> = {
@@ -62,7 +65,46 @@ const tableMap: Record<string, PgTable> = {
let pubSubClient: PubSub | null | undefined;
const pubSubTopics = new Map<string, Topic>();
let pubSubShutdown: Promise<void> | undefined;
-const PUBSUB_PUBLISH_TIMEOUT_MS = 60_000;
+
+// Publish retry policy. Passing only `timeout` makes google-gax collapse the
+// whole retry budget to that one value (CallSettings.merge), so a stalled RPC
+// used to be a single 60 s attempt and then a lost row. An explicit `retry`
+// is applied after that override and replaces the backoff settings wholesale.
+// Short attempts detect a hung channel quickly; the total budget covers every
+// stall measured in production so far, all of which cleared within 140 s.
+// Retry codes stay the client's defaults for Publish (DEADLINE_EXCEEDED,
+// UNAVAILABLE, INTERNAL, UNKNOWN, ABORTED, CANCELLED, RESOURCE_EXHAUSTED).
+// A retry can deliver a batch twice when the first attempt was persisted but
+// its response was lost; the ClickHouse tables dedupe on row id, not message
+// id, so those copies collapse.
+const PUBSUB_PUBLISH_OPTIONS: PublishOptions = {
+ gaxOpts: {
+ retry: {
+ backoffSettings: {
+ initialRetryDelayMillis: 250,
+ retryDelayMultiplier: 2,
+ maxRetryDelayMillis: 15_000,
+ initialRpcTimeoutMillis: 15_000,
+ rpcTimeoutMultiplier: 1,
+ maxRpcTimeoutMillis: 15_000,
+ totalTimeoutMillis: 300_000,
+ },
+ },
+ },
+};
+
+// Shutdown waits this long for in-flight publishes before closing the client.
+// It must fit inside the pods' termination grace period (60 s for nuq
+// workers); a stall that outlives it loses what is still in flight, which is
+// the same outcome as today and what a durable relay is for.
+const PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS = 40_000;
+
+// Rows waiting on Pub/Sub across all topics. Publishing is fire-and-forget,
+// so during a stall this is what grows; see PUBSUB_MAX_OUTSTANDING_* in config.
+let outstandingMessages = 0;
+let outstandingBytes = 0;
+let droppedTotal = 0;
+let lastDropWarningAt = 0;
function getPubSubClient(logger: Logger): PubSub | null {
if (pubSubClient !== undefined) return pubSubClient;
@@ -90,9 +132,7 @@ function getPubSubClient(logger: Logger): PubSub | null {
function getTopic(client: PubSub, table: string): Topic {
let topic = pubSubTopics.get(table);
if (!topic) {
- topic = client.topic(table, {
- gaxOpts: { timeout: PUBSUB_PUBLISH_TIMEOUT_MS },
- });
+ topic = client.topic(table, PUBSUB_PUBLISH_OPTIONS);
pubSubTopics.set(table, topic);
}
return topic;
@@ -102,15 +142,39 @@ async function publishLog(table: string, data: any, logger: Logger) {
const client = getPubSubClient(logger);
if (!client) return;
+ const payload = Buffer.from(JSON.stringify(data));
+ if (
+ outstandingMessages >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||
+ outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES
+ ) {
+ droppedTotal++;
+ pubsubLogPublishTotal.inc({ table, outcome: "dropped" });
+ const now = Date.now();
+ if (now - lastDropWarningAt >= 60_000) {
+ lastDropWarningAt = now;
+ logger.warn("Dropping Pub/Sub log: publisher backlog is full", {
+ outstandingMessages,
+ outstandingBytes,
+ droppedTotal,
+ });
+ }
+ return;
+ }
+
+ outstandingMessages++;
+ outstandingBytes += payload.length;
try {
- await getTopic(client, table).publishMessage({
- data: Buffer.from(JSON.stringify(data)),
- });
+ await getTopic(client, table).publishMessage({ data: payload });
+ pubsubLogPublishTotal.inc({ table, outcome: "published" });
} catch (error) {
+ pubsubLogPublishTotal.inc({ table, outcome: "failed" });
logger.error("Failed to publish log to Pub/Sub", { error });
Sentry.captureException(error, {
tags: { table, operation: "publishPubSubLog" },
});
+ } finally {
+ outstandingMessages--;
+ outstandingBytes -= payload.length;
}
}
@@ -129,19 +193,40 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {
module: "log_job",
method: "shutdownPubSubLogging",
});
- const results = await Promise.allSettled(
+ const flushed = Promise.allSettled(
[...pubSubTopics.values()].map(topic => topic.flush()),
);
- const errors = results.flatMap(result =>
- result.status === "rejected" ? [result.reason] : [],
- );
+ let deadline: NodeJS.Timeout | undefined;
+ const timedOut = new Promise<"timeout">(resolve => {
+ deadline = setTimeout(
+ () => resolve("timeout"),
+ PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,
+ );
+ });
+ const results = await Promise.race([flushed, timedOut]);
+ clearTimeout(deadline);
+
+ if (results === "timeout") {
+ logger.warn(
+ "Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
+ {
+ timeoutMs: PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,
+ outstandingMessages,
+ outstandingBytes,
+ },
+ );
+ } else {
+ const errors = results.flatMap(result =>
+ result.status === "rejected" ? [result.reason] : [],
+ );
- if (errors.length > 0) {
- logger.error("Failed to flush Pub/Sub log publisher", { errors });
- Sentry.captureException(errors[0], {
- tags: { operation: "flushPubSubLogPublisher" },
- extra: { failures: errors.length },
- });
+ if (errors.length > 0) {
+ logger.error("Failed to flush Pub/Sub log publisher", { errors });
+ Sentry.captureException(errors[0], {
+ tags: { operation: "flushPubSubLogPublisher" },
+ extra: { failures: errors.length },
+ });
+ }
}
try {
@@ -175,7 +260,12 @@ async function robustInsert(
}
const target = tableMap[table];
- data = { ...data, created_at: data.created_at ?? new Date() };
+ // The single point where a row leaves for both stores: clean it once so
+ // PostgreSQL and ClickHouse receive identical, accepted values.
+ data = sanitizeLogData({
+ ...data,
+ created_at: data.created_at ?? new Date(),
+ });
void publishLog(table, data, logger);
const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];
diff --git a/apps/api/src/services/logging/sanitize.test.ts b/apps/api/src/services/logging/sanitize.test.ts
new file mode 100644
index 000000000..4c800af8f
--- /dev/null
+++ b/apps/api/src/services/logging/sanitize.test.ts
@@ -0,0 +1,119 @@
+import { sanitizeLogData, sanitizeText } from "./sanitize";
+
+const NUL = "\x00";
+const FFFD = "\uFFFD";
+
+describe("sanitizeText", () => {
+ it("drops NUL bytes", () => {
+ expect(sanitizeText(`hello${NUL}world`)).toBe("helloworld");
+ expect(sanitizeText(NUL + NUL)).toBe("");
+ });
+
+ it("replaces an unpaired low surrogate with U+FFFD", () => {
+ // "Łódź" mis-decoded by a client: the stray 0x81 byte became a lone
+ // low surrogate, which JSON.stringify emits as "\udc81".
+ const input = "wyciek danych Å\udc81ódź";
+ const output = sanitizeText(input);
+ expect(output).toBe(`wyciek danych Å${FFFD}ódź`);
+ expect(JSON.stringify(output)).not.toContain("\\ud");
+ });
+
+ it("replaces an unpaired high surrogate with U+FFFD", () => {
+ expect(sanitizeText("a\ud83db")).toBe(`a${FFFD}b`);
+ expect(sanitizeText("trailing\ud83d")).toBe(`trailing${FFFD}`);
+ });
+
+ it("keeps well-formed surrogate pairs", () => {
+ const emoji = "fire 🔥 crawl 🦀";
+ expect(sanitizeText(emoji)).toBe(emoji);
+ });
+
+ it("handles a lone surrogate next to a valid pair", () => {
+ expect(sanitizeText("\udd25🔥\ud83d")).toBe(`${FFFD}🔥${FFFD}`);
+ });
+
+ it("returns a clean string by reference", () => {
+ const clean = "https://example.com/path?q=ok";
+ expect(sanitizeText(clean)).toBe(clean);
+ });
+});
+
+describe("sanitizeLogData", () => {
+ it("cleans strings nested in objects, arrays and keys", () => {
+ const row = {
+ url: `https://example.com/${NUL}`,
+ options: {
+ query: `nested${NUL}query`,
+ sources: [{ type: "web", location: `New${NUL}York` }],
+ headers: { [`X-${NUL}Bad`]: "va\udc81lue" },
+ },
+ tags: [`a${NUL}`, "🔥"],
+ };
+
+ expect(sanitizeLogData(row)).toEqual({
+ url: "https://example.com/",
+ options: {
+ query: "nestedquery",
+ sources: [{ type: "web", location: "NewYork" }],
+ headers: { "X-Bad": `va${FFFD}lue` },
+ },
+ tags: ["a", "🔥"],
+ });
+ });
+
+ it("does not mutate the input", () => {
+ const row = { options: { query: `bad${NUL}query` } };
+ sanitizeLogData(row);
+ expect(row.options.query).toBe(`bad${NUL}query`);
+ });
+
+ it("returns Dates and Buffers by reference", () => {
+ const created_at = new Date("2026-09-04T13:09:19.785Z");
+ const blob = Buffer.from("raw bytes");
+ const out = sanitizeLogData({
+ created_at,
+ blob,
+ n: 3,
+ ok: true,
+ none: null,
+ });
+
+ expect(out.created_at).toBe(created_at);
+ expect(out.blob).toBe(blob);
+ expect(out.n).toBe(3);
+ expect(out.ok).toBe(true);
+ expect(out.none).toBeNull();
+ });
+
+ it("keeps an own __proto__ key as a plain field", () => {
+ // JSON.parse creates an own property for this key. A plain assignment
+ // would set the prototype instead and drop the field.
+ const input = JSON.parse('{"__proto__":{"a":"x"},"b":1}');
+ input["__proto__"].a = `x${NUL}y`;
+
+ const out = sanitizeLogData(input);
+
+ expect(Object.keys(out)).toEqual(["__proto__", "b"]);
+ expect(out["__proto__"]).toEqual({ a: "xy" });
+ expect(Object.getPrototypeOf(out)).toBe(Object.prototype);
+ expect(JSON.stringify(out)).toBe('{"__proto__":{"a":"xy"},"b":1}');
+ });
+
+ it("passes primitives through", () => {
+ expect(sanitizeLogData(42)).toBe(42);
+ expect(sanitizeLogData(null)).toBeNull();
+ expect(sanitizeLogData(undefined)).toBeUndefined();
+ });
+
+ it("produces JSON that a strict parser accepts", () => {
+ const published = JSON.stringify(
+ sanitizeLogData({ target_hint: "Å\udc81ód", q: `a${NUL}b` }),
+ );
+ expect(published).not.toMatch(/\\u[dD][89a-fA-F]/);
+ expect(published).not.toMatch(/\\u0{4}/);
+ expect(JSON.parse(published)).toEqual({
+ target_hint: `Å${FFFD}ód`,
+ q: "ab",
+ });
+ });
+});
diff --git a/apps/api/src/services/logging/sanitize.ts b/apps/api/src/services/logging/sanitize.ts
new file mode 100644
index 000000000..a561b6461
--- /dev/null
+++ b/apps/api/src/services/logging/sanitize.ts
@@ -0,0 +1,63 @@
+/**
+ * Text sanitizer for every value that leaves the logging boundary.
+ *
+ * Log rows go to two stores that disagree on what a string may contain:
+ *
+ * - PostgreSQL rejects a NUL byte anywhere in `text` or `jsonb`, including
+ * the escaped form JSON.stringify produces for it inside `jsonb`
+ * (error 22P05).
+ * - ClickPipes parses the published JSON strictly and rejects an unpaired
+ * UTF-16 surrogate, which JSON.stringify emits as an escape such as
+ * `\udc81` when a client sends mis-decoded bytes. PostgreSQL's driver
+ * quietly stores the same character as U+FFFD.
+ *
+ * So the row is cleaned once, before the insert and the publish, and both
+ * stores end up holding identical values: NUL bytes are dropped and lone
+ * surrogates become U+FFFD, the same replacement PostgreSQL applies.
+ */
+
+// A quick pre-check so clean strings are returned untouched, no allocation.
+const NEEDS_SANITIZING = /[\uD800-\uDFFF]|\x00/;
+
+// One pass: a well-formed surrogate pair (kept as is), a lone surrogate
+// (replaced), or a NUL byte (dropped). Ordering matters: the pair
+// alternative must come first so valid pairs are never split.
+const UNSAFE_TEXT = /[\uD800-\uDBFF][\uDC00-\uDFFF]|[\uD800-\uDFFF]|\x00/g;
+
+export function sanitizeText(value: string): string {
+ if (!NEEDS_SANITIZING.test(value)) return value;
+ return value.replace(UNSAFE_TEXT, match => {
+ if (match.length === 2) return match;
+ return match === "\x00" ? "" : "\uFFFD";
+ });
+}
+
+/**
+ * Deep-sanitizes a log row. Strings are cleaned wherever they appear: top
+ * level, nested inside plain objects and arrays (the `options` and
+ * `cost_tracking` JSON columns), and in object keys. Dates, Buffers and other
+ * class instances are returned by reference, so `created_at` survives intact.
+ */
+export function sanitizeLogData<T>(value: T): T {
+ if (typeof value === "string") {
+ return sanitizeText(value) as T;
+ }
+ if (value === null || typeof value !== "object") {
+ return value;
+ }
+ if (Array.isArray(value)) {
+ return value.map(item => sanitizeLogData(item)) as T;
+ }
+ const proto = Object.getPrototypeOf(value);
+ if (proto !== Object.prototype && proto !== null) {
+ return value;
+ }
+ // Object.fromEntries defines own properties, so a key named `__proto__`
+ // stays a field instead of replacing the prototype.
+ return Object.fromEntries(
+ Object.entries(value).map(([key, entry]) => [
+ sanitizeText(key),
+ sanitizeLogData(entry),
+ ]),
+ ) as T;
+}