Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Record the caller's External-Request-Id on the requests row
Rakshith Ramprakashcef06b55d43a
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 3cc6ec708..fe1bccb57 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -38,6 +38,7 @@ import {resolveNewGroupBackend,} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import {isThreatProtectionForced,@@ -74,6 +75,7 @@ export async function crawlController(req: Request, res: Response) {id,kind: "crawl",api_version: "v0",+ external_request_id: externalRequestId(req),team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 8ae9a1a7c..9d71790c1 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -25,6 +25,7 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq-router";import { getErrorContactMessage } from "../../lib/deployment";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import {isThreatProtectionForced,@@ -219,6 +220,7 @@ export async function scrapeController(req: Request, res: Response) {id: jobId,kind: "scrape",api_version: "v0",+ external_request_id: externalRequestId(req),team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 55bd5ee8b..9909986ba 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -4,6 +4,7 @@ import { autumnService } from "../../services/autumn/autumn.service";import { authenticateUser } from "../auth";import { RateLimiterMode, ScrapeJobSingleUrls } from "../../types";import { logSearch, logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { PageOptions, SearchOptions } from "../../lib/entities";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";@@ -209,6 +210,7 @@ export async function searchController(req: Request, res: Response) {id: jobId,kind: "search",api_version: "v0",+ external_request_id: externalRequestId(req),team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 1e69f7bce..d0369c9d8 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -43,6 +43,7 @@ import {resolveNewGroupBackend,} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { CrawlDenialError } from "../../lib/error";@@ -284,6 +285,7 @@ export async function batchScrapeController(id,kind: "batch_scrape",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 4557802d5..ec35bbfac 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -28,6 +28,7 @@ import {resolveNewGroupBackend,} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { resolveThreatProtection } from "../../lib/threat-protection/request";import { checkUrl } from "../../lib/threat-protection";@@ -148,6 +149,7 @@ export async function crawlController(id,kind: "crawl",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 49b2f8a1d..919126963 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -6,6 +6,7 @@ import * as Sentry from "@sentry/node";import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";import { z } from "zod";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";const deepResearchRequestSchema = z@@ -98,6 +99,7 @@ export async function deepResearchController(id: researchId,kind: "deep_research",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: "api",target_hint: req.body.query ?? "",diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 9b4fbf893..68538ca6d 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -20,6 +20,7 @@ import {} from "../v2/types";import { createWebhookSender, WebhookEvent } from "../../services/webhook";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { config } from "../../config";@@ -248,6 +249,7 @@ export async function extractController(id: extractId,kind: "extract",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex c0c8f376f..ced86715d 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -10,6 +10,7 @@ import { getGenerateLlmsTxtQueue } from "../../services/queue-service";import * as Sentry from "@sentry/node";import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";type GenerateLLMsTextResponse =@@ -45,6 +46,7 @@ export async function generateLLMsTextController(id: generationId,kind: "llmstxt",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: "api", // no origin field for llmstxttarget_hint: req.body.url,diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 2b46f79ac..e973efaf5 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -20,6 +20,7 @@ import {import { fireEngineMap } from "../../search/fireEngine";import { billTeam } from "../../services/billing/credit_billing";import { logMap, logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { performCosineSimilarity } from "../../lib/map-cosine";import { logger } from "../../lib/logger";import Redis from "ioredis";@@ -424,6 +425,7 @@ export async function mapController(id: mapId,kind: "map",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex c5292a2b9..b022a9768 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -25,6 +25,7 @@ import { processJobInternal } from "../../services/worker/scrape-worker";import { ScrapeJobData } from "../../types";import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManager";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import { getScrapeZDR } from "../../lib/zdr-helpers";@@ -115,6 +116,7 @@ export async function scrapeController(id: jobId,kind: "scrape",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin,integration: req.body.integration,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex fea667c0e..7393461fd 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -10,6 +10,7 @@ import {import { billTeam } from "../../services/billing/credit_billing";import { v7 as uuidv7 } from "uuid";import { logSearch, logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { search } from "../../search";import { logger as _logger } from "../../lib/logger";import {@@ -174,6 +175,7 @@ export async function searchController(id: jobId,kind: "search",api_version: "v1",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 86ee7f346..2d2edbc63 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -8,6 +8,7 @@ import {} from "./types";import { logger as _logger } from "../../lib/logger";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { config } from "../../config";import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";import { getScrapeZDR } from "../../lib/zdr-helpers";@@ -149,6 +150,7 @@ export async function agentController(id: agentId,kind: "agent",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 080cea700..978a58ae5 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -36,6 +36,7 @@ import {resolveNewGroupBackend,} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";import {@@ -313,6 +314,7 @@ export async function batchScrapeController(id,kind: "batch_scrape",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex 31985d76f..f914d3c4f 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -27,6 +27,7 @@ import { RequestWithAuth } from "./types";import { billTeam } from "../../services/billing/credit_billing";import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { integrationSchema } from "../../utils/integration";import {BROWSER_CREDITS_PER_HOUR,@@ -379,6 +380,7 @@ export async function browserCreateController(id: sessionId,kind: "browser",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,target_hint: "Browser session",origin: "api",diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 090aa94b2..eabd7234b 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -30,6 +30,7 @@ import {resolveNewGroupBackend,} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { resolveThreatProtection } from "../../lib/threat-protection/request";import { checkUrl } from "../../lib/threat-protection";@@ -152,6 +153,7 @@ export async function crawlController(id,kind: "crawl",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 5691cf498..5de299ca4 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -12,6 +12,7 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { logger as _logger } from "../../lib/logger";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { config } from "../../config";import { getScrapeZDR } from "../../lib/zdr-helpers";import {@@ -179,6 +180,7 @@ export async function extractController(id: extractId,kind: "extract",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex e6a2165b5..164627146 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -8,6 +8,7 @@ import {import { configDotenv } from "dotenv";import { billTeam } from "../../services/billing/credit_billing";import { logMap, logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { logger as _logger } from "../../lib/logger";import { MapTimeoutError, MapFailedError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";@@ -81,6 +82,7 @@ export async function mapController(id: mapId,kind: "map",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 8ee5d7a00..a54f932ec 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -22,6 +22,7 @@ import { ScrapeJobData } from "../../types";import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";import { getJobPriority } from "../../lib/job-priority";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";@@ -395,6 +396,7 @@ export async function parseController(id: jobId,kind: "parse",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex fde48922f..459396af0 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -1,4 +1,5 @@import express, { Request, Response } from "express";+import { externalRequestId } from "../../lib/external-request-id";import { z } from "zod";import { v7 as uuidv7 } from "uuid";import { logger as rootLogger } from "../../lib/logger";@@ -286,6 +287,7 @@ function createResearchController(id: jobId,kind: endpoint.kind,api_version: "v2",+ external_request_id: externalRequestId(authedReq),team_id: authedReq.auth.team_id,origin: requestOrigin(params, req),integration: params.integration ?? null,diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 90a3278e5..8d71d17a0 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -55,6 +55,7 @@ import {} from "../../lib/keyless";import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { integrationSchema } from "../../utils/integration";import { supabaseGetScrapeById } from "../../lib/supabase-jobs";import {@@ -814,6 +815,7 @@ async function createSessionForScrape(id: sessionId,kind: "interact",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,target_hint: "Interact session",origin: req.body?.origin ?? "api",diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 0b8aaaa9d..3f2358d4f 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -25,6 +25,7 @@ import { ScrapeJobData } from "../../types";import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";import { getJobPriority } from "../../lib/job-priority";import { logRequest } from "../../services/logging/log_job";+import { externalRequestId } from "../../lib/external-request-id";import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";@@ -221,6 +222,7 @@ export async function scrapeController(id: jobId,kind: "scrape",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 4370619d5..fef66830a 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { externalRequestId } from "../../lib/external-request-id";import { config } from "../../config";import {RequestWithAuth,@@ -186,6 +187,7 @@ export async function searchController(id: jobId,kind: "search",api_version: "v2",+ external_request_id: externalRequestId(req),team_id: req.auth.team_id,origin: req.body.origin ?? "api",integration: req.body.integration,diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 97a17a381..812a370d9 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -560,6 +560,7 @@ export const requests = pgTable("requests", {target_hint: text("target_hint").notNull(),dr_clean_by: ts("dr_clean_by"),api_key_id: bigintNum("api_key_id"),+ external_request_id: text("external_request_id"),});export const mcp_action_logs = pgTable(diff --git a/apps/api/src/lib/external-request-id.ts b/apps/api/src/lib/external-request-id.tsnew file mode 100644index 000000000..74c84dba8--- /dev/null+++ b/apps/api/src/lib/external-request-id.ts@@ -0,0 +1,50 @@+import type { Request } from "express";+import { logger } from "./logger";++/**+ * Longest External-Request-Id we will carry (bytes). Generous β an operation+ * id is an id, not a document β and the ceiling exists so a caller bug cannot+ * push arbitrary payloads into the requests table.+ */+export const EXTERNAL_REQUEST_ID_MAX_BYTES = 2048;++/**+ * The opaque per-operation id a caller attached to this request, if it sent+ * one and it is usable.+ *+ * Some integrations stamp the requests they proxy with an+ * `External-Request-Id` header so usage can be attributed back to their own+ * operations. It is stored on the `requests` row at receipt (`logRequest`)+ * and read back later by internal billing, keyed by the request id billing+ * events already carry β it is deliberately NOT threaded through job data.+ *+ * Never a 400: a customer's request must not fail over telemetry. An id that+ * is too long is dropped, loudly, and the request proceeds β the consequence+ * is a missing attribution label, which is a reporting gap, not an outage.+ *+ * The value is opaque. No parsing, no interpretation, forwarded verbatim.+ */+export function externalRequestId(+ req: Pick<Request, "headers">,+): string | null {+ const raw = req.headers["external-request-id"];+ const first = Array.isArray(raw) ? raw[0] : raw;+ if (typeof first !== "string") return null;++ const value = first.trim();+ if (value.length === 0) return null;++ if (Buffer.byteLength(value) > EXTERNAL_REQUEST_ID_MAX_BYTES) {+ logger.warn(+ "External-Request-Id is too long; dropping it and serving the request anyway",+ {+ module: "external-request-id",+ bytes: Buffer.byteLength(value),+ max: EXTERNAL_REQUEST_ID_MAX_BYTES,+ },+ );+ return null;+ }++ return value;+}diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 56ee7a336..cb70ecdfd 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -48,7 +48,11 @@ vi.mock("../../lib/extract/extract-redis", () => ({saveExtractResult: vi.fn(),}));-import { logSearch, type LoggedSearch } from "./log_job";+vi.mock("../posthog", () => ({+ trackFirstSurfaceUse: vi.fn(),+}));++import { logRequest, logSearch, type LoggedSearch } from "./log_job";import * as schema from "../../db/schema";function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {@@ -118,3 +122,55 @@ describe("logSearch", () => {expect(context.extra.data).toContain("badquery");});});++describe("logRequest", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ values.mockResolvedValue(undefined);+ });++ function makeRequest(externalRequestId: string | null) {+ return {+ id: "019e6f45-7778-727d-adf0-0abe9d5062b6",+ kind: "scrape" as const,+ api_version: "v2",+ team_id: "team-id",+ origin: "api",+ target_hint: "https://example.com",+ zeroDataRetention: false,+ api_key_id: null,+ external_request_id: externalRequestId,+ };+ }++ it("stores the caller's external_request_id verbatim", async () => {+ await logRequest(makeRequest("op_integration_42"));++ expect(insert).toHaveBeenCalledWith(schema.requests);+ expect(values.mock.calls[0][0].external_request_id).toBe("op_integration_42");+ });++ it("stores null, not a truncation, when the id exceeds the byte cap", async () => {+ // The header helper already drops these; this asserts the bound holds at+ // the insert boundary too, for any writer that bypasses the helper. Null+ // rather than a DB constraint, which would fail the whole requests row+ // (and its scrapes/crawls children) over a telemetry field β and null+ // rather than truncation, which would hand a wrong id back downstream.+ await logRequest(makeRequest("x".repeat(2049)));++ const inserted = values.mock.calls[0][0];+ expect(inserted.external_request_id).toBeNull();+ expect(inserted.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");+ expect(logger.warn).toHaveBeenCalled();+ });++ it("counts the cap in bytes, not characters", async () => {+ // 1025 two-byte characters: 1025 chars, 2050 bytes β over.+ await logRequest(makeRequest("Γ©".repeat(1025)));+ expect(values.mock.calls[0][0].external_request_id).toBeNull();++ // 1024 two-byte characters: 2048 bytes exactly β allowed.+ await logRequest(makeRequest("Γ©".repeat(1024)));+ expect(values.mock.calls[1][0].external_request_id).toBe("Γ©".repeat(1024));+ });+});diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 12787b1ac..fb85f5c83 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -4,6 +4,7 @@ import { changeTrackingInsertScrape } from "../../db/rpc";import { config } from "../../config";import "dotenv/config";import { logger as _logger } from "../../lib/logger";+import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";import { configDotenv } from "dotenv";import * as Sentry from "@sentry/node";import type { PgTable } from "drizzle-orm/pg-core";@@ -186,8 +187,41 @@ type LoggedRequest = {target_hint: string;zeroDataRetention: boolean;api_key_id?: number | null;+ /**+ * Opaque per-operation id a caller sent as `External-Request-Id` (see+ * `lib/external-request-id.ts`), stored for internal billing attribution+ * and read back off this row by the request id.+ */+ external_request_id?: string | null;};+/**+ * The 2048-byte cap, re-checked at the one place the column is written.+ *+ * Belt and braces: the header helper (`lib/external-request-id.ts`) already+ * drops oversized ids, but the bound must hold even for a future writer that+ * bypasses it β and it cannot live in the database, where a length constraint+ * would fail the whole `requests` insert (and the `scrapes`/`crawls` rows that+ * FK into it) over a telemetry field. Oversized means null, never truncation:+ * a truncated opaque id handed back downstream would be actively wrong, where+ * an absent one is an honest reporting gap.+ */+function boundedExternalRequestId(+ value: string | null,+ logger: Logger,+): string | null {+ if (value === null) return null;+ if (Buffer.byteLength(value) <= EXTERNAL_REQUEST_ID_MAX_BYTES) return value;+ logger.warn(+ "external_request_id exceeds the cap at the insert boundary; storing null",+ {+ bytes: Buffer.byteLength(value),+ max: EXTERNAL_REQUEST_ID_MAX_BYTES,+ },+ );+ return null;+}+export async function logRequest(request: LoggedRequest) {const logger = _logger.child({module: "log_job",@@ -235,6 +269,13 @@ export async function logRequest(request: LoggedRequest) {? new Date(Date.now() + 24 * 60 * 60 * 1000): null,api_key_id: request.api_key_id ?? null,+ // Not redacted under zero data retention: it is the caller's own+ // operation id (attribution it asked for), not customer content β and+ // the row is cleaned at dr_clean_by regardless.+ external_request_id: boundedExternalRequestId(+ sanitizeString(request.external_request_id ?? null),+ logger,+ ),},true,logger,