Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Record the caller's External-Request-Id on the requests row

Rakshith Ramprakashcef06b55d43a
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 3cc6ec708..fe1bccb57 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -38,6 +38,7 @@ import {
resolveNewGroupBackend,
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
isThreatProtectionForced,
@@ -74,6 +75,7 @@ export async function crawlController(req: Request, res: Response) {
id,
kind: "crawl",
api_version: "v0",
+ external_request_id: externalRequestId(req),
team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 8ae9a1a7c..9d71790c1 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -25,6 +25,7 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq-router";
import { getErrorContactMessage } from "../../lib/deployment";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
isThreatProtectionForced,
@@ -219,6 +220,7 @@ export async function scrapeController(req: Request, res: Response) {
id: jobId,
kind: "scrape",
api_version: "v0",
+ external_request_id: externalRequestId(req),
team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 55bd5ee8b..9909986ba 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -4,6 +4,7 @@ import { autumnService } from "../../services/autumn/autumn.service";
import { authenticateUser } from "../auth";
import { RateLimiterMode, ScrapeJobSingleUrls } from "../../types";
import { logSearch, logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { PageOptions, SearchOptions } from "../../lib/entities";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
@@ -209,6 +210,7 @@ export async function searchController(req: Request, res: Response) {
id: jobId,
kind: "search",
api_version: "v0",
+ external_request_id: externalRequestId(req),
team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 1e69f7bce..d0369c9d8 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -43,6 +43,7 @@ import {
resolveNewGroupBackend,
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { CrawlDenialError } from "../../lib/error";
@@ -284,6 +285,7 @@ export async function batchScrapeController(
id,
kind: "batch_scrape",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 4557802d5..ec35bbfac 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -28,6 +28,7 @@ import {
resolveNewGroupBackend,
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { checkUrl } from "../../lib/threat-protection";
@@ -148,6 +149,7 @@ export async function crawlController(
id,
kind: "crawl",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts
index 49b2f8a1d..919126963 100644
--- a/apps/api/src/controllers/v1/deep-research.ts
+++ b/apps/api/src/controllers/v1/deep-research.ts
@@ -6,6 +6,7 @@ import * as Sentry from "@sentry/node";
import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";
import { z } from "zod";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
const deepResearchRequestSchema = z
@@ -98,6 +99,7 @@ export async function deepResearchController(
id: researchId,
kind: "deep_research",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: "api",
target_hint: req.body.query ?? "",
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 9b4fbf893..68538ca6d 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -20,6 +20,7 @@ import {
} from "../v2/types";
import { createWebhookSender, WebhookEvent } from "../../services/webhook";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { config } from "../../config";
@@ -248,6 +249,7 @@ export async function extractController(
id: extractId,
kind: "extract",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts
index c0c8f376f..ced86715d 100644
--- a/apps/api/src/controllers/v1/generate-llmstxt.ts
+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts
@@ -10,6 +10,7 @@ import { getGenerateLlmsTxtQueue } from "../../services/queue-service";
import * as Sentry from "@sentry/node";
import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
type GenerateLLMsTextResponse =
@@ -45,6 +46,7 @@ export async function generateLLMsTextController(
id: generationId,
kind: "llmstxt",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: "api", // no origin field for llmstxt
target_hint: req.body.url,
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 2b46f79ac..e973efaf5 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -20,6 +20,7 @@ import {
import { fireEngineMap } from "../../search/fireEngine";
import { billTeam } from "../../services/billing/credit_billing";
import { logMap, logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { performCosineSimilarity } from "../../lib/map-cosine";
import { logger } from "../../lib/logger";
import Redis from "ioredis";
@@ -424,6 +425,7 @@ export async function mapController(
id: mapId,
kind: "map",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index c5292a2b9..b022a9768 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -25,6 +25,7 @@ import { processJobInternal } from "../../services/worker/scrape-worker";
import { ScrapeJobData } from "../../types";
import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManager";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import { getScrapeZDR } from "../../lib/zdr-helpers";
@@ -115,6 +116,7 @@ export async function scrapeController(
id: jobId,
kind: "scrape",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin,
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index fea667c0e..7393461fd 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -10,6 +10,7 @@ import {
import { billTeam } from "../../services/billing/credit_billing";
import { v7 as uuidv7 } from "uuid";
import { logSearch, logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { search } from "../../search";
import { logger as _logger } from "../../lib/logger";
import {
@@ -174,6 +175,7 @@ export async function searchController(
id: jobId,
kind: "search",
api_version: "v1",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.ts
index 86ee7f346..2d2edbc63 100644
--- a/apps/api/src/controllers/v2/agent.ts
+++ b/apps/api/src/controllers/v2/agent.ts
@@ -8,6 +8,7 @@ import {
} from "./types";
import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { config } from "../../config";
import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";
import { getScrapeZDR } from "../../lib/zdr-helpers";
@@ -149,6 +150,7 @@ export async function agentController(
id: agentId,
kind: "agent",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 080cea700..978a58ae5 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -36,6 +36,7 @@ import {
resolveNewGroupBackend,
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
@@ -313,6 +314,7 @@ export async function batchScrapeController(
id,
kind: "batch_scrape",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index 31985d76f..f914d3c4f 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -27,6 +27,7 @@ import { RequestWithAuth } from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { integrationSchema } from "../../utils/integration";
import {
BROWSER_CREDITS_PER_HOUR,
@@ -379,6 +380,7 @@ export async function browserCreateController(
id: sessionId,
kind: "browser",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
target_hint: "Browser session",
origin: "api",
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 090aa94b2..eabd7234b 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -30,6 +30,7 @@ import {
resolveNewGroupBackend,
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { checkUrl } from "../../lib/threat-protection";
@@ -152,6 +153,7 @@ export async function crawlController(
id,
kind: "crawl",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index 5691cf498..5de299ca4 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -12,6 +12,7 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { config } from "../../config";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
@@ -179,6 +180,7 @@ export async function extractController(
id: extractId,
kind: "extract",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index e6a2165b5..164627146 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -8,6 +8,7 @@ import {
import { configDotenv } from "dotenv";
import { billTeam } from "../../services/billing/credit_billing";
import { logMap, logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { logger as _logger } from "../../lib/logger";
import { MapTimeoutError, MapFailedError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
@@ -81,6 +82,7 @@ export async function mapController(
id: mapId,
kind: "map",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 8ee5d7a00..a54f932ec 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -22,6 +22,7 @@ import { ScrapeJobData } from "../../types";
import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";
import { getJobPriority } from "../../lib/job-priority";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
@@ -395,6 +396,7 @@ export async function parseController(
id: jobId,
kind: "parse",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index fde48922f..459396af0 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -1,4 +1,5 @@
import express, { Request, Response } from "express";
+import { externalRequestId } from "../../lib/external-request-id";
import { z } from "zod";
import { v7 as uuidv7 } from "uuid";
import { logger as rootLogger } from "../../lib/logger";
@@ -286,6 +287,7 @@ function createResearchController(
id: jobId,
kind: endpoint.kind,
api_version: "v2",
+ external_request_id: externalRequestId(authedReq),
team_id: authedReq.auth.team_id,
origin: requestOrigin(params, req),
integration: params.integration ?? null,
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 90a3278e5..8d71d17a0 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -55,6 +55,7 @@ import {
} from "../../lib/keyless";
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { integrationSchema } from "../../utils/integration";
import { supabaseGetScrapeById } from "../../lib/supabase-jobs";
import {
@@ -814,6 +815,7 @@ async function createSessionForScrape(
id: sessionId,
kind: "interact",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
target_hint: "Interact session",
origin: req.body?.origin ?? "api",
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 0b8aaaa9d..3f2358d4f 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -25,6 +25,7 @@ import { ScrapeJobData } from "../../types";
import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";
import { getJobPriority } from "../../lib/job-priority";
import { logRequest } from "../../services/logging/log_job";
+import { externalRequestId } from "../../lib/external-request-id";
import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
@@ -221,6 +222,7 @@ export async function scrapeController(
id: jobId,
kind: "scrape",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 4370619d5..fef66830a 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { externalRequestId } from "../../lib/external-request-id";
import { config } from "../../config";
import {
RequestWithAuth,
@@ -186,6 +187,7 @@ export async function searchController(
id: jobId,
kind: "search",
api_version: "v2",
+ external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 97a17a381..812a370d9 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -560,6 +560,7 @@ export const requests = pgTable("requests", {
target_hint: text("target_hint").notNull(),
dr_clean_by: ts("dr_clean_by"),
api_key_id: bigintNum("api_key_id"),
+ external_request_id: text("external_request_id"),
});
export const mcp_action_logs = pgTable(
diff --git a/apps/api/src/lib/external-request-id.ts b/apps/api/src/lib/external-request-id.ts
new file mode 100644
index 000000000..74c84dba8
--- /dev/null
+++ b/apps/api/src/lib/external-request-id.ts
@@ -0,0 +1,50 @@
+import type { Request } from "express";
+import { logger } from "./logger";
+
+/**
+ * Longest External-Request-Id we will carry (bytes). Generous β€” an operation
+ * id is an id, not a document β€” and the ceiling exists so a caller bug cannot
+ * push arbitrary payloads into the requests table.
+ */
+export const EXTERNAL_REQUEST_ID_MAX_BYTES = 2048;
+
+/**
+ * The opaque per-operation id a caller attached to this request, if it sent
+ * one and it is usable.
+ *
+ * Some integrations stamp the requests they proxy with an
+ * `External-Request-Id` header so usage can be attributed back to their own
+ * operations. It is stored on the `requests` row at receipt (`logRequest`)
+ * and read back later by internal billing, keyed by the request id billing
+ * events already carry β€” it is deliberately NOT threaded through job data.
+ *
+ * Never a 400: a customer's request must not fail over telemetry. An id that
+ * is too long is dropped, loudly, and the request proceeds β€” the consequence
+ * is a missing attribution label, which is a reporting gap, not an outage.
+ *
+ * The value is opaque. No parsing, no interpretation, forwarded verbatim.
+ */
+export function externalRequestId(
+ req: Pick<Request, "headers">,
+): string | null {
+ const raw = req.headers["external-request-id"];
+ const first = Array.isArray(raw) ? raw[0] : raw;
+ if (typeof first !== "string") return null;
+
+ const value = first.trim();
+ if (value.length === 0) return null;
+
+ if (Buffer.byteLength(value) > EXTERNAL_REQUEST_ID_MAX_BYTES) {
+ logger.warn(
+ "External-Request-Id is too long; dropping it and serving the request anyway",
+ {
+ module: "external-request-id",
+ bytes: Buffer.byteLength(value),
+ max: EXTERNAL_REQUEST_ID_MAX_BYTES,
+ },
+ );
+ return null;
+ }
+
+ return value;
+}
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index 56ee7a336..cb70ecdfd 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -48,7 +48,11 @@ vi.mock("../../lib/extract/extract-redis", () => ({
saveExtractResult: vi.fn(),
}));
-import { logSearch, type LoggedSearch } from "./log_job";
+vi.mock("../posthog", () => ({
+ trackFirstSurfaceUse: vi.fn(),
+}));
+
+import { logRequest, logSearch, type LoggedSearch } from "./log_job";
import * as schema from "../../db/schema";
function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
@@ -118,3 +122,55 @@ describe("logSearch", () => {
expect(context.extra.data).toContain("badquery");
});
});
+
+describe("logRequest", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ values.mockResolvedValue(undefined);
+ });
+
+ function makeRequest(externalRequestId: string | null) {
+ return {
+ id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
+ kind: "scrape" as const,
+ api_version: "v2",
+ team_id: "team-id",
+ origin: "api",
+ target_hint: "https://example.com",
+ zeroDataRetention: false,
+ api_key_id: null,
+ external_request_id: externalRequestId,
+ };
+ }
+
+ it("stores the caller's external_request_id verbatim", async () => {
+ await logRequest(makeRequest("op_integration_42"));
+
+ expect(insert).toHaveBeenCalledWith(schema.requests);
+ expect(values.mock.calls[0][0].external_request_id).toBe("op_integration_42");
+ });
+
+ it("stores null, not a truncation, when the id exceeds the byte cap", async () => {
+ // The header helper already drops these; this asserts the bound holds at
+ // the insert boundary too, for any writer that bypasses the helper. Null
+ // rather than a DB constraint, which would fail the whole requests row
+ // (and its scrapes/crawls children) over a telemetry field β€” and null
+ // rather than truncation, which would hand a wrong id back downstream.
+ await logRequest(makeRequest("x".repeat(2049)));
+
+ const inserted = values.mock.calls[0][0];
+ expect(inserted.external_request_id).toBeNull();
+ expect(inserted.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");
+ expect(logger.warn).toHaveBeenCalled();
+ });
+
+ it("counts the cap in bytes, not characters", async () => {
+ // 1025 two-byte characters: 1025 chars, 2050 bytes β€” over.
+ await logRequest(makeRequest("Γ©".repeat(1025)));
+ expect(values.mock.calls[0][0].external_request_id).toBeNull();
+
+ // 1024 two-byte characters: 2048 bytes exactly β€” allowed.
+ await logRequest(makeRequest("Γ©".repeat(1024)));
+ expect(values.mock.calls[1][0].external_request_id).toBe("Γ©".repeat(1024));
+ });
+});
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 12787b1ac..fb85f5c83 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -4,6 +4,7 @@ import { changeTrackingInsertScrape } from "../../db/rpc";
import { config } from "../../config";
import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
+import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";
import { configDotenv } from "dotenv";
import * as Sentry from "@sentry/node";
import type { PgTable } from "drizzle-orm/pg-core";
@@ -186,8 +187,41 @@ type LoggedRequest = {
target_hint: string;
zeroDataRetention: boolean;
api_key_id?: number | null;
+ /**
+ * Opaque per-operation id a caller sent as `External-Request-Id` (see
+ * `lib/external-request-id.ts`), stored for internal billing attribution
+ * and read back off this row by the request id.
+ */
+ external_request_id?: string | null;
};
+/**
+ * The 2048-byte cap, re-checked at the one place the column is written.
+ *
+ * Belt and braces: the header helper (`lib/external-request-id.ts`) already
+ * drops oversized ids, but the bound must hold even for a future writer that
+ * bypasses it β€” and it cannot live in the database, where a length constraint
+ * would fail the whole `requests` insert (and the `scrapes`/`crawls` rows that
+ * FK into it) over a telemetry field. Oversized means null, never truncation:
+ * a truncated opaque id handed back downstream would be actively wrong, where
+ * an absent one is an honest reporting gap.
+ */
+function boundedExternalRequestId(
+ value: string | null,
+ logger: Logger,
+): string | null {
+ if (value === null) return null;
+ if (Buffer.byteLength(value) <= EXTERNAL_REQUEST_ID_MAX_BYTES) return value;
+ logger.warn(
+ "external_request_id exceeds the cap at the insert boundary; storing null",
+ {
+ bytes: Buffer.byteLength(value),
+ max: EXTERNAL_REQUEST_ID_MAX_BYTES,
+ },
+ );
+ return null;
+}
+
export async function logRequest(request: LoggedRequest) {
const logger = _logger.child({
module: "log_job",
@@ -235,6 +269,13 @@ export async function logRequest(request: LoggedRequest) {
? new Date(Date.now() + 24 * 60 * 60 * 1000)
: null,
api_key_id: request.api_key_id ?? null,
+ // Not redacted under zero data retention: it is the caller's own
+ // operation id (attribution it asked for), not customer content β€” and
+ // the row is cleaned at dr_clean_by regardless.
+ external_request_id: boundedExternalRequestId(
+ sanitizeString(request.external_request_id ?? null),
+ logger,
+ ),
},
true,
logger,