Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add Bigtable operational job stores (#4645)
Gergő Móricz847a87f8a57e
diff --git a/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts b/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.tsindex 0707d4af7..fbeebe3bf 100644--- a/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts+++ b/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts@@ -1,5 +1,20 @@import type { Mock } from "vitest";import { redisEvictConnection } from "../../../services/redis";++const { writeApiJobAccess, redisMock } = vi.hoisted(() => ({+ writeApiJobAccess: vi.fn(async () => true),+ redisMock: {+ set: vi.fn(async () => "OK"),+ get: vi.fn(),+ pttl: vi.fn(),+ },+}));++vi.mock("../../../lib/job-access-store", () => ({ writeApiJobAccess }));+vi.mock("../../../services/redis", () => ({+ redisEvictConnection: redisMock,+}));+import {saveDeepResearch,getDeepResearch,@@ -8,15 +23,6 @@ import {StoredDeepResearch,} from "../../../lib/deep-research/deep-research-redis";-vi.mock("../../../services/queue-service", () => ({- redisConnection: {- set: vi.fn(),- get: vi.fn(),- expire: vi.fn(),- pttl: vi.fn(),- },-}));-describe("Deep Research Redis Operations", () => {const mockResearch: StoredDeepResearch = {id: "test-id",@@ -35,6 +41,7 @@ describe("Deep Research Redis Operations", () => {beforeEach(() => {vi.clearAllMocks();+ writeApiJobAccess.mockResolvedValue(true);});describe("saveDeepResearch", () => {@@ -44,11 +51,27 @@ describe("Deep Research Redis Operations", () => {expect(redisEvictConnection.set).toHaveBeenCalledWith("deep-research:test-id",JSON.stringify(mockResearch),- );- expect(redisEvictConnection.expire).toHaveBeenCalledWith(- "deep-research:test-id",+ "EX",6 * 60 * 60,);+ expect(writeApiJobAccess).toHaveBeenCalledWith({+ id: "test-id",+ teamId: "team-1",+ kind: "deep_research",+ expiresAt: expect.any(Date),+ });+ });++ it("keeps the Redis write when the access refresh fails", async () => {+ writeApiJobAccess.mockRejectedValueOnce(+ new Error("Bigtable unavailable"),+ );++ await expect(+ saveDeepResearch("test-id", mockResearch),+ ).resolves.toBeUndefined();++ expect(redisEvictConnection.set).toHaveBeenCalledOnce();});});@@ -104,11 +127,15 @@ describe("Deep Research Redis Operations", () => {expect(redisEvictConnection.set).toHaveBeenCalledWith("deep-research:test-id",JSON.stringify(expectedUpdate),- );- expect(redisEvictConnection.expire).toHaveBeenCalledWith(- "deep-research:test-id",+ "EX",6 * 60 * 60,);+ expect(writeApiJobAccess).toHaveBeenCalledWith({+ id: "test-id",+ teamId: "team-1",+ kind: "deep_research",+ expiresAt: expect.any(Date),+ });});it("should do nothing if research not found", async () => {@@ -117,7 +144,7 @@ describe("Deep Research Redis Operations", () => {await updateDeepResearch("test-id", { status: "completed" });expect(redisEvictConnection.set).not.toHaveBeenCalled();- expect(redisEvictConnection.expire).not.toHaveBeenCalled();+ expect(writeApiJobAccess).not.toHaveBeenCalled();});});@@ -126,13 +153,12 @@ describe("Deep Research Redis Operations", () => {const mockTTL = 3600000; // 1 hour in milliseconds(redisEvictConnection.pttl as Mock).mockResolvedValue(mockTTL);+ const before = Date.now();const result = await getDeepResearchExpiry("test-id");expect(result).toBeInstanceOf(Date);- expect(result.getTime()).toBeCloseTo(- new Date().getTime() + mockTTL,- -2, // Allow 100ms precision- );+ expect(result.getTime()).toBeGreaterThanOrEqual(before + mockTTL - 999);+ expect(result.getTime()).toBeLessThanOrEqual(Date.now() + mockTTL);});});});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex e798dee2b..dcff8d868 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -275,6 +275,8 @@ const configSchema = z.object({BIGTABLE_INSTANCE_ID: z.string().optional(),BIGTABLE_APP_PROFILE_ID: z.string().optional(),BIGTABLE_CHANGE_TRACKING_TABLE: z.string().optional(),+ BIGTABLE_JOB_ACCESS_TABLE: z.string().optional(),+ BIGTABLE_FEEDBACK_JOBS_TABLE: z.string().optional(),BIGTABLE_CREDENTIALS: z.string().optional(),// ClickHouse (Search Analytics)diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 5227beabd..06acd2184 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -91,6 +91,9 @@ export async function crawlController(req: Request, res: Response) {target_hint: req.body.url ?? "",zeroDataRetention: false, // not supported on v0api_key_id: chunk?.api_key_id ?? null,+ jobAccessExpiresAt: new Date(+ Date.now() + (chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ ),});redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 9f4dbfe3d..2a65685dc 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -316,6 +316,9 @@ export async function batchScrapeController(target_hint: urls[0] ?? "",zeroDataRetention: zeroDataRetention || false,api_key_id: req.acuc?.api_key_id ?? null,+ jobAccessExpiresAt: new Date(+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ ),});}diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex a494aaecb..7432f50b3 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -192,6 +192,9 @@ export async function crawlController(target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,api_key_id: req.acuc?.api_key_id ?? null,+ jobAccessExpiresAt: new Date(+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ ),});// checkCreditsMiddleware (always runs before this controller) is the sourcediff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex b6faff9a6..ef2aabffd 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -2,7 +2,10 @@ import { v7 as uuidv7 } from "uuid";import { Request, Response } from "express";import { ErrorResponse, extractOptions, RequestWithAuth } from "./types";import { getDeepResearchQueue } from "../../services/queue-service";-import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";+import {+ DEEP_RESEARCH_TTL,+ saveDeepResearch,+} from "../../lib/deep-research/deep-research-redis";import { z } from "zod";import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";@@ -104,6 +107,7 @@ export async function deepResearchController(target_hint: req.body.query ?? "",zeroDataRetention: false, // not supported for deep researchapi_key_id: req.acuc?.api_key_id ?? null,+ jobAccessExpiresAt: new Date(Date.now() + DEEP_RESEARCH_TTL * 1000),});const jobData = {diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 8aeedcdda..0668c19cd 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -345,6 +345,9 @@ export async function batchScrapeController(target_hint: urls[0] ?? "",zeroDataRetention: zeroDataRetention || false,api_key_id: req.acuc?.api_key_id ?? null,+ jobAccessExpiresAt: new Date(+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ ),});}diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 4c051a8d0..6d2fcb227 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -197,6 +197,9 @@ export async function crawlController(target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,api_key_id: req.acuc?.api_key_id ?? null,+ jobAccessExpiresAt: new Date(+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ ),});// checkCreditsMiddleware (always runs before this controller) is the sourcediff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.tsindex 2d67c2dea..0334d084e 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts@@ -93,6 +93,9 @@ it("returns the Scrape contract, shares identity with the legacy route, and logsscrapeId: undefined,});expect(mocks.log).toHaveBeenCalledTimes(1);+ expect(mocks.log).toHaveBeenCalledWith(+ expect.objectContaining({ kind: "scrape", jobAccess: false }),+ );});it("relays a failed single legacy call as an error, not a success", async () => {diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex 1cfb6ec57..29b207995 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -153,6 +153,7 @@ export async function providerScrapeController(integration: body.integration ?? null,target_hint: `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`,zeroDataRetention: false,+ jobAccess: false,}).catch(error =>logger.warn("Provider request logging failed", {error,diff --git a/apps/api/src/lib/bigtable-client.ts b/apps/api/src/lib/bigtable-client.tsnew file mode 100644index 000000000..dc35f9998--- /dev/null+++ b/apps/api/src/lib/bigtable-client.ts@@ -0,0 +1,44 @@+import type { Bigtable, Table } from "@google-cloud/bigtable";+import { config } from "../config";+import { setSpanAttributes, withSpan } from "./otel-tracer";++let clientPromise: Promise<Bigtable> | null = null;+const tables = new Map<string, Table>();++export async function getBigtableTable(tableId: string): Promise<Table> {+ return withSpan("bigtable.table.resolve", async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": tableId,+ "bigtable.client.cached": clientPromise !== null,+ "bigtable.table.cached": tables.has(tableId),+ });+ if (!config.BIGTABLE_INSTANCE_ID) {+ throw new Error("BIGTABLE_INSTANCE_ID is not configured");+ }++ clientPromise ??= import("@google-cloud/bigtable").then(+ ({ Bigtable }) =>+ new Bigtable({+ projectId: config.BIGTABLE_PROJECT_ID,+ ...(config.BIGTABLE_APP_PROFILE_ID+ ? { appProfileId: config.BIGTABLE_APP_PROFILE_ID }+ : {}),+ ...(config.BIGTABLE_CREDENTIALS+ ? {+ credentials: JSON.parse(atob(config.BIGTABLE_CREDENTIALS)),+ }+ : {}),+ metricsEnabled: false,+ }),+ );+ const client = await clientPromise;++ let table = tables.get(tableId);+ if (!table) {+ table = client.instance(config.BIGTABLE_INSTANCE_ID).table(tableId);+ tables.set(tableId, table);+ }+ return table;+ });+}diff --git a/apps/api/src/lib/bigtable-row-key.ts b/apps/api/src/lib/bigtable-row-key.tsnew file mode 100644index 000000000..fa25f9a21--- /dev/null+++ b/apps/api/src/lib/bigtable-row-key.ts@@ -0,0 +1,19 @@+import crypto from "crypto";++const UUID_V7 =+ /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;++export function saltedUuidV7RowKey(id: string): string {+ if (!UUID_V7.test(id)) {+ throw new Error(`Expected a UUIDv7 row id, received ${id}`);+ }++ const uuid = Buffer.from(id.replaceAll("-", ""), "hex");+ const salt = crypto+ .createHash("sha256")+ .update(uuid)+ .digest()+ .subarray(0, 1)+ .toString("hex");+ return salt + uuid.toString("base64url");+}diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.tsindex a011e1206..f4b770a73 100644--- a/apps/api/src/lib/change-tracking-store.ts+++ b/apps/api/src/lib/change-tracking-store.ts@@ -1,7 +1,8 @@-import type { Bigtable, Table } from "@google-cloud/bigtable";import crypto from "crypto";import { config } from "../config";import { diffGetLastScrape } from "../db/rpc";+import { getBigtableTable } from "./bigtable-client";+import { setSpanAttributes, withSpan } from "./otel-tracer";// Change tracking bookkeeping. Final architecture (#4484 was the// transition): Bigtable is the store. Writes go to Bigtable only --@@ -58,48 +59,6 @@ function changeTrackingRowKey(]);}-let bigtableClient: Bigtable | null = null;-let table: Table | null = null;--// The Bigtable package is loaded lazily so a broken or slow package load-// surfaces as a caught error in the callers, not an import-time crash.-async function getChangeTrackingTable(): Promise<Table> {- if (!bigtableClient) {- const { Bigtable } = await import("@google-cloud/bigtable");- bigtableClient = new Bigtable({- projectId: config.BIGTABLE_PROJECT_ID,- ...(config.BIGTABLE_APP_PROFILE_ID- ? { appProfileId: config.BIGTABLE_APP_PROFILE_ID }- : {}),- // Mirrors GCS_CREDENTIALS: base64-encoded service-account JSON.- // Parsed here (not at module load) so a malformed value fails as a- // caught error, never an import-time crash. Unset falls back to- // Application Default Credentials.- ...(config.BIGTABLE_CREDENTIALS- ? {- credentials: JSON.parse(atob(config.BIGTABLE_CREDENTIALS)),- }- : {}),- // The client's Cloud Monitoring metrics handler requires the- // OTel 1.x line, which GHSA-8988-4f7v-96qf only patches on 2.x;- // we don't consume client-side metrics, so disable the handler- // and let the dependency overrides move the tree to 2.x.- metricsEnabled: false,- });- }- if (!table) {- if (!config.BIGTABLE_INSTANCE_ID) {- throw new Error(- "BIGTABLE_INSTANCE_ID is not configured; change tracking requires the Bigtable store",- );- }- table = bigtableClient- .instance(config.BIGTABLE_INSTANCE_ID)- .table(TABLE_ID);- }- return table;-}-export async function changeTrackingInsertScrape(params: {team_id: string;url: string;@@ -108,29 +67,36 @@ export async function changeTrackingInsertScrape(params: {/** When the scrape was logged; becomes the cell timestamp. */date_added: Date;}): Promise<void> {- const table = await getChangeTrackingTable();- // Runtime accepts Buffer keys (converted verbatim); the .d.ts only- // declares string.- await table.mutate([- {- key: changeTrackingRowKey(- params.team_id,- params.url,- params.tag,- ) as unknown as string,- // Required by Mutation.parse -- without it the entry carries no- // setCell mutations.- method: "insert" as const,- data: {- [FAMILY]: {- [QUALIFIER]: {- value: params.job_id,- timestamp: params.date_added,+ await withSpan("bigtable.change_tracking.write", async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": TABLE_ID,+ "bigtable.operation": "mutate",+ });+ const table = await getBigtableTable(TABLE_ID);+ // Runtime accepts Buffer keys (converted verbatim); the .d.ts only+ // declares string.+ await table.mutate([+ {+ key: changeTrackingRowKey(+ params.team_id,+ params.url,+ params.tag,+ ) as unknown as string,+ // Required by Mutation.parse -- without it the entry carries no+ // setCell mutations.+ method: "insert" as const,+ data: {+ [FAMILY]: {+ [QUALIFIER]: {+ value: params.job_id,+ timestamp: params.date_added,+ },},},},- },- ]);+ ]);+ });}/**@@ -144,13 +110,24 @@ export async function changeTrackingGetLastScrape(params: {url: string;tag: string | null;}): Promise<{ job_id: string; date_added: string } | null> {- const table = await getChangeTrackingTable();+ const table = await getBigtableTable(TABLE_ID);const key = changeTrackingRowKey(params.team_id, params.url, params.tag);- const [rows] = await table.getRows({- keys: [key as unknown as string],- // Qualifier regex + latest-version-only, expressed via the column- // filter's cellLimit (there is no standalone `versions` filter key).- filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],+ const rows = await withSpan("bigtable.change_tracking.read", async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": TABLE_ID,+ "bigtable.operation": "getRows",+ });+ const [result] = await table.getRows({+ keys: [key as unknown as string],+ // Qualifier regex + latest-version-only, expressed via the column+ // filter's cellLimit (there is no standalone `versions` filter key).+ filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],+ });+ setSpanAttributes(span, {+ "bigtable.rows_returned": result.length,+ });+ return result;});const row = rows[0];if (row) {diff --git a/apps/api/src/lib/deep-research/deep-research-redis.ts b/apps/api/src/lib/deep-research/deep-research-redis.tsindex d9d4e8d82..bb45e6dc9 100644--- a/apps/api/src/lib/deep-research/deep-research-redis.ts+++ b/apps/api/src/lib/deep-research/deep-research-redis.ts@@ -1,4 +1,5 @@import { redisEvictConnection } from "../../services/redis";+import { writeApiJobAccess } from "../job-access-store";import { logger as _logger } from "../logger";export type DeepResearchActivity = {@@ -45,19 +46,41 @@ export type StoredDeepResearch = {};// TTL of 6 hours-const DEEP_RESEARCH_TTL = 6 * 60 * 60;+export const DEEP_RESEARCH_TTL = 6 * 60 * 60;++async function refreshDeepResearchAccess(+ id: string,+ teamId: string,+ expiresAt: Date,+): Promise<void> {+ try {+ await writeApiJobAccess({+ id,+ teamId,+ kind: "deep_research",+ expiresAt,+ });+ } catch (error) {+ _logger.error("Failed to refresh deep research access in Bigtable", {+ error,+ researchId: id,+ });+ }+}export async function saveDeepResearch(id: string,research: StoredDeepResearch,) {_logger.debug("Saving deep research " + id + " to Redis...");+ const expiresAt = new Date(Date.now() + DEEP_RESEARCH_TTL * 1000);await redisEvictConnection.set("deep-research:" + id,JSON.stringify(research),"EX",DEEP_RESEARCH_TTL,);+ await refreshDeepResearchAccess(id, research.team_id, expiresAt);}export async function getDeepResearch(@@ -95,12 +118,14 @@ export async function updateDeepResearch(: current.summaries,};+ const expiresAt = new Date(Date.now() + DEEP_RESEARCH_TTL * 1000);await redisEvictConnection.set("deep-research:" + id,JSON.stringify(updatedResearch),"EX",DEEP_RESEARCH_TTL,);+ await refreshDeepResearchAccess(id, current.team_id, expiresAt);}export async function getDeepResearchExpiry(id: string): Promise<Date> {diff --git a/apps/api/src/lib/feedback-job-store.ts b/apps/api/src/lib/feedback-job-store.tsnew file mode 100644index 000000000..756287b05--- /dev/null+++ b/apps/api/src/lib/feedback-job-store.ts@@ -0,0 +1,114 @@+import { config } from "../config";+import type { ScrapeOptions } from "../controllers/v2/types";+import { includesFormat } from "./format-utils";+import { getBigtableTable } from "./bigtable-client";+import { saltedUuidV7RowKey } from "./bigtable-row-key";+import { setSpanAttributes, withSpan } from "./otel-tracer";++const FAMILY = "f";+const QUALIFIER = "v";++export type FeedbackEndpoint = "search" | "scrape" | "parse" | "map";+export type RefundClass =+ | "search"+ | "map"+ | "parse"+ | "scrape_basic"+ | "scrape_pdf"+ | "scrape_json"+ | "scrape_addon";++function scrapeRefundClass(options: ScrapeOptions): RefundClass {+ if (+ options.parsers?.some(+ parser =>+ parser === "pdf" ||+ (typeof parser === "object" && parser.type === "pdf"),+ )+ ) {+ return "scrape_pdf";+ }+ if (includesFormat(options.formats, "json")) return "scrape_json";+ if (+ includesFormat(options.formats, "screenshot") ||+ (options.actions?.length ?? 0) > 0+ ) {+ return "scrape_addon";+ }+ return "scrape_basic";+}++type FeedbackJobBase = {+ jobId: string;+ requestId: string;+ teamId: string;+ succeeded: boolean;+ creditsBilled: number;+ zeroDataRetention: boolean;+ completedAt?: Date;+};++export type FeedbackJobWrite = FeedbackJobBase &+ (+ | { endpoint: "scrape"; scrapeOptions: ScrapeOptions }+ | { endpoint: Exclude<FeedbackEndpoint, "scrape"> }+ );++export async function writeFeedbackJob(+ params: FeedbackJobWrite,+): Promise<boolean> {+ const tableId = config.BIGTABLE_FEEDBACK_JOBS_TABLE;+ if (!tableId) return false;++ return withSpan(+ "bigtable.feedback_job.write",+ async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": tableId,+ "bigtable.operation": "mutate",+ "feedback.endpoint": params.endpoint,+ });+ const completedAt = params.completedAt ?? new Date();+ const feedbackWindowSec =+ params.endpoint === "search"+ ? config.SEARCH_FEEDBACK_MAX_AGE_SEC+ : config.FEEDBACK_MAX_AGE_SEC;+ const feedbackDeadline = new Date(+ completedAt.getTime() + feedbackWindowSec * 1000,+ );+ const value = Buffer.from(+ JSON.stringify({+ version: 1,+ requestId: params.requestId,+ teamId: params.teamId,+ refundClass:+ params.endpoint === "scrape"+ ? scrapeRefundClass(params.scrapeOptions)+ : params.endpoint,+ feedbackDeadlineMs: feedbackDeadline.getTime(),+ succeeded: params.succeeded,+ creditsBilled: params.creditsBilled,+ zeroDataRetention: params.zeroDataRetention,+ }),+ );+ const table = await getBigtableTable(tableId);+ await table.mutate([+ {+ key: saltedUuidV7RowKey(params.jobId),+ method: "insert",+ data: {+ [FAMILY]: {+ [QUALIFIER]: {+ value,+ timestamp: feedbackDeadline,+ },+ },+ },+ },+ ]);+ return true;+ },+ { zeroDataRetention: params.zeroDataRetention },+ );+}diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.tsnew file mode 100644index 000000000..b514b2bb7--- /dev/null+++ b/apps/api/src/lib/job-access-store.ts@@ -0,0 +1,75 @@+import { config } from "../config";+import { getBigtableTable } from "./bigtable-client";+import { saltedUuidV7RowKey } from "./bigtable-row-key";+import { setSpanAttributes, withSpan } from "./otel-tracer";++const FAMILY = "j";+const QUALIFIER = "v";++export const API_JOB_KINDS = [+ "scrape",+ "crawl",+ "batch_scrape",+ "extract",+ "agent",+ "llmstxt",+ "deep_research",+] as const;++export type ApiJobKind = (typeof API_JOB_KINDS)[number];++export function isApiJobKind(kind: string): kind is ApiJobKind {+ return API_JOB_KINDS.includes(kind as ApiJobKind);+}++export async function writeApiJobAccess(params: {+ id: string;+ teamId: string;+ kind: ApiJobKind;+ expiresAt: Date;+ clientOrigin?: string | null;+ zeroDataRetention?: boolean;+}): Promise<boolean> {+ const tableId = config.BIGTABLE_JOB_ACCESS_TABLE;+ if (!tableId) return false;++ return withSpan(+ "bigtable.job_access.write",+ async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": tableId,+ "bigtable.operation": "mutate",+ "job_access.kind": params.kind,+ });+ const value = Buffer.from(+ JSON.stringify({+ version: 1,+ teamId: params.teamId,+ kind: params.kind,+ expiresAtMs: params.expiresAt.getTime(),+ ...(params.kind === "agent" && params.clientOrigin+ ? { clientOrigin: params.clientOrigin }+ : {}),+ }),+ );+ const table = await getBigtableTable(tableId);+ await table.mutate([+ {+ key: saltedUuidV7RowKey(params.id),+ method: "insert",+ data: {+ [FAMILY]: {+ [QUALIFIER]: {+ value,+ timestamp: params.expiresAt,+ },+ },+ },+ },+ ]);+ return true;+ },+ { zeroDataRetention: params.zeroDataRetention },+ );+}diff --git a/apps/api/src/lib/job-operational-stores.test.ts b/apps/api/src/lib/job-operational-stores.test.tsnew file mode 100644index 000000000..92b5a8a74--- /dev/null+++ b/apps/api/src/lib/job-operational-stores.test.ts@@ -0,0 +1,216 @@+import { vi } from "vitest";+import type { MutationConstructorObj } from "@google-cloud/bigtable";++const { mutate, getBigtableTable, mutableConfig, withSpan, spans } = vi.hoisted(+ () => {+ const spans: { name: string; options: any }[] = [];+ return {+ mutate: vi.fn<(mutations: MutationConstructorObj[]) => Promise<void>>(+ async () => {},+ ),+ getBigtableTable: vi.fn(),+ mutableConfig: {+ BIGTABLE_JOB_ACCESS_TABLE: "job-access",+ BIGTABLE_FEEDBACK_JOBS_TABLE: "feedback-jobs",+ SEARCH_FEEDBACK_MAX_AGE_SEC: 120,+ FEEDBACK_MAX_AGE_SEC: 180,+ } as {+ BIGTABLE_JOB_ACCESS_TABLE?: string;+ BIGTABLE_FEEDBACK_JOBS_TABLE?: string;+ SEARCH_FEEDBACK_MAX_AGE_SEC: number;+ FEEDBACK_MAX_AGE_SEC: number;+ },+ withSpan: vi.fn(+ async (name: string, fn: (span: any) => any, options?: any) => {+ spans.push({ name, options });+ return fn({});+ },+ ),+ spans,+ };+ },+);++vi.mock("../config", () => ({ config: mutableConfig }));+vi.mock("./bigtable-client", () => ({ getBigtableTable }));+vi.mock("./otel-tracer", () => ({+ withSpan,+ setSpanAttributes: vi.fn(),+}));++import {+ API_JOB_KINDS,+ isApiJobKind,+ writeApiJobAccess,+} from "./job-access-store";+import { writeFeedbackJob } from "./feedback-job-store";+import { saltedUuidV7RowKey } from "./bigtable-row-key";+import { scrapeOptions } from "../controllers/v2/types";++const JOB_ID = "019e6f45-7778-727d-adf0-0abe9d5062b6";+const REQUEST_ID = "019e6f45-7778-727d-adf0-0abe9d5062b7";++function writtenValue(): Record<string, string | number | boolean> {+ const mutation = mutate.mock.calls[0][0][0];+ return JSON.parse(+ mutation.data.j?.v?.value ?? mutation.data.f.v.value,+ ) as Record<string, string | number | boolean>;+}++describe("operational Bigtable stores", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ getBigtableTable.mockResolvedValue({ mutate });+ mutableConfig.BIGTABLE_JOB_ACCESS_TABLE = "job-access";+ mutableConfig.BIGTABLE_FEEDBACK_JOBS_TABLE = "feedback-jobs";+ spans.length = 0;+ });++ it("salts UUIDv7 row keys while preserving compact binary ids", () => {+ const key = saltedUuidV7RowKey(JOB_ID);+ expect(key).toHaveLength(24);+ expect(key.slice(0, 2)).toMatch(/^[0-9a-f]{2}$/);+ expect(Buffer.from(key.slice(2), "base64url").toString("hex")).toBe(+ JOB_ID.replaceAll("-", ""),+ );+ expect(() =>+ saltedUuidV7RowKey("550e8400-e29b-41d4-a716-446655440000"),+ ).toThrow("Expected a UUIDv7 row id");+ });++ it("writes an acknowledged job access row", async () => {+ const expiresAt = new Date("2026-09-16T12:00:00.000Z");+ await expect(+ writeApiJobAccess({+ id: JOB_ID,+ teamId: "team-id",+ kind: "agent",+ expiresAt,+ clientOrigin: "[email protected]",+ }),+ ).resolves.toBe(true);++ expect(getBigtableTable).toHaveBeenCalledWith("job-access");+ expect(writtenValue()).toEqual({+ version: 1,+ teamId: "team-id",+ kind: "agent",+ expiresAtMs: expiresAt.getTime(),+ clientOrigin: "[email protected]",+ });+ expect(mutate.mock.calls[0][0][0].data.j.v.timestamp).toEqual(expiresAt);+ expect(spans).toContainEqual({+ name: "bigtable.job_access.write",+ options: { zeroDataRetention: undefined },+ });+ });++ it("limits job access rows to externally operable job kinds", () => {+ expect(API_JOB_KINDS).toEqual([+ "scrape",+ "crawl",+ "batch_scrape",+ "extract",+ "agent",+ "llmstxt",+ "deep_research",+ ]);+ expect(API_JOB_KINDS.every(isApiJobKind)).toBe(true);+ expect(+ [+ "search",+ "map",+ "parse",+ "browser",+ "interact",+ "research_paper_search",+ "research_paper_inspect",+ "research_paper_read",+ "research_related_papers",+ "research_github_search",+ "code_search",+ ].some(isApiJobKind),+ ).toBe(false);+ });++ it("does not initialize Bigtable when a store is disabled", async () => {+ mutableConfig.BIGTABLE_JOB_ACCESS_TABLE = undefined;+ await expect(+ writeApiJobAccess({+ id: JOB_ID,+ teamId: "team-id",+ kind: "scrape",+ expiresAt: new Date(),+ }),+ ).resolves.toBe(false);+ expect(getBigtableTable).not.toHaveBeenCalled();+ });++ it("writes only the precomputed feedback decision inputs", async () => {+ const completedAt = new Date("2026-09-15T12:00:00.000Z");+ await expect(+ writeFeedbackJob({+ jobId: JOB_ID,+ requestId: REQUEST_ID,+ teamId: "team-id",+ endpoint: "scrape",+ scrapeOptions: scrapeOptions.parse({+ parsers: ["pdf"],+ formats: [+ { type: "json", schema: {} },+ { type: "screenshot", fullPage: false },+ ],+ actions: [{ type: "click", selector: "button", all: false }],+ }),+ succeeded: true,+ creditsBilled: 12,+ zeroDataRetention: false,+ completedAt,+ }),+ ).resolves.toBe(true);++ expect(getBigtableTable).toHaveBeenCalledWith("feedback-jobs");+ expect(writtenValue()).toEqual({+ version: 1,+ requestId: REQUEST_ID,+ teamId: "team-id",+ refundClass: "scrape_pdf",+ feedbackDeadlineMs: completedAt.getTime() + 180_000,+ succeeded: true,+ creditsBilled: 12,+ zeroDataRetention: false,+ });+ expect(mutate.mock.calls[0][0][0].data.f.v.timestamp).toEqual(+ new Date(completedAt.getTime() + 180_000),+ );+ expect(spans).toContainEqual({+ name: "bigtable.feedback_job.write",+ options: { zeroDataRetention: false },+ });+ });++ it("uses the search-specific feedback window", async () => {+ const completedAt = new Date("2026-09-15T12:00:00.000Z");+ await writeFeedbackJob({+ jobId: JOB_ID,+ requestId: JOB_ID,+ teamId: "team-id",+ endpoint: "search",+ succeeded: false,+ creditsBilled: 0,+ zeroDataRetention: true,+ completedAt,+ });++ expect(writtenValue()).toMatchObject({+ refundClass: "search",+ feedbackDeadlineMs: completedAt.getTime() + 120_000,+ succeeded: false,+ zeroDataRetention: true,+ });+ expect(spans).toContainEqual({+ name: "bigtable.feedback_job.write",+ options: { zeroDataRetention: true },+ });+ });+});diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex ce6eda1f8..131e2d952 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -13,6 +13,11 @@ const {flush,close,metricInc,+ writeApiJobAccess,+ writeFeedbackJob,+ withSpan,+ setSpanAttributes,+ spans,} = vi.hoisted(() => {const logger: any = {info: vi.fn(),@@ -27,6 +32,13 @@ const {const flush = vi.fn(async () => {});const close = vi.fn(async () => {});const publishes: { name: string; options: any }[] = [];+ const spans: { name: string; options: any }[] = [];+ const withSpan = vi.fn(+ async (name: string, fn: (span: any) => any, options?: any) => {+ spans.push({ name, options });+ return fn({});+ },+ );const topic = vi.fn((name: string, options: any) => {return {publishMessage: (message: any) => {@@ -46,6 +58,11 @@ const {flush,close,metricInc: vi.fn(),+ writeApiJobAccess: vi.fn(async () => true),+ writeFeedbackJob: vi.fn(async () => true),+ withSpan,+ setSpanAttributes: vi.fn(),+ spans,};});@@ -80,6 +97,17 @@ vi.mock("../../lib/change-tracking-store", () => ({changeTrackingInsertScrape: vi.fn(),}));+vi.mock("../../lib/job-access-store", async () => {+ const actual = await vi.importActual<+ typeof import("../../lib/job-access-store")+ >("../../lib/job-access-store");+ return { ...actual, writeApiJobAccess };+});++vi.mock("../../lib/feedback-job-store", () => ({+ writeFeedbackJob,+}));+vi.mock("../../lib/keyless", () => ({keylessTeamUuid: vi.fn(() => null),}));@@ -105,6 +133,11 @@ vi.mock("../../lib/pubsub-log-metrics", () => ({pubsubLogPublishTotal: { inc: metricInc },}));+vi.mock("../../lib/otel-tracer", () => ({+ withSpan,+ setSpanAttributes,+}));+import {logRequest,logSearch,@@ -150,6 +183,7 @@ describe("logSearch", () => {values.mockResolvedValue(undefined);publishMessage.mockResolvedValue("message-id");publishes.length = 0;+ spans.length = 0;});it("removes null bytes from search query log fields", async () => {@@ -169,6 +203,14 @@ describe("logSearch", () => {expect(inserted.options.query).toBe("nestedquery");expect(inserted.options.sources[0].location).toBe("NewYork");expect(search.options.query).toBe("nested\u0000query");+ expect(writeFeedbackJob).toHaveBeenCalledWith(+ expect.objectContaining({+ jobId: search.id,+ endpoint: "search",+ succeeded: true,+ creditsBilled: 1,+ }),+ );});it("reports serialization failures without losing the PostgreSQL attempt", async () => {@@ -187,6 +229,42 @@ describe("logSearch", () => {expect.objectContaining({ logId: search.id, error: expect.any(Error) }),);});++ it("keeps logging when the feedback Bigtable shadow write fails", async () => {+ writeFeedbackJob.mockRejectedValueOnce(new Error("Bigtable unavailable"));++ await expect(logSearch(makeSearch())).resolves.toBeUndefined();++ expect(values).toHaveBeenCalledOnce();+ expect(logger.error).toHaveBeenCalledWith(+ "Failed to write feedback job to Bigtable",+ expect.objectContaining({+ error: expect.any(Error),+ endpoint: "search",+ }),+ );+ });++ it("profiles the log, PostgreSQL insert, and Pub/Sub publish", async () => {+ await logSearch(makeSearch());++ expect(spans).toEqual(+ expect.arrayContaining([+ { name: "log_job.search", options: { zeroDataRetention: false } },+ { name: "log_job.postgres.insert", options: undefined },+ { name: "log_job.pubsub.publish", options: undefined },+ ]),+ );+ });++ it("forwards zeroDataRetention to the root log span", async () => {+ await logSearch(makeSearch({ zeroDataRetention: true }));++ expect(spans).toContainEqual({+ name: "log_job.search",+ options: { zeroDataRetention: true },+ });+ });});describe("logRequest", () => {@@ -246,6 +324,15 @@ describe("logRequest", () => {expect(values.mock.calls[0][0].created_at.toISOString()).toBe(published.created_at,);+ expect(writeApiJobAccess).toHaveBeenCalledWith(+ expect.objectContaining({+ id: "019e6f45-7778-727d-adf0-0abe9d5062b6",+ teamId: "team-id",+ kind: "scrape",+ clientOrigin: "api",+ expiresAt: expect.any(Date),+ }),+ );});it("keeps the database write when Pub/Sub fails", async () => {@@ -260,6 +347,39 @@ describe("logRequest", () => {);});+ it("keeps logging when the job access Bigtable shadow write fails", async () => {+ writeApiJobAccess.mockRejectedValueOnce(new Error("Bigtable unavailable"));++ await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();++ expect(values).toHaveBeenCalled();+ expect(logger.error).toHaveBeenCalledWith(+ "Failed to write API job access to Bigtable",+ expect.objectContaining({ error: expect.any(Error), kind: "scrape" }),+ );+ });++ it("does not write access metadata for a non-operational request id", async () => {+ await logRequest({ ...makeRequest(null), jobAccess: false });++ expect(writeApiJobAccess).not.toHaveBeenCalled();+ expect(values).toHaveBeenCalledOnce();+ });++ it("uses a job's explicit operational expiry", async () => {+ const expiresAt = new Date("2026-09-15T18:00:00.000Z");++ await logRequest({+ ...makeRequest(null),+ kind: "deep_research",+ jobAccessExpiresAt: expiresAt,+ });++ expect(writeApiJobAccess).toHaveBeenCalledWith(+ expect.objectContaining({ kind: "deep_research", expiresAt }),+ );+ });+it("does not hold the caller on a slow publish", async () => {const publication = deferred<string>();publishMessage.mockReturnValueOnce(publication.promise);diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 9ed445b8a..6b1e7e853 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -25,9 +25,54 @@ import { trackFirstSurfaceUse } from "../posthog";import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";import { sanitizeLogData, sanitizeText } from "./sanitize";+import { isApiJobKind, writeApiJobAccess } from "../../lib/job-access-store";+import { writeFeedbackJob } from "../../lib/feedback-job-store";+import { setSpanAttributes, withSpan } from "../../lib/otel-tracer";configDotenv();const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";+const DEFAULT_JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;++async function withLogSpan<T>(+ params: {+ operation: string;+ table: string;+ id: string;+ requestId?: string;+ force?: boolean;+ zeroDataRetention?: boolean;+ },+ fn: () => Promise<T>,+): Promise<T> {+ return withSpan(+ `log_job.${params.operation}`,+ async span => {+ setSpanAttributes(span, {+ "log_job.table": params.table,+ "log_job.id": params.id,+ "log_job.request_id": params.requestId,+ "log_job.force": params.force,+ });+ return fn();+ },+ { zeroDataRetention: params.zeroDataRetention },+ );+}++async function writeFeedbackJobSafely(+ params: Parameters<typeof writeFeedbackJob>[0],+ logger: Logger,+): Promise<void> {+ try {+ await writeFeedbackJob(params);+ } catch (error) {+ logger.error("Failed to write feedback job to Bigtable", {+ error,+ jobId: params.jobId,+ endpoint: params.endpoint,+ });+ }+}/*** Null-aware wrapper around the shared text sanitizer, kept where a cleaned@@ -138,52 +183,74 @@ function getTopic(client: PubSub, table: string): Topic {async function publishLog(table: string, data: any, logger: Logger) {const startedAt = Date.now();try {- if (pubSubShutdown) {- throw new Error("Pub/Sub log publisher is shutting down");- }- const client = getPubSubClient(logger);- if (!client) {- if (config.PUBSUB_CREDENTIALS) {- throw new Error("Pub/Sub log publisher initialization failed");- }- return;- }+ await withSpan("log_job.pubsub.publish", async span => {+ setSpanAttributes(span, {+ "log_job.table": table,+ "log_job.id": data.id,+ "messaging.system": "gcp_pubsub",+ });- const payload = Buffer.from(JSON.stringify(data));- if (- pendingPublications.size >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||- outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES- ) {- droppedTotal++;- pubsubLogPublishTotal.inc({ table, outcome: "dropped" });- const now = Date.now();- if (now - lastDropWarningAt >= 60_000) {- lastDropWarningAt = now;- logger.warn("Dropping Pub/Sub log: publisher backlog is full", {- table,- logId: data.id,- payloadBytes: payload.length,- outstandingMessages: pendingPublications.size,- outstandingBytes,- droppedTotal,+ if (pubSubShutdown) {+ throw new Error("Pub/Sub log publisher is shutting down");+ }+ const client = getPubSubClient(logger);+ if (!client) {+ if (config.PUBSUB_CREDENTIALS) {+ throw new Error("Pub/Sub log publisher initialization failed");+ }+ setSpanAttributes(span, {+ "log_job.pubsub.enabled": false,+ "log_job.pubsub.outcome": "skipped",});+ return;}- return;- }- const publication = getTopic(client, table).publishMessage({- data: payload,- });- pendingPublications.set(publication, { table, logId: data.id, startedAt });- outstandingBytes += payload.length;+ const payload = Buffer.from(JSON.stringify(data));+ setSpanAttributes(span, {+ "log_job.pubsub.enabled": true,+ "log_job.pubsub.payload_bytes": payload.length,+ });+ if (+ pendingPublications.size >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||+ outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES+ ) {+ droppedTotal++;+ pubsubLogPublishTotal.inc({ table, outcome: "dropped" });+ setSpanAttributes(span, { "log_job.pubsub.outcome": "dropped" });+ const now = Date.now();+ if (now - lastDropWarningAt >= 60_000) {+ lastDropWarningAt = now;+ logger.warn("Dropping Pub/Sub log: publisher backlog is full", {+ table,+ logId: data.id,+ payloadBytes: payload.length,+ outstandingMessages: pendingPublications.size,+ outstandingBytes,+ droppedTotal,+ });+ }+ return;+ }- try {- await publication;- pubsubLogPublishTotal.inc({ table, outcome: "published" });- } finally {- pendingPublications.delete(publication);- outstandingBytes -= payload.length;- }+ const publication = getTopic(client, table).publishMessage({+ data: payload,+ });+ pendingPublications.set(publication, {+ table,+ logId: data.id,+ startedAt,+ });+ outstandingBytes += payload.length;++ try {+ await publication;+ pubsubLogPublishTotal.inc({ table, outcome: "published" });+ setSpanAttributes(span, { "log_job.pubsub.outcome": "published" });+ } finally {+ pendingPublications.delete(publication);+ outstandingBytes -= payload.length;+ }+ });} catch (error) {pubsubLogPublishTotal.inc({ table, outcome: "failed" });@@ -289,68 +356,87 @@ async function robustInsert(canonicalLog: "log_job/robustInsert",});- if (config.USE_DB_AUTHENTICATION !== true) {- logger.info(- "Skipping database insertion due to USE_DB_AUTHENTICATION being off",- );- return;- }-- const target = tableMap[table];- // The single point where a row leaves for both stores: clean it once so- // PostgreSQL and ClickHouse receive identical, accepted values.- data = sanitizeLogData({- ...data,- created_at: data.created_at ?? new Date(),- });- // Publish in the background. Customer responses must not wait for Pub/Sub.- void publishLog(table, data, logger);-const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];+ try {+ const inserted = await withSpan("log_job.postgres.insert", async span => {+ setSpanAttributes(span, {+ "db.system": "postgresql",+ "log_job.table": table,+ "log_job.id": data.id,+ "log_job.force": force,+ });- if (force) {- for (let i = 0; i < 10; i++) {- const start = Date.now();- try {- await db.insert(target).values(data);- attempts.push({- error: null,- timeMs: Date.now() - start,- backoffMs: i === 0 ? 0 : 75,- });- break;- } catch (error) {- attempts.push({- error,- timeMs: Date.now() - start,- backoffMs: i === 0 ? 0 : 75,+ if (config.USE_DB_AUTHENTICATION !== true) {+ logger.info(+ "Skipping database insertion due to USE_DB_AUTHENTICATION being off",+ );+ setSpanAttributes(span, {+ "log_job.postgres.enabled": false,+ "log_job.postgres.outcome": "skipped",});- await new Promise(resolve => setTimeout(resolve, 75));+ return false;}- }- if (attempts.length === 1 && attempts[0].error === null) {+ setSpanAttributes(span, { "log_job.postgres.enabled": true });+ const target = tableMap[table];+ // The single point where a row leaves for both stores: clean it once so+ // PostgreSQL and ClickHouse receive identical, accepted values.+ data = sanitizeLogData({+ ...data,+ created_at: data.created_at ?? new Date(),+ });+ // Publish in the background. Customer responses must not wait for Pub/Sub.+ void publishLog(table, data, logger);++ const maxAttempts = force ? 10 : 1;+ for (let i = 0; i < maxAttempts; i++) {+ const backoffMs = i === 0 ? 0 : 75;+ const start = Date.now();+ try {+ await db.insert(target).values(data);+ attempts.push({+ error: null,+ timeMs: Date.now() - start,+ backoffMs,+ });+ break;+ } catch (error) {+ attempts.push({+ error,+ timeMs: Date.now() - start,+ backoffMs,+ });+ if (force) {+ await new Promise(resolve => setTimeout(resolve, 75));+ }+ }+ }++ const lastAttempt = attempts.at(-1);+ setSpanAttributes(span, {+ "log_job.postgres.attempts": attempts.length,+ "log_job.postgres.retries": Math.max(0, attempts.length - 1),+ "log_job.postgres.outcome":+ lastAttempt?.error === null ? "inserted" : "failed",+ });+ if (lastAttempt?.error !== null) {+ throw (+ lastAttempt?.error ?? new Error("Database insert was not attempted")+ );+ }+ return true;+ });++ if (!inserted) return;+ if (attempts.length === 1) {logger.debug("Inserted into database successfully", { attempts });- } else if (- attempts.length > 1 &&- attempts[attempts.length - 1].error === null- ) {+ } else {logger.warn("Inserted into database successfully with retries", {attempts,});- } else {- logger.error("Failed to insert into database", { attempts });- }- } else {- const start = Date.now();- try {- await db.insert(target).values(data);- attempts.push({ error: null, timeMs: Date.now() - start, backoffMs: 0 });- logger.debug("Inserted into database successfully", { attempts });- } catch (error) {- attempts.push({ error, timeMs: Date.now() - start, backoffMs: 0 });- logger.error("Failed to insert into database", { attempts });}+ } catch {+ logger.error("Failed to insert into database", { attempts });}}@@ -388,6 +474,8 @@ type LoggedRequest = {* and read back off this row by the request id.*/external_request_id?: string | null;+ jobAccess?: boolean;+ jobAccessExpiresAt?: Date;};/**@@ -418,6 +506,18 @@ function boundedExternalRequestId(}export async function logRequest(request: LoggedRequest) {+ return withLogSpan(+ {+ operation: "request",+ table: "requests",+ id: request.id,+ zeroDataRetention: request.zeroDataRetention,+ },+ () => logRequestInternal(request),+ );+}++async function logRequestInternal(request: LoggedRequest) {const logger = _logger.child({module: "log_job",method: "logRequest",@@ -446,6 +546,31 @@ export async function logRequest(request: LoggedRequest) {const sanitizedTargetHint = request.zeroDataRetention? "<redacted due to zero data retention>": sanitizeString(request.target_hint);+ const storedTeamId =+ request.team_id === "preview" || request.team_id?.startsWith("preview_")+ ? previewTeamId+ : request.team_id;+ const jobAccessTeamId = keylessTeamUuid(request.team_id) ?? storedTeamId;++ if (request.jobAccess !== false && isApiJobKind(request.kind)) {+ try {+ await writeApiJobAccess({+ id: request.id,+ teamId: jobAccessTeamId,+ kind: request.kind,+ expiresAt:+ request.jobAccessExpiresAt ??+ new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),+ clientOrigin: sanitizedOrigin,+ zeroDataRetention: request.zeroDataRetention,+ });+ } catch (error) {+ logger.error("Failed to write API job access to Bigtable", {+ error,+ kind: request.kind,+ });+ }+ }await robustInsert("requests",@@ -453,10 +578,7 @@ export async function logRequest(request: LoggedRequest) {id: request.id,kind: request.kind,api_version: request.api_version,- team_id:- request.team_id === "preview" || request.team_id?.startsWith("preview_")- ? previewTeamId- : request.team_id,+ team_id: storedTeamId,origin: sanitizedOrigin,integration: sanitizedIntegration,target_hint: sanitizedTargetHint,@@ -499,6 +621,20 @@ export type LoggedScrape = {};export async function logScrape(scrape: LoggedScrape, force: boolean = false) {+ return withLogSpan(+ {+ operation: scrape.is_parse ? "parse" : "scrape",+ table: scrape.is_parse ? "parses" : "scrapes",+ id: scrape.id,+ requestId: scrape.request_id,+ force,+ zeroDataRetention: scrape.zeroDataRetention,+ },+ () => logScrapeInternal(scrape, force),+ );+}++async function logScrapeInternal(scrape: LoggedScrape, force: boolean = false) {const logger = _logger.child({module: "log_job",method: "logScrape",@@ -509,6 +645,26 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {});const tableName = scrape.is_parse ? "parses" : "scrapes";+ const storedTeamId =+ keylessTeamUuid(scrape.team_id) ??+ (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")+ ? previewTeamId+ : scrape.team_id);++ const feedbackJob = {+ jobId: scrape.id,+ requestId: scrape.request_id,+ teamId: storedTeamId,+ succeeded: scrape.is_successful,+ creditsBilled: scrape.credits_cost,+ zeroDataRetention: scrape.zeroDataRetention,+ };+ await writeFeedbackJobSafely(+ scrape.is_parse+ ? { ...feedbackJob, endpoint: "parse" }+ : { ...feedbackJob, endpoint: "scrape", scrapeOptions: scrape.options },+ logger,+ );await robustInsert(tableName,@@ -521,11 +677,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {is_successful: scrape.is_successful,error: scrape.error ?? null,time_taken: scrape.time_taken,- team_id:- keylessTeamUuid(scrape.team_id) ??- (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")- ? previewTeamId- : scrape.team_id),+ team_id: storedTeamId,options: scrape.zeroDataRetention ? null : scrape.options,cost_tracking: scrape.zeroDataRetention? null@@ -604,6 +756,20 @@ type LoggedCrawl = {};export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {+ return withLogSpan(+ {+ operation: "crawl",+ table: "crawls",+ id: crawl.id,+ requestId: crawl.request_id,+ force,+ zeroDataRetention: crawl.zeroDataRetention,+ },+ () => logCrawlInternal(crawl, force),+ );+}++async function logCrawlInternal(crawl: LoggedCrawl, force: boolean = false) {const logger = _logger.child({module: "log_job",method: "logCrawl",@@ -650,6 +816,23 @@ type LoggedBatchScrape = {export async function logBatchScrape(batchScrape: LoggedBatchScrape,force: boolean = false,+) {+ return withLogSpan(+ {+ operation: "batch_scrape",+ table: "batch_scrapes",+ id: batchScrape.id,+ requestId: batchScrape.request_id,+ force,+ zeroDataRetention: batchScrape.zeroDataRetention,+ },+ () => logBatchScrapeInternal(batchScrape, force),+ );+}++async function logBatchScrapeInternal(+ batchScrape: LoggedBatchScrape,+ force: boolean = false,) {const logger = _logger.child({module: "log_job",@@ -695,6 +878,20 @@ export type LoggedSearch = {};export async function logSearch(search: LoggedSearch, force: boolean = false) {+ return withLogSpan(+ {+ operation: "search",+ table: "searches",+ id: search.id,+ requestId: search.request_id,+ force,+ zeroDataRetention: search.zeroDataRetention,+ },+ () => logSearchInternal(search, force),+ );+}++async function logSearchInternal(search: LoggedSearch, force: boolean = false) {const logger = _logger.child({module: "log_job",method: "logSearch",@@ -708,6 +905,23 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {search.zeroDataRetention || typeof search.options?.query !== "string"? search.options: { ...search.options, query: sanitizeString(search.options.query) };+ const storedTeamId =+ search.team_id === "preview" || search.team_id?.startsWith("preview_")+ ? previewTeamId+ : search.team_id;++ await writeFeedbackJobSafely(+ {+ jobId: search.id,+ requestId: search.request_id,+ teamId: storedTeamId,+ endpoint: "search",+ succeeded: search.is_successful,+ creditsBilled: search.credits_cost,+ zeroDataRetention: search.zeroDataRetention,+ },+ logger,+ );await robustInsert("searches",@@ -717,10 +931,7 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {query: search.zeroDataRetention? "<redacted due to zero data retention>": sanitizeString(search.query),- team_id:- search.team_id === "preview" || search.team_id?.startsWith("preview_")- ? previewTeamId- : search.team_id,+ team_id: storedTeamId,options: search.zeroDataRetention? { enterprise: search.options?.enterprise }: options,@@ -774,6 +985,23 @@ type LoggedResearchEndpoint = {export async function logResearchEndpoint(research: LoggedResearchEndpoint,force: boolean = false,+) {+ return withLogSpan(+ {+ operation: "research",+ table: research.table,+ id: research.id,+ requestId: research.request_id,+ force,+ zeroDataRetention: research.zeroDataRetention,+ },+ () => logResearchEndpointInternal(research, force),+ );+}++async function logResearchEndpointInternal(+ research: LoggedResearchEndpoint,+ force: boolean = false,) {const logger = _logger.child({module: "log_job",@@ -828,6 +1056,22 @@ export type LoggedExtract = {export async function logExtract(extract: LoggedExtract,force: boolean = false,+) {+ return withLogSpan(+ {+ operation: "extract",+ table: "extracts",+ id: extract.id,+ requestId: extract.request_id,+ force,+ },+ () => logExtractInternal(extract, force),+ );+}++async function logExtractInternal(+ extract: LoggedExtract,+ force: boolean = false,) {const logger = _logger.child({module: "log_job",@@ -880,6 +1124,20 @@ export type LoggedMap = {};export async function logMap(map: LoggedMap, force: boolean = false) {+ return withLogSpan(+ {+ operation: "map",+ table: "maps",+ id: map.id,+ requestId: map.request_id,+ force,+ zeroDataRetention: map.zeroDataRetention,+ },+ () => logMapInternal(map, force),+ );+}++async function logMapInternal(map: LoggedMap, force: boolean = false) {const logger = _logger.child({module: "log_job",method: "logMap",@@ -888,6 +1146,23 @@ export async function logMap(map: LoggedMap, force: boolean = false) {teamId: map.team_id,zeroDataRetention: map.zeroDataRetention,});+ const storedTeamId =+ map.team_id === "preview" || map.team_id?.startsWith("preview_")+ ? previewTeamId+ : map.team_id;++ await writeFeedbackJobSafely(+ {+ jobId: map.id,+ requestId: map.request_id,+ teamId: storedTeamId,+ endpoint: "map",+ succeeded: true,+ creditsBilled: map.credits_cost,+ zeroDataRetention: map.zeroDataRetention,+ },+ logger,+ );await robustInsert("maps",@@ -897,10 +1172,7 @@ export async function logMap(map: LoggedMap, force: boolean = false) {url: map.zeroDataRetention? "<redacted due to zero data retention>": map.url,- team_id:- map.team_id === "preview" || map.team_id?.startsWith("preview_")- ? previewTeamId- : map.team_id,+ team_id: storedTeamId,options: map.zeroDataRetention ? null : map.options,num_results: map.results.length,credits_cost: map.credits_cost,@@ -929,6 +1201,22 @@ export type LoggedLlmsTxt = {export async function logLlmsTxt(llmsTxt: LoggedLlmsTxt,force: boolean = false,+) {+ return withLogSpan(+ {+ operation: "llmstxt",+ table: "llmstxts",+ id: llmsTxt.id,+ requestId: llmsTxt.request_id,+ force,+ },+ () => logLlmsTxtInternal(llmsTxt, force),+ );+}++async function logLlmsTxtInternal(+ llmsTxt: LoggedLlmsTxt,+ force: boolean = false,) {const logger = _logger.child({module: "log_job",@@ -977,6 +1265,22 @@ export type LoggedDeepResearch = {export async function logDeepResearch(deepResearch: LoggedDeepResearch,force: boolean = false,+) {+ return withLogSpan(+ {+ operation: "deep_research",+ table: "deep_researches",+ id: deepResearch.id,+ requestId: deepResearch.request_id,+ force,+ },+ () => logDeepResearchInternal(deepResearch, force),+ );+}++async function logDeepResearchInternal(+ deepResearch: LoggedDeepResearch,+ force: boolean = false,) {const logger = _logger.child({module: "log_job",