Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add Bigtable operational job stores (#4645)

Gergő Móricz847a87f8a57e
diff --git a/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts b/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts
index 0707d4af7..fbeebe3bf 100644
--- a/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts
+++ b/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts
@@ -1,5 +1,20 @@
import type { Mock } from "vitest";
import { redisEvictConnection } from "../../../services/redis";
+
+const { writeApiJobAccess, redisMock } = vi.hoisted(() => ({
+ writeApiJobAccess: vi.fn(async () => true),
+ redisMock: {
+ set: vi.fn(async () => "OK"),
+ get: vi.fn(),
+ pttl: vi.fn(),
+ },
+}));
+
+vi.mock("../../../lib/job-access-store", () => ({ writeApiJobAccess }));
+vi.mock("../../../services/redis", () => ({
+ redisEvictConnection: redisMock,
+}));
+
import {
saveDeepResearch,
getDeepResearch,
@@ -8,15 +23,6 @@ import {
StoredDeepResearch,
} from "../../../lib/deep-research/deep-research-redis";
-vi.mock("../../../services/queue-service", () => ({
- redisConnection: {
- set: vi.fn(),
- get: vi.fn(),
- expire: vi.fn(),
- pttl: vi.fn(),
- },
-}));
-
describe("Deep Research Redis Operations", () => {
const mockResearch: StoredDeepResearch = {
id: "test-id",
@@ -35,6 +41,7 @@ describe("Deep Research Redis Operations", () => {
beforeEach(() => {
vi.clearAllMocks();
+ writeApiJobAccess.mockResolvedValue(true);
});
describe("saveDeepResearch", () => {
@@ -44,11 +51,27 @@ describe("Deep Research Redis Operations", () => {
expect(redisEvictConnection.set).toHaveBeenCalledWith(
"deep-research:test-id",
JSON.stringify(mockResearch),
- );
- expect(redisEvictConnection.expire).toHaveBeenCalledWith(
- "deep-research:test-id",
+ "EX",
6 * 60 * 60,
);
+ expect(writeApiJobAccess).toHaveBeenCalledWith({
+ id: "test-id",
+ teamId: "team-1",
+ kind: "deep_research",
+ expiresAt: expect.any(Date),
+ });
+ });
+
+ it("keeps the Redis write when the access refresh fails", async () => {
+ writeApiJobAccess.mockRejectedValueOnce(
+ new Error("Bigtable unavailable"),
+ );
+
+ await expect(
+ saveDeepResearch("test-id", mockResearch),
+ ).resolves.toBeUndefined();
+
+ expect(redisEvictConnection.set).toHaveBeenCalledOnce();
});
});
@@ -104,11 +127,15 @@ describe("Deep Research Redis Operations", () => {
expect(redisEvictConnection.set).toHaveBeenCalledWith(
"deep-research:test-id",
JSON.stringify(expectedUpdate),
- );
- expect(redisEvictConnection.expire).toHaveBeenCalledWith(
- "deep-research:test-id",
+ "EX",
6 * 60 * 60,
);
+ expect(writeApiJobAccess).toHaveBeenCalledWith({
+ id: "test-id",
+ teamId: "team-1",
+ kind: "deep_research",
+ expiresAt: expect.any(Date),
+ });
});
it("should do nothing if research not found", async () => {
@@ -117,7 +144,7 @@ describe("Deep Research Redis Operations", () => {
await updateDeepResearch("test-id", { status: "completed" });
expect(redisEvictConnection.set).not.toHaveBeenCalled();
- expect(redisEvictConnection.expire).not.toHaveBeenCalled();
+ expect(writeApiJobAccess).not.toHaveBeenCalled();
});
});
@@ -126,13 +153,12 @@ describe("Deep Research Redis Operations", () => {
const mockTTL = 3600000; // 1 hour in milliseconds
(redisEvictConnection.pttl as Mock).mockResolvedValue(mockTTL);
+ const before = Date.now();
const result = await getDeepResearchExpiry("test-id");
expect(result).toBeInstanceOf(Date);
- expect(result.getTime()).toBeCloseTo(
- new Date().getTime() + mockTTL,
- -2, // Allow 100ms precision
- );
+ expect(result.getTime()).toBeGreaterThanOrEqual(before + mockTTL - 999);
+ expect(result.getTime()).toBeLessThanOrEqual(Date.now() + mockTTL);
});
});
});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index e798dee2b..dcff8d868 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -275,6 +275,8 @@ const configSchema = z.object({
BIGTABLE_INSTANCE_ID: z.string().optional(),
BIGTABLE_APP_PROFILE_ID: z.string().optional(),
BIGTABLE_CHANGE_TRACKING_TABLE: z.string().optional(),
+ BIGTABLE_JOB_ACCESS_TABLE: z.string().optional(),
+ BIGTABLE_FEEDBACK_JOBS_TABLE: z.string().optional(),
BIGTABLE_CREDENTIALS: z.string().optional(),
// ClickHouse (Search Analytics)
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 5227beabd..06acd2184 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -91,6 +91,9 @@ export async function crawlController(req: Request, res: Response) {
target_hint: req.body.url ?? "",
zeroDataRetention: false, // not supported on v0
api_key_id: chunk?.api_key_id ?? null,
+ jobAccessExpiresAt: new Date(
+ Date.now() + (chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ ),
});
redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 9f4dbfe3d..2a65685dc 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -316,6 +316,9 @@ export async function batchScrapeController(
target_hint: urls[0] ?? "",
zeroDataRetention: zeroDataRetention || false,
api_key_id: req.acuc?.api_key_id ?? null,
+ jobAccessExpiresAt: new Date(
+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ ),
});
}
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index a494aaecb..7432f50b3 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -192,6 +192,9 @@ export async function crawlController(
target_hint: req.body.url,
zeroDataRetention: zeroDataRetention || false,
api_key_id: req.acuc?.api_key_id ?? null,
+ jobAccessExpiresAt: new Date(
+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ ),
});
// checkCreditsMiddleware (always runs before this controller) is the source
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts
index b6faff9a6..ef2aabffd 100644
--- a/apps/api/src/controllers/v1/deep-research.ts
+++ b/apps/api/src/controllers/v1/deep-research.ts
@@ -2,7 +2,10 @@ import { v7 as uuidv7 } from "uuid";
import { Request, Response } from "express";
import { ErrorResponse, extractOptions, RequestWithAuth } from "./types";
import { getDeepResearchQueue } from "../../services/queue-service";
-import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";
+import {
+ DEEP_RESEARCH_TTL,
+ saveDeepResearch,
+} from "../../lib/deep-research/deep-research-redis";
import { z } from "zod";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
@@ -104,6 +107,7 @@ export async function deepResearchController(
target_hint: req.body.query ?? "",
zeroDataRetention: false, // not supported for deep research
api_key_id: req.acuc?.api_key_id ?? null,
+ jobAccessExpiresAt: new Date(Date.now() + DEEP_RESEARCH_TTL * 1000),
});
const jobData = {
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 8aeedcdda..0668c19cd 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -345,6 +345,9 @@ export async function batchScrapeController(
target_hint: urls[0] ?? "",
zeroDataRetention: zeroDataRetention || false,
api_key_id: req.acuc?.api_key_id ?? null,
+ jobAccessExpiresAt: new Date(
+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ ),
});
}
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 4c051a8d0..6d2fcb227 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -197,6 +197,9 @@ export async function crawlController(
target_hint: req.body.url,
zeroDataRetention: zeroDataRetention || false,
api_key_id: req.acuc?.api_key_id ?? null,
+ jobAccessExpiresAt: new Date(
+ Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ ),
});
// checkCreditsMiddleware (always runs before this controller) is the source
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
index 2d67c2dea..0334d084e 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
@@ -93,6 +93,9 @@ it("returns the Scrape contract, shares identity with the legacy route, and logs
scrapeId: undefined,
});
expect(mocks.log).toHaveBeenCalledTimes(1);
+ expect(mocks.log).toHaveBeenCalledWith(
+ expect.objectContaining({ kind: "scrape", jobAccess: false }),
+ );
});
it("relays a failed single legacy call as an error, not a success", async () => {
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.ts
index 1cfb6ec57..29b207995 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts
@@ -153,6 +153,7 @@ export async function providerScrapeController(
integration: body.integration ?? null,
target_hint: `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`,
zeroDataRetention: false,
+ jobAccess: false,
}).catch(error =>
logger.warn("Provider request logging failed", {
error,
diff --git a/apps/api/src/lib/bigtable-client.ts b/apps/api/src/lib/bigtable-client.ts
new file mode 100644
index 000000000..dc35f9998
--- /dev/null
+++ b/apps/api/src/lib/bigtable-client.ts
@@ -0,0 +1,44 @@
+import type { Bigtable, Table } from "@google-cloud/bigtable";
+import { config } from "../config";
+import { setSpanAttributes, withSpan } from "./otel-tracer";
+
+let clientPromise: Promise<Bigtable> | null = null;
+const tables = new Map<string, Table>();
+
+export async function getBigtableTable(tableId: string): Promise<Table> {
+ return withSpan("bigtable.table.resolve", async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": tableId,
+ "bigtable.client.cached": clientPromise !== null,
+ "bigtable.table.cached": tables.has(tableId),
+ });
+ if (!config.BIGTABLE_INSTANCE_ID) {
+ throw new Error("BIGTABLE_INSTANCE_ID is not configured");
+ }
+
+ clientPromise ??= import("@google-cloud/bigtable").then(
+ ({ Bigtable }) =>
+ new Bigtable({
+ projectId: config.BIGTABLE_PROJECT_ID,
+ ...(config.BIGTABLE_APP_PROFILE_ID
+ ? { appProfileId: config.BIGTABLE_APP_PROFILE_ID }
+ : {}),
+ ...(config.BIGTABLE_CREDENTIALS
+ ? {
+ credentials: JSON.parse(atob(config.BIGTABLE_CREDENTIALS)),
+ }
+ : {}),
+ metricsEnabled: false,
+ }),
+ );
+ const client = await clientPromise;
+
+ let table = tables.get(tableId);
+ if (!table) {
+ table = client.instance(config.BIGTABLE_INSTANCE_ID).table(tableId);
+ tables.set(tableId, table);
+ }
+ return table;
+ });
+}
diff --git a/apps/api/src/lib/bigtable-row-key.ts b/apps/api/src/lib/bigtable-row-key.ts
new file mode 100644
index 000000000..fa25f9a21
--- /dev/null
+++ b/apps/api/src/lib/bigtable-row-key.ts
@@ -0,0 +1,19 @@
+import crypto from "crypto";
+
+const UUID_V7 =
+ /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+
+export function saltedUuidV7RowKey(id: string): string {
+ if (!UUID_V7.test(id)) {
+ throw new Error(`Expected a UUIDv7 row id, received ${id}`);
+ }
+
+ const uuid = Buffer.from(id.replaceAll("-", ""), "hex");
+ const salt = crypto
+ .createHash("sha256")
+ .update(uuid)
+ .digest()
+ .subarray(0, 1)
+ .toString("hex");
+ return salt + uuid.toString("base64url");
+}
diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.ts
index a011e1206..f4b770a73 100644
--- a/apps/api/src/lib/change-tracking-store.ts
+++ b/apps/api/src/lib/change-tracking-store.ts
@@ -1,7 +1,8 @@
-import type { Bigtable, Table } from "@google-cloud/bigtable";
import crypto from "crypto";
import { config } from "../config";
import { diffGetLastScrape } from "../db/rpc";
+import { getBigtableTable } from "./bigtable-client";
+import { setSpanAttributes, withSpan } from "./otel-tracer";
// Change tracking bookkeeping. Final architecture (#4484 was the
// transition): Bigtable is the store. Writes go to Bigtable only --
@@ -58,48 +59,6 @@ function changeTrackingRowKey(
]);
}
-let bigtableClient: Bigtable | null = null;
-let table: Table | null = null;
-
-// The Bigtable package is loaded lazily so a broken or slow package load
-// surfaces as a caught error in the callers, not an import-time crash.
-async function getChangeTrackingTable(): Promise<Table> {
- if (!bigtableClient) {
- const { Bigtable } = await import("@google-cloud/bigtable");
- bigtableClient = new Bigtable({
- projectId: config.BIGTABLE_PROJECT_ID,
- ...(config.BIGTABLE_APP_PROFILE_ID
- ? { appProfileId: config.BIGTABLE_APP_PROFILE_ID }
- : {}),
- // Mirrors GCS_CREDENTIALS: base64-encoded service-account JSON.
- // Parsed here (not at module load) so a malformed value fails as a
- // caught error, never an import-time crash. Unset falls back to
- // Application Default Credentials.
- ...(config.BIGTABLE_CREDENTIALS
- ? {
- credentials: JSON.parse(atob(config.BIGTABLE_CREDENTIALS)),
- }
- : {}),
- // The client's Cloud Monitoring metrics handler requires the
- // OTel 1.x line, which GHSA-8988-4f7v-96qf only patches on 2.x;
- // we don't consume client-side metrics, so disable the handler
- // and let the dependency overrides move the tree to 2.x.
- metricsEnabled: false,
- });
- }
- if (!table) {
- if (!config.BIGTABLE_INSTANCE_ID) {
- throw new Error(
- "BIGTABLE_INSTANCE_ID is not configured; change tracking requires the Bigtable store",
- );
- }
- table = bigtableClient
- .instance(config.BIGTABLE_INSTANCE_ID)
- .table(TABLE_ID);
- }
- return table;
-}
-
export async function changeTrackingInsertScrape(params: {
team_id: string;
url: string;
@@ -108,29 +67,36 @@ export async function changeTrackingInsertScrape(params: {
/** When the scrape was logged; becomes the cell timestamp. */
date_added: Date;
}): Promise<void> {
- const table = await getChangeTrackingTable();
- // Runtime accepts Buffer keys (converted verbatim); the .d.ts only
- // declares string.
- await table.mutate([
- {
- key: changeTrackingRowKey(
- params.team_id,
- params.url,
- params.tag,
- ) as unknown as string,
- // Required by Mutation.parse -- without it the entry carries no
- // setCell mutations.
- method: "insert" as const,
- data: {
- [FAMILY]: {
- [QUALIFIER]: {
- value: params.job_id,
- timestamp: params.date_added,
+ await withSpan("bigtable.change_tracking.write", async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": TABLE_ID,
+ "bigtable.operation": "mutate",
+ });
+ const table = await getBigtableTable(TABLE_ID);
+ // Runtime accepts Buffer keys (converted verbatim); the .d.ts only
+ // declares string.
+ await table.mutate([
+ {
+ key: changeTrackingRowKey(
+ params.team_id,
+ params.url,
+ params.tag,
+ ) as unknown as string,
+ // Required by Mutation.parse -- without it the entry carries no
+ // setCell mutations.
+ method: "insert" as const,
+ data: {
+ [FAMILY]: {
+ [QUALIFIER]: {
+ value: params.job_id,
+ timestamp: params.date_added,
+ },
},
},
},
- },
- ]);
+ ]);
+ });
}
/**
@@ -144,13 +110,24 @@ export async function changeTrackingGetLastScrape(params: {
url: string;
tag: string | null;
}): Promise<{ job_id: string; date_added: string } | null> {
- const table = await getChangeTrackingTable();
+ const table = await getBigtableTable(TABLE_ID);
const key = changeTrackingRowKey(params.team_id, params.url, params.tag);
- const [rows] = await table.getRows({
- keys: [key as unknown as string],
- // Qualifier regex + latest-version-only, expressed via the column
- // filter's cellLimit (there is no standalone `versions` filter key).
- filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],
+ const rows = await withSpan("bigtable.change_tracking.read", async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": TABLE_ID,
+ "bigtable.operation": "getRows",
+ });
+ const [result] = await table.getRows({
+ keys: [key as unknown as string],
+ // Qualifier regex + latest-version-only, expressed via the column
+ // filter's cellLimit (there is no standalone `versions` filter key).
+ filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],
+ });
+ setSpanAttributes(span, {
+ "bigtable.rows_returned": result.length,
+ });
+ return result;
});
const row = rows[0];
if (row) {
diff --git a/apps/api/src/lib/deep-research/deep-research-redis.ts b/apps/api/src/lib/deep-research/deep-research-redis.ts
index d9d4e8d82..bb45e6dc9 100644
--- a/apps/api/src/lib/deep-research/deep-research-redis.ts
+++ b/apps/api/src/lib/deep-research/deep-research-redis.ts
@@ -1,4 +1,5 @@
import { redisEvictConnection } from "../../services/redis";
+import { writeApiJobAccess } from "../job-access-store";
import { logger as _logger } from "../logger";
export type DeepResearchActivity = {
@@ -45,19 +46,41 @@ export type StoredDeepResearch = {
};
// TTL of 6 hours
-const DEEP_RESEARCH_TTL = 6 * 60 * 60;
+export const DEEP_RESEARCH_TTL = 6 * 60 * 60;
+
+async function refreshDeepResearchAccess(
+ id: string,
+ teamId: string,
+ expiresAt: Date,
+): Promise<void> {
+ try {
+ await writeApiJobAccess({
+ id,
+ teamId,
+ kind: "deep_research",
+ expiresAt,
+ });
+ } catch (error) {
+ _logger.error("Failed to refresh deep research access in Bigtable", {
+ error,
+ researchId: id,
+ });
+ }
+}
export async function saveDeepResearch(
id: string,
research: StoredDeepResearch,
) {
_logger.debug("Saving deep research " + id + " to Redis...");
+ const expiresAt = new Date(Date.now() + DEEP_RESEARCH_TTL * 1000);
await redisEvictConnection.set(
"deep-research:" + id,
JSON.stringify(research),
"EX",
DEEP_RESEARCH_TTL,
);
+ await refreshDeepResearchAccess(id, research.team_id, expiresAt);
}
export async function getDeepResearch(
@@ -95,12 +118,14 @@ export async function updateDeepResearch(
: current.summaries,
};
+ const expiresAt = new Date(Date.now() + DEEP_RESEARCH_TTL * 1000);
await redisEvictConnection.set(
"deep-research:" + id,
JSON.stringify(updatedResearch),
"EX",
DEEP_RESEARCH_TTL,
);
+ await refreshDeepResearchAccess(id, current.team_id, expiresAt);
}
export async function getDeepResearchExpiry(id: string): Promise<Date> {
diff --git a/apps/api/src/lib/feedback-job-store.ts b/apps/api/src/lib/feedback-job-store.ts
new file mode 100644
index 000000000..756287b05
--- /dev/null
+++ b/apps/api/src/lib/feedback-job-store.ts
@@ -0,0 +1,114 @@
+import { config } from "../config";
+import type { ScrapeOptions } from "../controllers/v2/types";
+import { includesFormat } from "./format-utils";
+import { getBigtableTable } from "./bigtable-client";
+import { saltedUuidV7RowKey } from "./bigtable-row-key";
+import { setSpanAttributes, withSpan } from "./otel-tracer";
+
+const FAMILY = "f";
+const QUALIFIER = "v";
+
+export type FeedbackEndpoint = "search" | "scrape" | "parse" | "map";
+export type RefundClass =
+ | "search"
+ | "map"
+ | "parse"
+ | "scrape_basic"
+ | "scrape_pdf"
+ | "scrape_json"
+ | "scrape_addon";
+
+function scrapeRefundClass(options: ScrapeOptions): RefundClass {
+ if (
+ options.parsers?.some(
+ parser =>
+ parser === "pdf" ||
+ (typeof parser === "object" && parser.type === "pdf"),
+ )
+ ) {
+ return "scrape_pdf";
+ }
+ if (includesFormat(options.formats, "json")) return "scrape_json";
+ if (
+ includesFormat(options.formats, "screenshot") ||
+ (options.actions?.length ?? 0) > 0
+ ) {
+ return "scrape_addon";
+ }
+ return "scrape_basic";
+}
+
+type FeedbackJobBase = {
+ jobId: string;
+ requestId: string;
+ teamId: string;
+ succeeded: boolean;
+ creditsBilled: number;
+ zeroDataRetention: boolean;
+ completedAt?: Date;
+};
+
+export type FeedbackJobWrite = FeedbackJobBase &
+ (
+ | { endpoint: "scrape"; scrapeOptions: ScrapeOptions }
+ | { endpoint: Exclude<FeedbackEndpoint, "scrape"> }
+ );
+
+export async function writeFeedbackJob(
+ params: FeedbackJobWrite,
+): Promise<boolean> {
+ const tableId = config.BIGTABLE_FEEDBACK_JOBS_TABLE;
+ if (!tableId) return false;
+
+ return withSpan(
+ "bigtable.feedback_job.write",
+ async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": tableId,
+ "bigtable.operation": "mutate",
+ "feedback.endpoint": params.endpoint,
+ });
+ const completedAt = params.completedAt ?? new Date();
+ const feedbackWindowSec =
+ params.endpoint === "search"
+ ? config.SEARCH_FEEDBACK_MAX_AGE_SEC
+ : config.FEEDBACK_MAX_AGE_SEC;
+ const feedbackDeadline = new Date(
+ completedAt.getTime() + feedbackWindowSec * 1000,
+ );
+ const value = Buffer.from(
+ JSON.stringify({
+ version: 1,
+ requestId: params.requestId,
+ teamId: params.teamId,
+ refundClass:
+ params.endpoint === "scrape"
+ ? scrapeRefundClass(params.scrapeOptions)
+ : params.endpoint,
+ feedbackDeadlineMs: feedbackDeadline.getTime(),
+ succeeded: params.succeeded,
+ creditsBilled: params.creditsBilled,
+ zeroDataRetention: params.zeroDataRetention,
+ }),
+ );
+ const table = await getBigtableTable(tableId);
+ await table.mutate([
+ {
+ key: saltedUuidV7RowKey(params.jobId),
+ method: "insert",
+ data: {
+ [FAMILY]: {
+ [QUALIFIER]: {
+ value,
+ timestamp: feedbackDeadline,
+ },
+ },
+ },
+ },
+ ]);
+ return true;
+ },
+ { zeroDataRetention: params.zeroDataRetention },
+ );
+}
diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.ts
new file mode 100644
index 000000000..b514b2bb7
--- /dev/null
+++ b/apps/api/src/lib/job-access-store.ts
@@ -0,0 +1,75 @@
+import { config } from "../config";
+import { getBigtableTable } from "./bigtable-client";
+import { saltedUuidV7RowKey } from "./bigtable-row-key";
+import { setSpanAttributes, withSpan } from "./otel-tracer";
+
+const FAMILY = "j";
+const QUALIFIER = "v";
+
+export const API_JOB_KINDS = [
+ "scrape",
+ "crawl",
+ "batch_scrape",
+ "extract",
+ "agent",
+ "llmstxt",
+ "deep_research",
+] as const;
+
+export type ApiJobKind = (typeof API_JOB_KINDS)[number];
+
+export function isApiJobKind(kind: string): kind is ApiJobKind {
+ return API_JOB_KINDS.includes(kind as ApiJobKind);
+}
+
+export async function writeApiJobAccess(params: {
+ id: string;
+ teamId: string;
+ kind: ApiJobKind;
+ expiresAt: Date;
+ clientOrigin?: string | null;
+ zeroDataRetention?: boolean;
+}): Promise<boolean> {
+ const tableId = config.BIGTABLE_JOB_ACCESS_TABLE;
+ if (!tableId) return false;
+
+ return withSpan(
+ "bigtable.job_access.write",
+ async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": tableId,
+ "bigtable.operation": "mutate",
+ "job_access.kind": params.kind,
+ });
+ const value = Buffer.from(
+ JSON.stringify({
+ version: 1,
+ teamId: params.teamId,
+ kind: params.kind,
+ expiresAtMs: params.expiresAt.getTime(),
+ ...(params.kind === "agent" && params.clientOrigin
+ ? { clientOrigin: params.clientOrigin }
+ : {}),
+ }),
+ );
+ const table = await getBigtableTable(tableId);
+ await table.mutate([
+ {
+ key: saltedUuidV7RowKey(params.id),
+ method: "insert",
+ data: {
+ [FAMILY]: {
+ [QUALIFIER]: {
+ value,
+ timestamp: params.expiresAt,
+ },
+ },
+ },
+ },
+ ]);
+ return true;
+ },
+ { zeroDataRetention: params.zeroDataRetention },
+ );
+}
diff --git a/apps/api/src/lib/job-operational-stores.test.ts b/apps/api/src/lib/job-operational-stores.test.ts
new file mode 100644
index 000000000..92b5a8a74
--- /dev/null
+++ b/apps/api/src/lib/job-operational-stores.test.ts
@@ -0,0 +1,216 @@
+import { vi } from "vitest";
+import type { MutationConstructorObj } from "@google-cloud/bigtable";
+
+const { mutate, getBigtableTable, mutableConfig, withSpan, spans } = vi.hoisted(
+ () => {
+ const spans: { name: string; options: any }[] = [];
+ return {
+ mutate: vi.fn<(mutations: MutationConstructorObj[]) => Promise<void>>(
+ async () => {},
+ ),
+ getBigtableTable: vi.fn(),
+ mutableConfig: {
+ BIGTABLE_JOB_ACCESS_TABLE: "job-access",
+ BIGTABLE_FEEDBACK_JOBS_TABLE: "feedback-jobs",
+ SEARCH_FEEDBACK_MAX_AGE_SEC: 120,
+ FEEDBACK_MAX_AGE_SEC: 180,
+ } as {
+ BIGTABLE_JOB_ACCESS_TABLE?: string;
+ BIGTABLE_FEEDBACK_JOBS_TABLE?: string;
+ SEARCH_FEEDBACK_MAX_AGE_SEC: number;
+ FEEDBACK_MAX_AGE_SEC: number;
+ },
+ withSpan: vi.fn(
+ async (name: string, fn: (span: any) => any, options?: any) => {
+ spans.push({ name, options });
+ return fn({});
+ },
+ ),
+ spans,
+ };
+ },
+);
+
+vi.mock("../config", () => ({ config: mutableConfig }));
+vi.mock("./bigtable-client", () => ({ getBigtableTable }));
+vi.mock("./otel-tracer", () => ({
+ withSpan,
+ setSpanAttributes: vi.fn(),
+}));
+
+import {
+ API_JOB_KINDS,
+ isApiJobKind,
+ writeApiJobAccess,
+} from "./job-access-store";
+import { writeFeedbackJob } from "./feedback-job-store";
+import { saltedUuidV7RowKey } from "./bigtable-row-key";
+import { scrapeOptions } from "../controllers/v2/types";
+
+const JOB_ID = "019e6f45-7778-727d-adf0-0abe9d5062b6";
+const REQUEST_ID = "019e6f45-7778-727d-adf0-0abe9d5062b7";
+
+function writtenValue(): Record<string, string | number | boolean> {
+ const mutation = mutate.mock.calls[0][0][0];
+ return JSON.parse(
+ mutation.data.j?.v?.value ?? mutation.data.f.v.value,
+ ) as Record<string, string | number | boolean>;
+}
+
+describe("operational Bigtable stores", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getBigtableTable.mockResolvedValue({ mutate });
+ mutableConfig.BIGTABLE_JOB_ACCESS_TABLE = "job-access";
+ mutableConfig.BIGTABLE_FEEDBACK_JOBS_TABLE = "feedback-jobs";
+ spans.length = 0;
+ });
+
+ it("salts UUIDv7 row keys while preserving compact binary ids", () => {
+ const key = saltedUuidV7RowKey(JOB_ID);
+ expect(key).toHaveLength(24);
+ expect(key.slice(0, 2)).toMatch(/^[0-9a-f]{2}$/);
+ expect(Buffer.from(key.slice(2), "base64url").toString("hex")).toBe(
+ JOB_ID.replaceAll("-", ""),
+ );
+ expect(() =>
+ saltedUuidV7RowKey("550e8400-e29b-41d4-a716-446655440000"),
+ ).toThrow("Expected a UUIDv7 row id");
+ });
+
+ it("writes an acknowledged job access row", async () => {
+ const expiresAt = new Date("2026-09-16T12:00:00.000Z");
+ await expect(
+ writeApiJobAccess({
+ id: JOB_ID,
+ teamId: "team-id",
+ kind: "agent",
+ expiresAt,
+ clientOrigin: "[email protected]",
+ }),
+ ).resolves.toBe(true);
+
+ expect(getBigtableTable).toHaveBeenCalledWith("job-access");
+ expect(writtenValue()).toEqual({
+ version: 1,
+ teamId: "team-id",
+ kind: "agent",
+ expiresAtMs: expiresAt.getTime(),
+ clientOrigin: "[email protected]",
+ });
+ expect(mutate.mock.calls[0][0][0].data.j.v.timestamp).toEqual(expiresAt);
+ expect(spans).toContainEqual({
+ name: "bigtable.job_access.write",
+ options: { zeroDataRetention: undefined },
+ });
+ });
+
+ it("limits job access rows to externally operable job kinds", () => {
+ expect(API_JOB_KINDS).toEqual([
+ "scrape",
+ "crawl",
+ "batch_scrape",
+ "extract",
+ "agent",
+ "llmstxt",
+ "deep_research",
+ ]);
+ expect(API_JOB_KINDS.every(isApiJobKind)).toBe(true);
+ expect(
+ [
+ "search",
+ "map",
+ "parse",
+ "browser",
+ "interact",
+ "research_paper_search",
+ "research_paper_inspect",
+ "research_paper_read",
+ "research_related_papers",
+ "research_github_search",
+ "code_search",
+ ].some(isApiJobKind),
+ ).toBe(false);
+ });
+
+ it("does not initialize Bigtable when a store is disabled", async () => {
+ mutableConfig.BIGTABLE_JOB_ACCESS_TABLE = undefined;
+ await expect(
+ writeApiJobAccess({
+ id: JOB_ID,
+ teamId: "team-id",
+ kind: "scrape",
+ expiresAt: new Date(),
+ }),
+ ).resolves.toBe(false);
+ expect(getBigtableTable).not.toHaveBeenCalled();
+ });
+
+ it("writes only the precomputed feedback decision inputs", async () => {
+ const completedAt = new Date("2026-09-15T12:00:00.000Z");
+ await expect(
+ writeFeedbackJob({
+ jobId: JOB_ID,
+ requestId: REQUEST_ID,
+ teamId: "team-id",
+ endpoint: "scrape",
+ scrapeOptions: scrapeOptions.parse({
+ parsers: ["pdf"],
+ formats: [
+ { type: "json", schema: {} },
+ { type: "screenshot", fullPage: false },
+ ],
+ actions: [{ type: "click", selector: "button", all: false }],
+ }),
+ succeeded: true,
+ creditsBilled: 12,
+ zeroDataRetention: false,
+ completedAt,
+ }),
+ ).resolves.toBe(true);
+
+ expect(getBigtableTable).toHaveBeenCalledWith("feedback-jobs");
+ expect(writtenValue()).toEqual({
+ version: 1,
+ requestId: REQUEST_ID,
+ teamId: "team-id",
+ refundClass: "scrape_pdf",
+ feedbackDeadlineMs: completedAt.getTime() + 180_000,
+ succeeded: true,
+ creditsBilled: 12,
+ zeroDataRetention: false,
+ });
+ expect(mutate.mock.calls[0][0][0].data.f.v.timestamp).toEqual(
+ new Date(completedAt.getTime() + 180_000),
+ );
+ expect(spans).toContainEqual({
+ name: "bigtable.feedback_job.write",
+ options: { zeroDataRetention: false },
+ });
+ });
+
+ it("uses the search-specific feedback window", async () => {
+ const completedAt = new Date("2026-09-15T12:00:00.000Z");
+ await writeFeedbackJob({
+ jobId: JOB_ID,
+ requestId: JOB_ID,
+ teamId: "team-id",
+ endpoint: "search",
+ succeeded: false,
+ creditsBilled: 0,
+ zeroDataRetention: true,
+ completedAt,
+ });
+
+ expect(writtenValue()).toMatchObject({
+ refundClass: "search",
+ feedbackDeadlineMs: completedAt.getTime() + 120_000,
+ succeeded: false,
+ zeroDataRetention: true,
+ });
+ expect(spans).toContainEqual({
+ name: "bigtable.feedback_job.write",
+ options: { zeroDataRetention: true },
+ });
+ });
+});
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index ce6eda1f8..131e2d952 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -13,6 +13,11 @@ const {
flush,
close,
metricInc,
+ writeApiJobAccess,
+ writeFeedbackJob,
+ withSpan,
+ setSpanAttributes,
+ spans,
} = vi.hoisted(() => {
const logger: any = {
info: vi.fn(),
@@ -27,6 +32,13 @@ const {
const flush = vi.fn(async () => {});
const close = vi.fn(async () => {});
const publishes: { name: string; options: any }[] = [];
+ const spans: { name: string; options: any }[] = [];
+ const withSpan = vi.fn(
+ async (name: string, fn: (span: any) => any, options?: any) => {
+ spans.push({ name, options });
+ return fn({});
+ },
+ );
const topic = vi.fn((name: string, options: any) => {
return {
publishMessage: (message: any) => {
@@ -46,6 +58,11 @@ const {
flush,
close,
metricInc: vi.fn(),
+ writeApiJobAccess: vi.fn(async () => true),
+ writeFeedbackJob: vi.fn(async () => true),
+ withSpan,
+ setSpanAttributes: vi.fn(),
+ spans,
};
});
@@ -80,6 +97,17 @@ vi.mock("../../lib/change-tracking-store", () => ({
changeTrackingInsertScrape: vi.fn(),
}));
+vi.mock("../../lib/job-access-store", async () => {
+ const actual = await vi.importActual<
+ typeof import("../../lib/job-access-store")
+ >("../../lib/job-access-store");
+ return { ...actual, writeApiJobAccess };
+});
+
+vi.mock("../../lib/feedback-job-store", () => ({
+ writeFeedbackJob,
+}));
+
vi.mock("../../lib/keyless", () => ({
keylessTeamUuid: vi.fn(() => null),
}));
@@ -105,6 +133,11 @@ vi.mock("../../lib/pubsub-log-metrics", () => ({
pubsubLogPublishTotal: { inc: metricInc },
}));
+vi.mock("../../lib/otel-tracer", () => ({
+ withSpan,
+ setSpanAttributes,
+}));
+
import {
logRequest,
logSearch,
@@ -150,6 +183,7 @@ describe("logSearch", () => {
values.mockResolvedValue(undefined);
publishMessage.mockResolvedValue("message-id");
publishes.length = 0;
+ spans.length = 0;
});
it("removes null bytes from search query log fields", async () => {
@@ -169,6 +203,14 @@ describe("logSearch", () => {
expect(inserted.options.query).toBe("nestedquery");
expect(inserted.options.sources[0].location).toBe("NewYork");
expect(search.options.query).toBe("nested\u0000query");
+ expect(writeFeedbackJob).toHaveBeenCalledWith(
+ expect.objectContaining({
+ jobId: search.id,
+ endpoint: "search",
+ succeeded: true,
+ creditsBilled: 1,
+ }),
+ );
});
it("reports serialization failures without losing the PostgreSQL attempt", async () => {
@@ -187,6 +229,42 @@ describe("logSearch", () => {
expect.objectContaining({ logId: search.id, error: expect.any(Error) }),
);
});
+
+ it("keeps logging when the feedback Bigtable shadow write fails", async () => {
+ writeFeedbackJob.mockRejectedValueOnce(new Error("Bigtable unavailable"));
+
+ await expect(logSearch(makeSearch())).resolves.toBeUndefined();
+
+ expect(values).toHaveBeenCalledOnce();
+ expect(logger.error).toHaveBeenCalledWith(
+ "Failed to write feedback job to Bigtable",
+ expect.objectContaining({
+ error: expect.any(Error),
+ endpoint: "search",
+ }),
+ );
+ });
+
+ it("profiles the log, PostgreSQL insert, and Pub/Sub publish", async () => {
+ await logSearch(makeSearch());
+
+ expect(spans).toEqual(
+ expect.arrayContaining([
+ { name: "log_job.search", options: { zeroDataRetention: false } },
+ { name: "log_job.postgres.insert", options: undefined },
+ { name: "log_job.pubsub.publish", options: undefined },
+ ]),
+ );
+ });
+
+ it("forwards zeroDataRetention to the root log span", async () => {
+ await logSearch(makeSearch({ zeroDataRetention: true }));
+
+ expect(spans).toContainEqual({
+ name: "log_job.search",
+ options: { zeroDataRetention: true },
+ });
+ });
});
describe("logRequest", () => {
@@ -246,6 +324,15 @@ describe("logRequest", () => {
expect(values.mock.calls[0][0].created_at.toISOString()).toBe(
published.created_at,
);
+ expect(writeApiJobAccess).toHaveBeenCalledWith(
+ expect.objectContaining({
+ id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
+ teamId: "team-id",
+ kind: "scrape",
+ clientOrigin: "api",
+ expiresAt: expect.any(Date),
+ }),
+ );
});
it("keeps the database write when Pub/Sub fails", async () => {
@@ -260,6 +347,39 @@ describe("logRequest", () => {
);
});
+ it("keeps logging when the job access Bigtable shadow write fails", async () => {
+ writeApiJobAccess.mockRejectedValueOnce(new Error("Bigtable unavailable"));
+
+ await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();
+
+ expect(values).toHaveBeenCalled();
+ expect(logger.error).toHaveBeenCalledWith(
+ "Failed to write API job access to Bigtable",
+ expect.objectContaining({ error: expect.any(Error), kind: "scrape" }),
+ );
+ });
+
+ it("does not write access metadata for a non-operational request id", async () => {
+ await logRequest({ ...makeRequest(null), jobAccess: false });
+
+ expect(writeApiJobAccess).not.toHaveBeenCalled();
+ expect(values).toHaveBeenCalledOnce();
+ });
+
+ it("uses a job's explicit operational expiry", async () => {
+ const expiresAt = new Date("2026-09-15T18:00:00.000Z");
+
+ await logRequest({
+ ...makeRequest(null),
+ kind: "deep_research",
+ jobAccessExpiresAt: expiresAt,
+ });
+
+ expect(writeApiJobAccess).toHaveBeenCalledWith(
+ expect.objectContaining({ kind: "deep_research", expiresAt }),
+ );
+ });
+
it("does not hold the caller on a slow publish", async () => {
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 9ed445b8a..6b1e7e853 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -25,9 +25,54 @@ import { trackFirstSurfaceUse } from "../posthog";
import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";
import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";
import { sanitizeLogData, sanitizeText } from "./sanitize";
+import { isApiJobKind, writeApiJobAccess } from "../../lib/job-access-store";
+import { writeFeedbackJob } from "../../lib/feedback-job-store";
+import { setSpanAttributes, withSpan } from "../../lib/otel-tracer";
configDotenv();
const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
+const DEFAULT_JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;
+
+async function withLogSpan<T>(
+ params: {
+ operation: string;
+ table: string;
+ id: string;
+ requestId?: string;
+ force?: boolean;
+ zeroDataRetention?: boolean;
+ },
+ fn: () => Promise<T>,
+): Promise<T> {
+ return withSpan(
+ `log_job.${params.operation}`,
+ async span => {
+ setSpanAttributes(span, {
+ "log_job.table": params.table,
+ "log_job.id": params.id,
+ "log_job.request_id": params.requestId,
+ "log_job.force": params.force,
+ });
+ return fn();
+ },
+ { zeroDataRetention: params.zeroDataRetention },
+ );
+}
+
+async function writeFeedbackJobSafely(
+ params: Parameters<typeof writeFeedbackJob>[0],
+ logger: Logger,
+): Promise<void> {
+ try {
+ await writeFeedbackJob(params);
+ } catch (error) {
+ logger.error("Failed to write feedback job to Bigtable", {
+ error,
+ jobId: params.jobId,
+ endpoint: params.endpoint,
+ });
+ }
+}
/**
* Null-aware wrapper around the shared text sanitizer, kept where a cleaned
@@ -138,52 +183,74 @@ function getTopic(client: PubSub, table: string): Topic {
async function publishLog(table: string, data: any, logger: Logger) {
const startedAt = Date.now();
try {
- if (pubSubShutdown) {
- throw new Error("Pub/Sub log publisher is shutting down");
- }
- const client = getPubSubClient(logger);
- if (!client) {
- if (config.PUBSUB_CREDENTIALS) {
- throw new Error("Pub/Sub log publisher initialization failed");
- }
- return;
- }
+ await withSpan("log_job.pubsub.publish", async span => {
+ setSpanAttributes(span, {
+ "log_job.table": table,
+ "log_job.id": data.id,
+ "messaging.system": "gcp_pubsub",
+ });
- const payload = Buffer.from(JSON.stringify(data));
- if (
- pendingPublications.size >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||
- outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES
- ) {
- droppedTotal++;
- pubsubLogPublishTotal.inc({ table, outcome: "dropped" });
- const now = Date.now();
- if (now - lastDropWarningAt >= 60_000) {
- lastDropWarningAt = now;
- logger.warn("Dropping Pub/Sub log: publisher backlog is full", {
- table,
- logId: data.id,
- payloadBytes: payload.length,
- outstandingMessages: pendingPublications.size,
- outstandingBytes,
- droppedTotal,
+ if (pubSubShutdown) {
+ throw new Error("Pub/Sub log publisher is shutting down");
+ }
+ const client = getPubSubClient(logger);
+ if (!client) {
+ if (config.PUBSUB_CREDENTIALS) {
+ throw new Error("Pub/Sub log publisher initialization failed");
+ }
+ setSpanAttributes(span, {
+ "log_job.pubsub.enabled": false,
+ "log_job.pubsub.outcome": "skipped",
});
+ return;
}
- return;
- }
- const publication = getTopic(client, table).publishMessage({
- data: payload,
- });
- pendingPublications.set(publication, { table, logId: data.id, startedAt });
- outstandingBytes += payload.length;
+ const payload = Buffer.from(JSON.stringify(data));
+ setSpanAttributes(span, {
+ "log_job.pubsub.enabled": true,
+ "log_job.pubsub.payload_bytes": payload.length,
+ });
+ if (
+ pendingPublications.size >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||
+ outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES
+ ) {
+ droppedTotal++;
+ pubsubLogPublishTotal.inc({ table, outcome: "dropped" });
+ setSpanAttributes(span, { "log_job.pubsub.outcome": "dropped" });
+ const now = Date.now();
+ if (now - lastDropWarningAt >= 60_000) {
+ lastDropWarningAt = now;
+ logger.warn("Dropping Pub/Sub log: publisher backlog is full", {
+ table,
+ logId: data.id,
+ payloadBytes: payload.length,
+ outstandingMessages: pendingPublications.size,
+ outstandingBytes,
+ droppedTotal,
+ });
+ }
+ return;
+ }
- try {
- await publication;
- pubsubLogPublishTotal.inc({ table, outcome: "published" });
- } finally {
- pendingPublications.delete(publication);
- outstandingBytes -= payload.length;
- }
+ const publication = getTopic(client, table).publishMessage({
+ data: payload,
+ });
+ pendingPublications.set(publication, {
+ table,
+ logId: data.id,
+ startedAt,
+ });
+ outstandingBytes += payload.length;
+
+ try {
+ await publication;
+ pubsubLogPublishTotal.inc({ table, outcome: "published" });
+ setSpanAttributes(span, { "log_job.pubsub.outcome": "published" });
+ } finally {
+ pendingPublications.delete(publication);
+ outstandingBytes -= payload.length;
+ }
+ });
} catch (error) {
pubsubLogPublishTotal.inc({ table, outcome: "failed" });
@@ -289,68 +356,87 @@ async function robustInsert(
canonicalLog: "log_job/robustInsert",
});
- if (config.USE_DB_AUTHENTICATION !== true) {
- logger.info(
- "Skipping database insertion due to USE_DB_AUTHENTICATION being off",
- );
- return;
- }
-
- const target = tableMap[table];
- // The single point where a row leaves for both stores: clean it once so
- // PostgreSQL and ClickHouse receive identical, accepted values.
- data = sanitizeLogData({
- ...data,
- created_at: data.created_at ?? new Date(),
- });
- // Publish in the background. Customer responses must not wait for Pub/Sub.
- void publishLog(table, data, logger);
-
const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];
+ try {
+ const inserted = await withSpan("log_job.postgres.insert", async span => {
+ setSpanAttributes(span, {
+ "db.system": "postgresql",
+ "log_job.table": table,
+ "log_job.id": data.id,
+ "log_job.force": force,
+ });
- if (force) {
- for (let i = 0; i < 10; i++) {
- const start = Date.now();
- try {
- await db.insert(target).values(data);
- attempts.push({
- error: null,
- timeMs: Date.now() - start,
- backoffMs: i === 0 ? 0 : 75,
- });
- break;
- } catch (error) {
- attempts.push({
- error,
- timeMs: Date.now() - start,
- backoffMs: i === 0 ? 0 : 75,
+ if (config.USE_DB_AUTHENTICATION !== true) {
+ logger.info(
+ "Skipping database insertion due to USE_DB_AUTHENTICATION being off",
+ );
+ setSpanAttributes(span, {
+ "log_job.postgres.enabled": false,
+ "log_job.postgres.outcome": "skipped",
});
- await new Promise(resolve => setTimeout(resolve, 75));
+ return false;
}
- }
- if (attempts.length === 1 && attempts[0].error === null) {
+ setSpanAttributes(span, { "log_job.postgres.enabled": true });
+ const target = tableMap[table];
+ // The single point where a row leaves for both stores: clean it once so
+ // PostgreSQL and ClickHouse receive identical, accepted values.
+ data = sanitizeLogData({
+ ...data,
+ created_at: data.created_at ?? new Date(),
+ });
+ // Publish in the background. Customer responses must not wait for Pub/Sub.
+ void publishLog(table, data, logger);
+
+ const maxAttempts = force ? 10 : 1;
+ for (let i = 0; i < maxAttempts; i++) {
+ const backoffMs = i === 0 ? 0 : 75;
+ const start = Date.now();
+ try {
+ await db.insert(target).values(data);
+ attempts.push({
+ error: null,
+ timeMs: Date.now() - start,
+ backoffMs,
+ });
+ break;
+ } catch (error) {
+ attempts.push({
+ error,
+ timeMs: Date.now() - start,
+ backoffMs,
+ });
+ if (force) {
+ await new Promise(resolve => setTimeout(resolve, 75));
+ }
+ }
+ }
+
+ const lastAttempt = attempts.at(-1);
+ setSpanAttributes(span, {
+ "log_job.postgres.attempts": attempts.length,
+ "log_job.postgres.retries": Math.max(0, attempts.length - 1),
+ "log_job.postgres.outcome":
+ lastAttempt?.error === null ? "inserted" : "failed",
+ });
+ if (lastAttempt?.error !== null) {
+ throw (
+ lastAttempt?.error ?? new Error("Database insert was not attempted")
+ );
+ }
+ return true;
+ });
+
+ if (!inserted) return;
+ if (attempts.length === 1) {
logger.debug("Inserted into database successfully", { attempts });
- } else if (
- attempts.length > 1 &&
- attempts[attempts.length - 1].error === null
- ) {
+ } else {
logger.warn("Inserted into database successfully with retries", {
attempts,
});
- } else {
- logger.error("Failed to insert into database", { attempts });
- }
- } else {
- const start = Date.now();
- try {
- await db.insert(target).values(data);
- attempts.push({ error: null, timeMs: Date.now() - start, backoffMs: 0 });
- logger.debug("Inserted into database successfully", { attempts });
- } catch (error) {
- attempts.push({ error, timeMs: Date.now() - start, backoffMs: 0 });
- logger.error("Failed to insert into database", { attempts });
}
+ } catch {
+ logger.error("Failed to insert into database", { attempts });
}
}
@@ -388,6 +474,8 @@ type LoggedRequest = {
* and read back off this row by the request id.
*/
external_request_id?: string | null;
+ jobAccess?: boolean;
+ jobAccessExpiresAt?: Date;
};
/**
@@ -418,6 +506,18 @@ function boundedExternalRequestId(
}
export async function logRequest(request: LoggedRequest) {
+ return withLogSpan(
+ {
+ operation: "request",
+ table: "requests",
+ id: request.id,
+ zeroDataRetention: request.zeroDataRetention,
+ },
+ () => logRequestInternal(request),
+ );
+}
+
+async function logRequestInternal(request: LoggedRequest) {
const logger = _logger.child({
module: "log_job",
method: "logRequest",
@@ -446,6 +546,31 @@ export async function logRequest(request: LoggedRequest) {
const sanitizedTargetHint = request.zeroDataRetention
? "<redacted due to zero data retention>"
: sanitizeString(request.target_hint);
+ const storedTeamId =
+ request.team_id === "preview" || request.team_id?.startsWith("preview_")
+ ? previewTeamId
+ : request.team_id;
+ const jobAccessTeamId = keylessTeamUuid(request.team_id) ?? storedTeamId;
+
+ if (request.jobAccess !== false && isApiJobKind(request.kind)) {
+ try {
+ await writeApiJobAccess({
+ id: request.id,
+ teamId: jobAccessTeamId,
+ kind: request.kind,
+ expiresAt:
+ request.jobAccessExpiresAt ??
+ new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),
+ clientOrigin: sanitizedOrigin,
+ zeroDataRetention: request.zeroDataRetention,
+ });
+ } catch (error) {
+ logger.error("Failed to write API job access to Bigtable", {
+ error,
+ kind: request.kind,
+ });
+ }
+ }
await robustInsert(
"requests",
@@ -453,10 +578,7 @@ export async function logRequest(request: LoggedRequest) {
id: request.id,
kind: request.kind,
api_version: request.api_version,
- team_id:
- request.team_id === "preview" || request.team_id?.startsWith("preview_")
- ? previewTeamId
- : request.team_id,
+ team_id: storedTeamId,
origin: sanitizedOrigin,
integration: sanitizedIntegration,
target_hint: sanitizedTargetHint,
@@ -499,6 +621,20 @@ export type LoggedScrape = {
};
export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
+ return withLogSpan(
+ {
+ operation: scrape.is_parse ? "parse" : "scrape",
+ table: scrape.is_parse ? "parses" : "scrapes",
+ id: scrape.id,
+ requestId: scrape.request_id,
+ force,
+ zeroDataRetention: scrape.zeroDataRetention,
+ },
+ () => logScrapeInternal(scrape, force),
+ );
+}
+
+async function logScrapeInternal(scrape: LoggedScrape, force: boolean = false) {
const logger = _logger.child({
module: "log_job",
method: "logScrape",
@@ -509,6 +645,26 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
});
const tableName = scrape.is_parse ? "parses" : "scrapes";
+ const storedTeamId =
+ keylessTeamUuid(scrape.team_id) ??
+ (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
+ ? previewTeamId
+ : scrape.team_id);
+
+ const feedbackJob = {
+ jobId: scrape.id,
+ requestId: scrape.request_id,
+ teamId: storedTeamId,
+ succeeded: scrape.is_successful,
+ creditsBilled: scrape.credits_cost,
+ zeroDataRetention: scrape.zeroDataRetention,
+ };
+ await writeFeedbackJobSafely(
+ scrape.is_parse
+ ? { ...feedbackJob, endpoint: "parse" }
+ : { ...feedbackJob, endpoint: "scrape", scrapeOptions: scrape.options },
+ logger,
+ );
await robustInsert(
tableName,
@@ -521,11 +677,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
is_successful: scrape.is_successful,
error: scrape.error ?? null,
time_taken: scrape.time_taken,
- team_id:
- keylessTeamUuid(scrape.team_id) ??
- (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
- ? previewTeamId
- : scrape.team_id),
+ team_id: storedTeamId,
options: scrape.zeroDataRetention ? null : scrape.options,
cost_tracking: scrape.zeroDataRetention
? null
@@ -604,6 +756,20 @@ type LoggedCrawl = {
};
export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {
+ return withLogSpan(
+ {
+ operation: "crawl",
+ table: "crawls",
+ id: crawl.id,
+ requestId: crawl.request_id,
+ force,
+ zeroDataRetention: crawl.zeroDataRetention,
+ },
+ () => logCrawlInternal(crawl, force),
+ );
+}
+
+async function logCrawlInternal(crawl: LoggedCrawl, force: boolean = false) {
const logger = _logger.child({
module: "log_job",
method: "logCrawl",
@@ -650,6 +816,23 @@ type LoggedBatchScrape = {
export async function logBatchScrape(
batchScrape: LoggedBatchScrape,
force: boolean = false,
+) {
+ return withLogSpan(
+ {
+ operation: "batch_scrape",
+ table: "batch_scrapes",
+ id: batchScrape.id,
+ requestId: batchScrape.request_id,
+ force,
+ zeroDataRetention: batchScrape.zeroDataRetention,
+ },
+ () => logBatchScrapeInternal(batchScrape, force),
+ );
+}
+
+async function logBatchScrapeInternal(
+ batchScrape: LoggedBatchScrape,
+ force: boolean = false,
) {
const logger = _logger.child({
module: "log_job",
@@ -695,6 +878,20 @@ export type LoggedSearch = {
};
export async function logSearch(search: LoggedSearch, force: boolean = false) {
+ return withLogSpan(
+ {
+ operation: "search",
+ table: "searches",
+ id: search.id,
+ requestId: search.request_id,
+ force,
+ zeroDataRetention: search.zeroDataRetention,
+ },
+ () => logSearchInternal(search, force),
+ );
+}
+
+async function logSearchInternal(search: LoggedSearch, force: boolean = false) {
const logger = _logger.child({
module: "log_job",
method: "logSearch",
@@ -708,6 +905,23 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {
search.zeroDataRetention || typeof search.options?.query !== "string"
? search.options
: { ...search.options, query: sanitizeString(search.options.query) };
+ const storedTeamId =
+ search.team_id === "preview" || search.team_id?.startsWith("preview_")
+ ? previewTeamId
+ : search.team_id;
+
+ await writeFeedbackJobSafely(
+ {
+ jobId: search.id,
+ requestId: search.request_id,
+ teamId: storedTeamId,
+ endpoint: "search",
+ succeeded: search.is_successful,
+ creditsBilled: search.credits_cost,
+ zeroDataRetention: search.zeroDataRetention,
+ },
+ logger,
+ );
await robustInsert(
"searches",
@@ -717,10 +931,7 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {
query: search.zeroDataRetention
? "<redacted due to zero data retention>"
: sanitizeString(search.query),
- team_id:
- search.team_id === "preview" || search.team_id?.startsWith("preview_")
- ? previewTeamId
- : search.team_id,
+ team_id: storedTeamId,
options: search.zeroDataRetention
? { enterprise: search.options?.enterprise }
: options,
@@ -774,6 +985,23 @@ type LoggedResearchEndpoint = {
export async function logResearchEndpoint(
research: LoggedResearchEndpoint,
force: boolean = false,
+) {
+ return withLogSpan(
+ {
+ operation: "research",
+ table: research.table,
+ id: research.id,
+ requestId: research.request_id,
+ force,
+ zeroDataRetention: research.zeroDataRetention,
+ },
+ () => logResearchEndpointInternal(research, force),
+ );
+}
+
+async function logResearchEndpointInternal(
+ research: LoggedResearchEndpoint,
+ force: boolean = false,
) {
const logger = _logger.child({
module: "log_job",
@@ -828,6 +1056,22 @@ export type LoggedExtract = {
export async function logExtract(
extract: LoggedExtract,
force: boolean = false,
+) {
+ return withLogSpan(
+ {
+ operation: "extract",
+ table: "extracts",
+ id: extract.id,
+ requestId: extract.request_id,
+ force,
+ },
+ () => logExtractInternal(extract, force),
+ );
+}
+
+async function logExtractInternal(
+ extract: LoggedExtract,
+ force: boolean = false,
) {
const logger = _logger.child({
module: "log_job",
@@ -880,6 +1124,20 @@ export type LoggedMap = {
};
export async function logMap(map: LoggedMap, force: boolean = false) {
+ return withLogSpan(
+ {
+ operation: "map",
+ table: "maps",
+ id: map.id,
+ requestId: map.request_id,
+ force,
+ zeroDataRetention: map.zeroDataRetention,
+ },
+ () => logMapInternal(map, force),
+ );
+}
+
+async function logMapInternal(map: LoggedMap, force: boolean = false) {
const logger = _logger.child({
module: "log_job",
method: "logMap",
@@ -888,6 +1146,23 @@ export async function logMap(map: LoggedMap, force: boolean = false) {
teamId: map.team_id,
zeroDataRetention: map.zeroDataRetention,
});
+ const storedTeamId =
+ map.team_id === "preview" || map.team_id?.startsWith("preview_")
+ ? previewTeamId
+ : map.team_id;
+
+ await writeFeedbackJobSafely(
+ {
+ jobId: map.id,
+ requestId: map.request_id,
+ teamId: storedTeamId,
+ endpoint: "map",
+ succeeded: true,
+ creditsBilled: map.credits_cost,
+ zeroDataRetention: map.zeroDataRetention,
+ },
+ logger,
+ );
await robustInsert(
"maps",
@@ -897,10 +1172,7 @@ export async function logMap(map: LoggedMap, force: boolean = false) {
url: map.zeroDataRetention
? "<redacted due to zero data retention>"
: map.url,
- team_id:
- map.team_id === "preview" || map.team_id?.startsWith("preview_")
- ? previewTeamId
- : map.team_id,
+ team_id: storedTeamId,
options: map.zeroDataRetention ? null : map.options,
num_results: map.results.length,
credits_cost: map.credits_cost,
@@ -929,6 +1201,22 @@ export type LoggedLlmsTxt = {
export async function logLlmsTxt(
llmsTxt: LoggedLlmsTxt,
force: boolean = false,
+) {
+ return withLogSpan(
+ {
+ operation: "llmstxt",
+ table: "llmstxts",
+ id: llmsTxt.id,
+ requestId: llmsTxt.request_id,
+ force,
+ },
+ () => logLlmsTxtInternal(llmsTxt, force),
+ );
+}
+
+async function logLlmsTxtInternal(
+ llmsTxt: LoggedLlmsTxt,
+ force: boolean = false,
) {
const logger = _logger.child({
module: "log_job",
@@ -977,6 +1265,22 @@ export type LoggedDeepResearch = {
export async function logDeepResearch(
deepResearch: LoggedDeepResearch,
force: boolean = false,
+) {
+ return withLogSpan(
+ {
+ operation: "deep_research",
+ table: "deep_researches",
+ id: deepResearch.id,
+ requestId: deepResearch.request_id,
+ force,
+ },
+ () => logDeepResearchInternal(deepResearch, force),
+ );
+}
+
+async function logDeepResearchInternal(
+ deepResearch: LoggedDeepResearch,
+ force: boolean = false,
) {
const logger = _logger.child({
module: "log_job",