Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): the job log is Pub/Sub only, no PostgreSQL copy (#5010)
Gergő Móricz897354b915b5
diff --git a/apps/api/src/__tests__/snips/cost-tracking-helpers.ts b/apps/api/src/__tests__/snips/cost-tracking-helpers.tsindex 686fbf858..47159980b 100644--- a/apps/api/src/__tests__/snips/cost-tracking-helpers.ts+++ b/apps/api/src/__tests__/snips/cost-tracking-helpers.ts@@ -1,6 +1,4 @@-import { eq } from "drizzle-orm";-import { db } from "../../db/connection";-import * as schema from "../../db/schema";+import { jobLogJson, jobLogRows } from "./job-log";export type CostTrackingCall = {model: string;@@ -9,23 +7,21 @@ export type CostTrackingCall = {tokens?: { input: number; output: number };};-// The scrape row is written when the job finishes; give the insert a moment.+// The scrape row is published when the job finishes and lands in ClickHouse a+// few seconds later; poll until it carries parseable cost tracking.export async function getCostTrackingCalls(scrapeId: string,): Promise<CostTrackingCall[]> {- for (let attempt = 0; attempt < 10; attempt++) {- const rows = await db- .select({ cost_tracking: schema.scrapes.cost_tracking })- .from(schema.scrapes)- .where(eq(schema.scrapes.id, scrapeId))- .limit(1);- if (rows.length === 1) {- const costTracking = rows[0].cost_tracking as {- calls?: CostTrackingCall[];- } | null;- return costTracking?.calls ?? [];- }+ const deadline = Date.now() + 30000;+ while (Date.now() < deadline) {+ const [row] = await jobLogRows("scrapes", "id = {id: UUID}", {+ id: scrapeId,+ });+ const costTracking = jobLogJson(row?.cost_tracking) as {+ calls?: CostTrackingCall[];+ } | null;+ if (costTracking) return costTracking.calls ?? [];await new Promise(resolve => setTimeout(resolve, 1000));}- throw new Error(`No scrapes row for ${scrapeId}`);+ throw new Error(`No cost tracking on the scrapes row for ${scrapeId}`);}diff --git a/apps/api/src/__tests__/snips/job-log.ts b/apps/api/src/__tests__/snips/job-log.tsnew file mode 100644index 000000000..19bcb08c4--- /dev/null+++ b/apps/api/src/__tests__/snips/job-log.ts@@ -0,0 +1,71 @@+import { clickhouseClient } from "../../lib/clickhouse-client";++/** Tests that read the job log skip unless CLICKHOUSE_ANALYTICS_URL is set. */+export const HAS_JOB_LOG = clickhouseClient !== null;++// The job log lives in ClickHouse, fed by ClickPipes a few seconds behind the+// publish. Tables are ReplacingMergeTree ordered by (team_id, id); FINAL keeps+// the latest publication of an id and the day bound prunes partitions.+export async function jobLogRows<T extends Record<string, unknown>>(+ table: string,+ where: string,+ params: Record<string, string | number>,+ options: { orderBy?: string; limit?: number } = {},+): Promise<T[]> {+ if (clickhouseClient === null) {+ throw new Error("CLICKHOUSE_ANALYTICS_URL is required to read the job log");+ }+ const result = await clickhouseClient.query({+ query: `SELECT * FROM ${table} FINAL WHERE ${where} AND created_at >= now() - INTERVAL 1 DAY${+ options.orderBy ? ` ORDER BY ${options.orderBy}` : ""+ }${options.limit ? ` LIMIT ${options.limit}` : ""}`,+ query_params: params,+ format: "JSONEachRow",+ });+ return result.json<T>();+}++/** Polls until at least `min` rows match; returns the last read, which may be short. */+export async function waitForJobLogRows<T extends Record<string, unknown>>(+ table: string,+ where: string,+ params: Record<string, string | number>,+ options: { orderBy?: string; limit?: number; min?: number } = {},+ timeoutMs = 30000,+): Promise<T[]> {+ const { min = 1, ...query } = options;+ const deadline = Date.now() + timeoutMs;+ let rows = await jobLogRows<T>(table, where, params, query);+ while (rows.length < min && Date.now() < deadline) {+ await new Promise(resolve => setTimeout(resolve, 1000));+ rows = await jobLogRows<T>(table, where, params, query);+ }+ return rows;+}++/** Polls for the newest matching row; null when none lands in time. */+export async function waitForJobLogRow<T extends Record<string, unknown>>(+ table: string,+ where: string,+ params: Record<string, string | number>,+ timeoutMs = 30000,+): Promise<T | null> {+ const [row] = await waitForJobLogRows<T>(+ table,+ where,+ params,+ { orderBy: "created_at DESC", limit: 1 },+ timeoutMs,+ );+ return row ?? null;+}++/** Parses a job log JSON column; empty or malformed text reads as null. */+export function jobLogJson(value: unknown): unknown {+ if (typeof value !== "string") return value ?? null;+ try {+ return JSON.parse(value);+ } catch {+ return null;+ }+}diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.tsindex ab10290ca..9f823332f 100644--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts@@ -16,8 +16,9 @@ import {expectScrapesAreFullyCleanedAfterZDRCleaner,expectBatchScrapeIsCleanedUp,} from "../zdr-helpers";+import { HAS_JOB_LOG } from "../job-log";-describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {+describeIf(TEST_PRODUCTION && HAS_JOB_LOG)("Zero Data Retention", () => {describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {it("should clean up a scrape immediately", async () => {let identity = await idmux({diff --git a/apps/api/src/__tests__/snips/v2/developer.test.ts b/apps/api/src/__tests__/snips/v2/developer.test.tsindex b5233eb4c..2466749e9 100644--- a/apps/api/src/__tests__/snips/v2/developer.test.ts+++ b/apps/api/src/__tests__/snips/v2/developer.test.ts@@ -1,9 +1,7 @@import { config } from "../../../config";import { describeIf, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchPostRaw, researchRaw } from "./lib";-import { and, desc, eq } from "drizzle-orm";-import { db } from "../../../db/connection";-import * as schema from "../../../db/schema";+import { HAS_JOB_LOG, jobLogJson, waitForJobLogRow } from "../job-log";const HAS_RESEARCH = !!config.RESEARCH_PROXY_URL;const KEYLESS_ENABLED =@@ -18,20 +16,6 @@ const SERVING_PATHS = [CANONICAL_PATH, LEGACY_PATH];const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));const sleepForBilling = () => sleep(40000);-async function waitForSingleRow<T>(- fetcher: () => Promise<T | null>,- timeoutMs: number = 10000,- intervalMs: number = 250,-): Promise<T | null> {- const deadline = Date.now() + timeoutMs;- while (Date.now() < deadline) {- const row = await fetcher();- if (row) return row;- await sleep(intervalMs);- }- return null;-}-describeIf(HAS_RESEARCH)("Developer Search API", () => {describe.each(SERVING_PATHS)("developer search on %s", path => {it("serves developer search", async () => {@@ -118,7 +102,7 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {});it("logs the developer search request kind with origin and integration", async () => {- if (!config.USE_DB_AUTHENTICATION) return;+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;const identity = await idmux({name: "developer/logs metadata",@@ -136,27 +120,14 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {expect(res.statusCode).toBe(200);expect(res.body.success).toBe(true);- const requestLog = await waitForSingleRow<{+ const requestLog = await waitForJobLogRow<{origin: string | null;integration: string | null;- }>(async () => {- const data = await db- .select({- origin: schema.requests.origin,- integration: schema.requests.integration,- })- .from(schema.requests)- .where(- and(- eq(schema.requests.team_id, identity.teamId),- eq(schema.requests.kind, "code_search"),- eq(schema.requests.target_hint, query),- ),- )- .orderBy(desc(schema.requests.created_at))- .limit(1);- return data[0] ?? null;- });+ }>(+ "requests",+ "team_id = {teamId: UUID} AND kind = 'code_search' AND target_hint = {query: String}",+ { teamId: identity.teamId, query },+ );expect(requestLog).not.toBeNull();expect(requestLog?.origin).toBe("mcp");@@ -164,7 +135,7 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {}, 120000);it("redacts stored payloads for a forced-ZDR team", async () => {- if (!config.USE_DB_AUTHENTICATION) return;+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;const identity = await idmux({name: "developer/forced ZDR retention",@@ -182,54 +153,32 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {);expect(res.statusCode).toBe(200);- const requestLog = await waitForSingleRow<{+ const requestLog = await waitForJobLogRow<{id: string;target_hint: string;- dr_clean_by: string | null;- }>(async () => {- const data = await db- .select({- id: schema.requests.id,- target_hint: schema.requests.target_hint,- dr_clean_by: schema.requests.dr_clean_by,- })- .from(schema.requests)- .where(- and(- eq(schema.requests.team_id, identity.teamId),- eq(schema.requests.kind, "code_search"),- ),- )- .orderBy(desc(schema.requests.created_at))- .limit(1);- return data[0] ?? null;+ }>("requests", "team_id = {teamId: UUID} AND kind = 'code_search'", {+ teamId: identity.teamId,});expect(requestLog).not.toBeNull();expect(requestLog?.target_hint).toBe("<redacted due to zero data retention>",);- expect(requestLog?.dr_clean_by).not.toBeNull();- const usageLog = await waitForSingleRow<{+ const usageRow = await waitForJobLogRow<{target: string;- options: unknown;- response: unknown;+ options: string | null;+ response: string | null;error: string | null;- }>(async () => {- if (!requestLog) return null;- const data = await db- .select({- target: schema.code_searches.target,- options: schema.code_searches.options,- response: schema.code_searches.response,- error: schema.code_searches.error,- })- .from(schema.code_searches)- .where(eq(schema.code_searches.request_id, requestLog.id))- .limit(1);- return data[0] ?? null;+ }>("code_searches", "request_id = {requestId: UUID}", {+ requestId: requestLog!.id,});+ const usageLog = usageRow && {+ target: usageRow.target,+ options: jobLogJson(usageRow.options),+ response: jobLogJson(usageRow.response),+ error: usageRow.error,+ };expect(usageLog).toEqual({target: "<redacted due to zero data retention>",@@ -240,7 +189,7 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {}, 120000);it("writes a usage row with the billed credits", async () => {- if (!config.USE_DB_AUTHENTICATION) return;+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;const identity = await idmux({name: "developer/logs usage",@@ -252,23 +201,11 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {expect(res.statusCode).toBe(200);const expected = Math.ceil(res.body.results.length / 10) * 2;- const usageLog = await waitForSingleRow<{+ const usageLog = await waitForJobLogRow<{credits_cost: number;num_results: number;is_successful: boolean;- }>(async () => {- const data = await db- .select({- credits_cost: schema.code_searches.credits_cost,- num_results: schema.code_searches.num_results,- is_successful: schema.code_searches.is_successful,- })- .from(schema.code_searches)- .where(eq(schema.code_searches.target, query))- .orderBy(desc(schema.code_searches.created_at))- .limit(1);- return data[0] ?? null;- });+ }>("code_searches", "target = {query: String}", { query });expect(usageLog).not.toBeNull();expect(usageLog?.is_successful).toBe(true);diff --git a/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts b/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.tsindex 48ad1e233..2bb12d008 100644--- a/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts+++ b/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts@@ -10,9 +10,7 @@ import {scrapeTimeout,searchRawFull,} from "./lib";-import { and, eq } from "drizzle-orm";-import { db } from "../../../db/connection";-import * as schema from "../../../db/schema";+import { writeFeedbackJob } from "../../../lib/feedback-job-store";let identity: Identity;let secondaryIdentity: Identity;@@ -123,42 +121,32 @@ describeIf(TEST_PRODUCTION)("Generic endpoint feedback tests", () => {.toString(16).padStart(12, "0");- await db.insert(schema.searches).values({- id: searchId,- request_id: searchId,- query: "failed generic feedback search",- team_id: identity.teamId,- options: { query: "failed generic feedback search" },- time_taken: 0,- credits_cost: 2,- is_successful: false,- error: "Synthetic failed search for generic feedback policy coverage.",- num_results: 0,- });-- try {- const failed = await endpointFeedbackRaw(- {- endpoint: "search",- jobId: searchId,- rating: "bad",- missingContent: [{ topic: "Results" }],- },- identity,- );-- expect(failed.statusCode).toBe(409);- expect(failed.body.success).toBe(false);- expect(failed.body.feedbackErrorCode).toBe("SEARCH_FAILED");- } finally {- await db- .delete(schema.searches)- .where(- and(- eq(schema.searches.id, searchId),- eq(schema.searches.team_id, identity.teamId),- ),- );- }+ // The feedback policy reads the job's outcome from the feedback job+ // store, so a failed search is a failed feedback job row.+ expect(+ await writeFeedbackJob({+ endpoint: "search",+ jobId: searchId,+ requestId: searchId,+ teamId: identity.teamId,+ succeeded: false,+ creditsBilled: 2,+ zeroDataRetention: false,+ }),+ ).toBe(true);++ const failed = await endpointFeedbackRaw(+ {+ endpoint: "search",+ jobId: searchId,+ rating: "bad",+ missingContent: [{ topic: "Results" }],+ },+ identity,+ );++ expect(failed.statusCode).toBe(409);+ expect(failed.body.success).toBe(false);+ expect(failed.body.feedbackErrorCode).toBe("SEARCH_FAILED");}, 30000);});diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.tsindex 83d84922b..1f3ede094 100644--- a/apps/api/src/__tests__/snips/v2/parse.test.ts+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts@@ -14,9 +14,7 @@ import request, {scrapeTimeout,TEST_API_URL,} from "./lib";-import { and, desc, eq } from "drizzle-orm";-import { db } from "../../../db/connection";-import * as schema from "../../../db/schema";+import { HAS_JOB_LOG, jobLogRows, waitForJobLogRow } from "../job-log";import { config } from "../../../config";import { getRedisConnection } from "../../../services/queue-service";@@ -715,7 +713,7 @@ describe("/v2/parse", () => {it("logs parse metadata into the parses table",async () => {- if (!config.USE_DB_AUTHENTICATION) return;+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;const filename = `parse-log-${Date.now()}.html`;await parse(@@ -732,38 +730,19 @@ describe("/v2/parse", () => {identity,);- const requestLog = await waitForSingleRow<{ id: string }>(async () => {- const data = await db- .select({ id: schema.requests.id })- .from(schema.requests)- .where(- and(- eq(schema.requests.team_id, identity.teamId),- eq(schema.requests.kind, "parse"),- eq(schema.requests.target_hint, filename),- ),- )- .orderBy(desc(schema.requests.created_at))- .limit(1);- return data[0] ?? null;- });+ const requestLog = await waitForJobLogRow<{ id: string }>(+ "requests",+ "team_id = {teamId: UUID} AND kind = 'parse' AND target_hint = {filename: String}",+ { teamId: identity.teamId, filename },+ );expect(requestLog).not.toBeNull();- const parseLog = await waitForSingleRow<{+ const parseLog = await waitForJobLogRow<{request_id: string;url: string;- }>(async () => {- const data = await db- .select({- request_id: schema.parses.request_id,- url: schema.parses.url,- })- .from(schema.parses)- .where(eq(schema.parses.request_id, requestLog!.id))- .orderBy(desc(schema.parses.created_at))- .limit(1);- return data[0] ?? null;+ }>("parses", "request_id = {requestId: UUID}", {+ requestId: requestLog!.id,});expect(parseLog).not.toBeNull();@@ -772,11 +751,12 @@ describe("/v2/parse", () => {`https://parse.firecrawl.dev/uploads/${encodeURIComponent(filename)}`,);- const scrapeRows = await db- .select({ id: schema.scrapes.id })- .from(schema.scrapes)- .where(eq(schema.scrapes.request_id, requestLog!.id))- .limit(1);+ const scrapeRows = await jobLogRows<{ id: string }>(+ "scrapes",+ "request_id = {requestId: UUID}",+ { requestId: requestLog!.id },+ { limit: 1 },+ );expect(scrapeRows).toHaveLength(0);},scrapeTimeout,diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex 64a433b09..6ad20c6ea 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -3,6 +3,7 @@ import { config } from "../../../config";import { describeIf, itIf, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchRaw } from "./lib";import { and, desc, eq, gt } from "drizzle-orm";+import { HAS_JOB_LOG, waitForJobLogRow } from "../job-log";import { db } from "../../../db/connection";import * as schema from "../../../db/schema";import { redisRateLimitClient } from "../../../services/rate-limiter";@@ -244,7 +245,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {});it("logs research origin from X-Origin and integration from query", async () => {- if (!config.USE_DB_AUTHENTICATION) return;+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;const identity = await idmux({name: "research/logs metadata",@@ -262,27 +263,14 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.statusCode).toBe(200);expect(res.body.success).toBe(true);- const requestLog = await waitForSingleRow<{+ const requestLog = await waitForJobLogRow<{origin: string | null;integration: string | null;- }>(async () => {- const data = await db- .select({- origin: schema.requests.origin,- integration: schema.requests.integration,- })- .from(schema.requests)- .where(- and(- eq(schema.requests.team_id, identity.teamId),- eq(schema.requests.kind, "research_paper_search"),- eq(schema.requests.target_hint, query),- ),- )- .orderBy(desc(schema.requests.created_at))- .limit(1);- return data[0] ?? null;- });+ }>(+ "requests",+ "team_id = {teamId: UUID} AND kind = 'research_paper_search' AND target_hint = {query: String}",+ { teamId: identity.teamId, query },+ );expect(requestLog).not.toBeNull();expect(requestLog?.origin).toBe("mcp");diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.tsindex 3db906a95..6611568bd 100644--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts@@ -2,6 +2,7 @@ import {type CostTrackingCall,getCostTrackingCalls,} from "../cost-tracking-helpers";+import { HAS_JOB_LOG } from "../job-log";import {ALLOW_TEST_SUITE_WEBSITE,concurrentIf,@@ -66,7 +67,7 @@ const JEV_MAY_APPLY =Number(process.env.BRANDING_JEV_ROLLOUT_PERCENT || 0) > 0);describe("Branding cost tracking", () => {- concurrentIf(TEST_PRODUCTION && !JEV_MAY_APPLY)(+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && !JEV_MAY_APPLY)("records the branding LLM call with its model and cost",async () => {const response = await scrape(@@ -90,7 +91,7 @@ describe("Branding cost tracking", () => {scrapeTimeout + 15000,);- concurrentIf(TEST_PRODUCTION)(+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG)("records no branding call when branding is not requested",async () => {const response = await scrape(@@ -117,7 +118,7 @@ const JEV_ON =!!process.env.TYPESAFE_API_KEY && process.env.BRANDING_JEV === "true";describe("Branding with Jev", () => {- concurrentIf(TEST_PRODUCTION && JEV_ON)(+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && JEV_ON)("answers branding with Jev and records its cost, not an LLM call",async () => {const response = await scrape(diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.tsindex e8fca4d52..9f4e73e66 100644--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts@@ -1,6 +1,7 @@-import { describeIf, TEST_PRODUCTION } from "../lib";+import { describeIf, testIf, TEST_PRODUCTION } from "../lib";import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";import { expectScrapeIsCleanedUp } from "../zdr-helpers";+import { HAS_JOB_LOG } from "../job-log";import { getJobFromGCS } from "../../../lib/gcs-jobs";import crypto from "crypto";@@ -73,7 +74,7 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {scrapeTimeout,);- test(+ testIf(HAS_JOB_LOG)("should treat lockdown as ZDR — no URL in DB, no GCS blob",async () => {const id = crypto.randomUUID();diff --git a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.tsindex 114ecae78..198e6dc14 100644--- a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts@@ -1,4 +1,5 @@import { getCostTrackingCalls } from "../cost-tracking-helpers";+import { HAS_JOB_LOG } from "../job-log";import { concurrentIf, HAS_XAI, TEST_PRODUCTION } from "../lib";import { scrape, scrapeTimeout, idmux, Identity } from "./lib";@@ -15,7 +16,7 @@ beforeAll(async () => {// X/Twitter scrapes go through xAI's X Search, which bills per profile and// post fetched.describe("X/Twitter profile scrape", () => {- concurrentIf(TEST_PRODUCTION && HAS_XAI)(+ concurrentIf(TEST_PRODUCTION && HAS_XAI && HAS_JOB_LOG)("returns the profile and its latest posts from a single profile lookup",async () => {const response = await scrape(diff --git a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts b/apps/api/src/__tests__/snips/v2/search-feedback.test.tsindex 036f0b14f..7ae446d49 100644--- a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts+++ b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts@@ -7,9 +7,14 @@ import {idmux,Identity,} from "./lib";-import { and, eq } from "drizzle-orm";+import { eq } from "drizzle-orm";import { db } from "../../../db/connection";import * as schema from "../../../db/schema";+import { clickhouseClient } from "../../../lib/clickhouse-client";+import {+ readFeedbackJob,+ writeFeedbackJob,+} from "../../../lib/feedback-job-store";let identity: Identity;let secondaryIdentity: Identity;@@ -47,19 +52,18 @@ describeIf(TEST_PRODUCTION)("Search feedback tests", () => {expect(typeof raw.body.id).toBe("string");expect((raw.body.data?.web ?? []).length).toBeGreaterThan(0);+ // The searches row lands in ClickHouse a few seconds after the publish.let searchRow: { options: unknown } | undefined;- for (let attempt = 0; attempt < 20 && !searchRow; attempt++) {- [searchRow] = await db- .select({ options: schema.searches.options })- .from(schema.searches)- .where(- and(- eq(schema.searches.id, raw.body.id),- eq(schema.searches.team_id, identity.teamId),- ),- )- .limit(1);- if (!searchRow) await new Promise(resolve => setTimeout(resolve, 100));+ for (let attempt = 0; attempt < 30 && !searchRow; attempt++) {+ const result = await clickhouseClient!.query({+ query:+ "SELECT options FROM searches FINAL WHERE id = {id: UUID} AND team_id = {teamId: UUID} AND created_at >= now() - INTERVAL 1 DAY",+ query_params: { id: raw.body.id, teamId: identity.teamId },+ format: "JSONEachRow",+ });+ const [row] = await result.json<{ options: string | null }>();+ if (row) searchRow = { options: JSON.parse(row.options ?? "null") };+ else await new Promise(resolve => setTimeout(resolve, 1000));}expect(searchRow?.options).toMatchObject({objective: "Find official Firecrawl information",@@ -450,17 +454,26 @@ describeIf(TEST_PRODUCTION)("Search feedback tests", () => {expect(raw.statusCode).toBe(200);const searchId = raw.body.id;- await new Promise(r => setTimeout(r, 750));- const aged = new Date(Date.now() - 60 * 60 * 1000).toISOString();- await db- .update(schema.searches)- .set({ created_at: aged })- .where(- and(- eq(schema.searches.id, searchId),- eq(schema.searches.team_id, identity.teamId),- ),- );+ // Back-date the feedback job (its deadline enforces the window) rather+ // than waiting out the configured window.+ let job = await readFeedbackJob(searchId);+ for (let attempt = 0; attempt < 30 && !job; attempt++) {+ await new Promise(r => setTimeout(r, 1000));+ job = await readFeedbackJob(searchId);+ }+ expect(job).not.toBeNull();+ expect(+ await writeFeedbackJob({+ endpoint: "search",+ jobId: searchId,+ requestId: job!.requestId,+ teamId: identity.teamId,+ succeeded: job!.succeeded,+ creditsBilled: job!.creditsBilled,+ zeroDataRetention: job!.zeroDataRetention,+ completedAt: new Date(Date.now() - 60 * 60 * 1000),+ }),+ ).toBe(true);const failed = await searchFeedbackWithFailure(searchId,diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex e9c5a66a0..ecb18b887 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -8,7 +8,7 @@ import {idmux,searchRaw,} from "./lib";-import { describeIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";+import { describeIf, itIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";import {getLogs,expectScrapeIsCleanedUp,@@ -17,9 +17,11 @@ import {expectScrapesAreFullyCleanedAfterZDRCleaner,expectBatchScrapeIsCleanedUp,} from "../zdr-helpers";+import { HAS_JOB_LOG } from "../job-log";describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {- describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {+ const scopes = ["Team-scoped", "Request-scoped"] as const;+ describeIf(HAS_JOB_LOG).each(scopes)("%s", scope => {it("should clean up a scrape immediately", async () => {let identity = await idmux({name: `zdr/${scope}/scrape`,@@ -150,7 +152,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {);});- it(+ itIf(HAS_JOB_LOG)("should allow screenshots and clean them up",async () => {const identity = await idmux({@@ -178,7 +180,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {scrapeTimeout,);- it(+ itIf(HAS_JOB_LOG)("should allow pdf actions and clean them up",async () => {const identity = await idmux({diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.tsindex 35da88409..419cbf948 100644--- a/apps/api/src/__tests__/snips/zdr-helpers.ts+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts@@ -1,8 +1,6 @@import { readFile, stat } from "node:fs/promises";-import { eq } from "drizzle-orm";-import { db } from "../../db/connection";-import * as schema from "../../db/schema";import { getJobFromGCS } from "../../lib/gcs-jobs";+import { waitForJobLogRows } from "./job-log";type Identity = { apiKey: string; teamId: string };type ScrapeStatusRawFn = (@@ -87,72 +85,52 @@ export async function getLogs() {);}+type JobRow = Record<string, unknown>;++// Scrape and crawl options are stored as a JSON string; ZDR rows carry none.+function expectNoStoredContent(row: JobRow) {+ expect(String(row.url)).not.toContain("://"); // no url stored+ expect(row.options ?? null).toBeNull();+}+export async function expectScrapeIsCleanedUp(scrapeId: string) {- const scrapeData = await db- .select()- .from(schema.scrapes)- .where(eq(schema.scrapes.id, scrapeId))- .limit(1);+ const scrapeData = await waitForJobLogRows("scrapes", "id = {id: UUID}", {+ id: scrapeId,+ });expect(scrapeData).toHaveLength(1);-- if (scrapeData.length === 1) {- const record = scrapeData[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.options).toBeNull();- }+ expectNoStoredContent(scrapeData[0]);}export async function expectCrawlIsCleanedUp(crawlId: string) {- const requestData = await db- .select()- .from(schema.requests)- .where(eq(schema.requests.id, crawlId))- .limit(1);+ const requestData = await waitForJobLogRows("requests", "id = {id: UUID}", {+ id: crawlId,+ });expect(requestData).toHaveLength(1);+ expect(requestData[0].kind).toBe("crawl");- if (requestData.length === 1) {- const record = requestData[0];- expect(record.kind).toBe("crawl");- expect(record.dr_clean_by).not.toBeNull();- }-- const crawlData = await db- .select()- .from(schema.crawls)- .where(eq(schema.crawls.id, crawlId))- .limit(1);+ const crawlData = await waitForJobLogRows("crawls", "id = {id: UUID}", {+ id: crawlId,+ });expect(crawlData).toHaveLength(1);-- if (crawlData.length === 1) {- const record = crawlData[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.options).toBeNull();- }+ expectNoStoredContent(crawlData[0]);}export async function expectBatchScrapeIsCleanedUp(batchScrapeId: string) {- const requestData = await db- .select()- .from(schema.requests)- .where(eq(schema.requests.id, batchScrapeId))- .limit(1);+ const requestData = await waitForJobLogRows("requests", "id = {id: UUID}", {+ id: batchScrapeId,+ });expect(requestData).toHaveLength(1);+ expect(requestData[0].kind).toBe("batch_scrape");- if (requestData.length === 1) {- const record = requestData[0];- expect(record.kind).toBe("batch_scrape");- expect(record.dr_clean_by).not.toBeNull();- }-- const batchScrapeData = await db- .select()- .from(schema.batch_scrapes)- .where(eq(schema.batch_scrapes.id, batchScrapeId))- .limit(1);+ const batchScrapeData = await waitForJobLogRows(+ "batch_scrapes",+ "id = {id: UUID}",+ { id: batchScrapeId },+ );expect(batchScrapeData).toHaveLength(1);}@@ -161,10 +139,12 @@ export async function expectScrapesOfRequestAreCleanedUp(requestId: string,expectedScrapeCount?: number,) {- const scrapes = await db- .select()- .from(schema.scrapes)- .where(eq(schema.scrapes.request_id, requestId));+ const scrapes = await waitForJobLogRows(+ "scrapes",+ "request_id = {requestId: UUID}",+ { requestId },+ { min: expectedScrapeCount ?? 1 },+ );if (expectedScrapeCount !== undefined) {expect(scrapes.length).toBe(expectedScrapeCount);@@ -173,11 +153,10 @@ export async function expectScrapesOfRequestAreCleanedUp(}for (const scrape of scrapes) {- expect(scrape.url).not.toContain("://"); // no url stored- expect(scrape.options).toBeNull();+ expectNoStoredContent(scrape);if (scrape.is_successful) {- const gcsJob = await getJobFromGCS(scrape.id);+ const gcsJob = await getJobFromGCS(String(scrape.id));expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry}}diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 88ba299d0..708aa1c5c 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -363,27 +363,24 @@ export async function searchController(const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;- logSearch(- {- id: jobId,- request_id: jobId,- query: req.body.query,- is_successful: true,- error: undefined,- results: responseData.data,- num_results: responseData.data.length,- time_taken: timeTakenInSeconds,- team_id: req.auth.team_id,- options: {- ...req.body,- query: undefined,- scrapeOptions: undefined,- },- credits_cost: result.searchCredits,- zeroDataRetention,+ logSearch({+ id: jobId,+ request_id: jobId,+ query: req.body.query,+ is_successful: true,+ error: undefined,+ results: responseData.data,+ num_results: responseData.data.length,+ time_taken: timeTakenInSeconds,+ team_id: req.auth.team_id,+ options: {+ ...req.body,+ query: undefined,+ scrapeOptions: undefined,},- false,- ).catch(error => {+ credits_cost: result.searchCredits,+ zeroDataRetention,+ }).catch(error => {logger.error("Failed to log search", { error, jobId });});diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 67714a8f5..cc9d4e17a 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -480,25 +480,22 @@ async function searchControllerInner(const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;- logSearch(- {- id: jobId,- request_id: agentRequestId ?? jobId,- query: req.body.query,- is_successful: true,- error: undefined,- results: result.response as any,- num_results: result.totalResultsCount,- time_taken: timeTakenInSeconds,- team_id: req.auth.team_id,- options: req.body,- // Don't record preview tokens as billed in the ledger — only record- // credits when billing is actually applied.- credits_cost: !isSearchPreview && shouldBill ? result.searchCredits : 0,- zeroDataRetention,- },- false,- ).catch(error => {+ logSearch({+ id: jobId,+ request_id: agentRequestId ?? jobId,+ query: req.body.query,+ is_successful: true,+ error: undefined,+ results: result.response as any,+ num_results: result.totalResultsCount,+ time_taken: timeTakenInSeconds,+ team_id: req.auth.team_id,+ options: req.body,+ // Don't record preview tokens as billed in the ledger — only record+ // credits when billing is actually applied.+ credits_cost: !isSearchPreview && shouldBill ? result.searchCredits : 0,+ zeroDataRetention,+ }).catch(error => {logger.error("Failed to log search", { error, jobId });});diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 569fdb273..74bc02da7 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -84,16 +84,6 @@ export const api_keys = pgTable(],);-export const batch_scrapes = pgTable("batch_scrapes", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- team_id: uuid("team_id").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- num_docs: integer("num_docs").notNull(),- credits_cost: integer("credits_cost").notNull(),- cancelled: boolean("cancelled").notNull(),-});-export const blocklist = pgTable("blocklist", {id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),data: jsonb("data").notNull(),@@ -163,72 +153,6 @@ export const browser_profiles = pgTable(table => [primaryKey({ columns: [table.team_id, table.name] })],);-export const crawls = pgTable("crawls", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- url: text("url").notNull(),- team_id: uuid("team_id").notNull(),- options: jsonb("options"),- created_at: ts("created_at").notNull().defaultNow(),- num_docs: integer("num_docs").notNull(),- credits_cost: integer("credits_cost").notNull(),- cancelled: boolean("cancelled").notNull(),- monitor_id: uuid("monitor_id"),- monitor_check_id: uuid("monitor_check_id"),-});--export const deep_researches = pgTable("deep_researches", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- query: text("query").notNull(),- team_id: uuid("team_id").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- time_taken: num("time_taken").notNull(),- credits_cost: integer("credits_cost").notNull(),- cost_tracking: jsonb("cost_tracking"),- options: jsonb("options"),-});--const researchEndpointTable = (name: string) =>- pgTable(name, {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- target: text("target").notNull(),- team_id: uuid("team_id").notNull(),- options: jsonb("options"),- response: jsonb("response"),- num_results: integer("num_results").notNull(),- time_taken: num("time_taken").notNull(),- credits_cost: integer("credits_cost").notNull(),- is_successful: boolean("is_successful").notNull(),- error: text("error"),- created_at: ts("created_at").notNull().defaultNow(),- });--export const research_paper_searches = researchEndpointTable(- "research_paper_searches",-);--export const research_paper_inspects = researchEndpointTable(- "research_paper_inspects",-);--export const research_paper_reads = researchEndpointTable(- "research_paper_reads",-);--export const research_related_papers = researchEndpointTable(- "research_related_papers",-);--export const research_github_searches = researchEndpointTable(- "research_github_searches",-);--export const code_searches = researchEndpointTable("code_searches");--export const gov_searches = researchEndpointTable("gov_searches");-export const deterministic_json_scripts = pgTable("deterministic_json_scripts",{@@ -261,20 +185,6 @@ export const eb_sync = pgTable("eb-sync", {team_id: text("team_id"),});-export const extracts = pgTable("extracts", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- urls: text("urls").array().notNull(),- options: jsonb("options"),- model_kind: text("model_kind").notNull(),- team_id: uuid("team_id").notNull(),- is_successful: boolean("is_successful").notNull(),- error: text("error"),- created_at: ts("created_at").notNull().defaultNow(),- credits_cost: integer("credits_cost").notNull(),- cost_tracking: jsonb("cost_tracking"),-});-export const idempotency_keys = pgTable("idempotency_keys", {key: uuid("key").notNull().defaultRandom(),created_at: ts("created_at").notNull().defaultNow(),@@ -340,29 +250,6 @@ export const llm_texts = pgTable("llm_texts", {updated_at: ts("updated_at"),});-export const llmstxts = pgTable("llmstxts", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- url: text("url").notNull(),- team_id: uuid("team_id").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- num_urls: integer("num_urls").notNull(),- options: jsonb("options"),- cost_tracking: jsonb("cost_tracking"),- credits_cost: integer("credits_cost").notNull(),-});--export const maps = pgTable("maps", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- url: text("url").notNull(),- options: jsonb("options"),- team_id: uuid("team_id").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- num_results: integer("num_results").notNull(),- credits_cost: integer("credits_cost").notNull(),-});-export const monitor_check_pages = pgTable("monitor_check_pages", {id: uuid("id").notNull(),check_id: uuid("check_id").notNull(),@@ -506,21 +393,6 @@ export const notification_preferences = pgTable("notification_preferences", {unsubscribed_all: boolean("unsubscribed_all").default(false),});-export const parses = pgTable("parses", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- url: text("url").notNull(),- is_successful: boolean("is_successful").notNull(),- error: text("error"),- time_taken: num("time_taken").notNull(),- team_id: uuid("team_id").notNull(),- options: jsonb("options"),- cost_tracking: jsonb("cost_tracking"),- pdf_num_pages: integer("pdf_num_pages"),- credits_cost: integer("credits_cost").notNull(),- created_at: ts("created_at").notNull().defaultNow(),-});-export const prices = pgTable("prices", {id: text("id").notNull(),product_id: text("product_id"),@@ -570,20 +442,6 @@ export const products = pgTable("products", {type: text("type"),});-export const requests = pgTable("requests", {- id: uuid("id").notNull(),- kind: text("kind").notNull(),- api_version: text("api_version").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- team_id: uuid("team_id").notNull(),- origin: text("origin").notNull(),- integration: text("integration"),- target_hint: text("target_hint").notNull(),- dr_clean_by: ts("dr_clean_by"),- api_key_id: bigintNum("api_key_id"),- external_request_id: text("external_request_id"),-});-export const mcp_action_logs = pgTable("mcp_action_logs",{@@ -616,24 +474,6 @@ export const mcp_action_logs = pgTable(],);-export const scrapes = pgTable("scrapes", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- url: text("url").notNull(),- is_successful: boolean("is_successful").notNull(),- error: text("error"),- time_taken: num("time_taken").notNull(),- team_id: uuid("team_id").notNull(),- options: jsonb("options"),- cost_tracking: jsonb("cost_tracking"),- pdf_num_pages: integer("pdf_num_pages"),- credits_cost: integer("credits_cost").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- monitor_id: uuid("monitor_id"),- monitor_check_id: uuid("monitor_check_id"),- content_type: text("content_type"),-});-export const search_feedback = pgTable("search_feedback", {id: uuid("id").notNull().defaultRandom(),search_id: uuid("search_id"),@@ -760,20 +600,6 @@ export const alexandria_feedback_capabilities = pgTable(],);-export const searches = pgTable("searches", {- id: uuid("id").notNull(),- request_id: uuid("request_id").notNull(),- query: text("query").notNull(),- team_id: uuid("team_id").notNull(),- options: jsonb("options"),- time_taken: num("time_taken").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- credits_cost: integer("credits_cost").notNull(),- is_successful: boolean("is_successful").notNull(),- error: text("error"),- num_results: integer("num_results").notNull(),-});-export const subscriptions = pgTable("subscriptions", {id: text("id").notNull(),user_id: uuid("user_id").notNull(),diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 59098750e..94771897e 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -4,8 +4,6 @@ import { vi } from "vitest";// vi.hoisted() (also hoisted).const {logger,- values,- insert,topic,publishes,publishMessage,@@ -29,8 +27,6 @@ const {debug: vi.fn(),child: vi.fn(() => logger),};- const values = vi.fn<(data: any) => Promise<void>>();- const insert = vi.fn(() => ({ values }));const publishMessage = vi.fn(async (_message: any) => "message-id");const flush = vi.fn(async () => {});const close = vi.fn(async () => {});@@ -53,8 +49,6 @@ const {});return {logger,- values,- insert,topic,publishes,publishMessage,@@ -97,10 +91,6 @@ vi.mock("../../lib/logger", () => ({logger,}));-vi.mock("../../db/connection", () => ({- db: { insert },-}));-vi.mock("../../lib/change-tracking-store", () => ({changeTrackingInsertScrape: vi.fn(),}));@@ -163,7 +153,6 @@ import {shutdownPubSubLogging,type LoggedSearch,} from "./log_job";-import * as schema from "../../db/schema";import { config } from "../../config";function deferred<T>() {@@ -199,7 +188,6 @@ function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {describe("logSearch", () => {beforeEach(() => {vi.clearAllMocks();- values.mockResolvedValue(undefined);publishMessage.mockResolvedValue("message-id");publishes.length = 0;spans.length = 0;@@ -216,8 +204,9 @@ describe("logSearch", () => {await logSearch(search);- expect(insert).toHaveBeenCalledWith(schema.searches);- const inserted = values.mock.calls[0][0];+ const inserted = JSON.parse(+ publishMessage.mock.calls[0][0].data.toString(),+ );expect(inserted.query).toBe("helloworld");expect(inserted.options.query).toBe("nestedquery");expect(inserted.options.sources[0].location).toBe("NewYork");@@ -241,7 +230,6 @@ describe("logSearch", () => {await logSearch(makeSearch({ query: options.query, options }));- expect(values.mock.calls[0][0].options).toMatchObject(options);const published = JSON.parse(publishMessage.mock.calls[0][0].data.toString(),);@@ -261,8 +249,6 @@ describe("logSearch", () => {}),);- expect(values.mock.calls[0][0].options).not.toHaveProperty("objective");- expect(values.mock.calls[0][0].options).not.toHaveProperty("clientModel");const published = JSON.parse(publishMessage.mock.calls[0][0].data.toString(),);@@ -270,12 +256,11 @@ describe("logSearch", () => {expect(published.options).not.toHaveProperty("clientModel");});- it("fails the log call on a serialization failure before touching PostgreSQL", async () => {+ it("fails the log call on a serialization failure", async () => {const search = makeSearch({ options: { unsupported: 1n } });await expect(logSearch(search)).rejects.toThrow();- expect(values).not.toHaveBeenCalled();expect(publishMessage).not.toHaveBeenCalled();expect(metricInc).toHaveBeenCalledWith({table: "searches",@@ -292,7 +277,7 @@ describe("logSearch", () => {await expect(logSearch(makeSearch())).resolves.toBeUndefined();- expect(values).toHaveBeenCalledOnce();+ expect(publishMessage).toHaveBeenCalledOnce();expect(logger.error).toHaveBeenCalledWith("Failed to write feedback job to Bigtable",expect.objectContaining({@@ -302,13 +287,12 @@ describe("logSearch", () => {);});- it("profiles the log, PostgreSQL insert, and Pub/Sub publish", async () => {+ it("profiles the log and the Pub/Sub publish", async () => {await logSearch(makeSearch());expect(spans).toEqual(expect.arrayContaining([{ name: "log_job.search", options: { zeroDataRetention: false } },- { name: "log_job.postgres.insert", options: undefined },{ name: "log_job.pubsub.publish", options: undefined },]),);@@ -327,7 +311,6 @@ describe("logSearch", () => {describe("operational job state logging", () => {beforeEach(() => {vi.clearAllMocks();- values.mockResolvedValue(undefined);publishMessage.mockResolvedValue("message-id");});@@ -356,13 +339,13 @@ describe("operational job state logging", () => {);});- it("reports the state written before the PostgreSQL insert starts", async () => {+ it("reports the state written before the Pub/Sub publish starts", async () => {const id = "019e6f45-7778-727d-adf0-0abe9d5062b8";const stateWrite = deferred<boolean>();writeScrapeJobState.mockReturnValueOnce(stateWrite.promise);let insertsWhenStateWritten = -1;const onStateWritten = vi.fn(() => {- insertsWhenStateWritten = values.mock.calls.length;+ insertsWhenStateWritten = publishMessage.mock.calls.length;});const logging = logScrape(@@ -378,21 +361,20 @@ describe("operational job state logging", () => {skipNuq: true,zeroDataRetention: false,},- false,{ onStateWritten },);await Promise.resolve();expect(writeScrapeJobState).toHaveBeenCalledTimes(1);expect(onStateWritten).not.toHaveBeenCalled();- expect(values).not.toHaveBeenCalled();+ expect(publishMessage).not.toHaveBeenCalled();stateWrite.resolve(true);await logging;expect(onStateWritten).toHaveBeenCalledWith("written");expect(insertsWhenStateWritten).toBe(0);- expect(values).toHaveBeenCalledTimes(1);+ expect(publishMessage).toHaveBeenCalledTimes(1);});it("reports a failed state write and keeps logging", async () => {@@ -413,12 +395,11 @@ describe("operational job state logging", () => {skipNuq: true,zeroDataRetention: false,},- false,{ onStateWritten },);expect(onStateWritten).toHaveBeenCalledWith("failed");- expect(values).toHaveBeenCalledTimes(1);+ expect(publishMessage).toHaveBeenCalledTimes(1);});it("reports a skipped state write for a parse, which stores none", async () => {@@ -439,7 +420,6 @@ describe("operational job state logging", () => {zeroDataRetention: false,is_parse: true,},- false,{ onStateWritten },);@@ -465,7 +445,6 @@ describe("operational job state logging", () => {skipNuq: true,zeroDataRetention: false,},- false,{ onStateWritten },);@@ -558,7 +537,6 @@ describe("operational job state logging", () => {describe("logRequest", () => {beforeEach(() => {vi.clearAllMocks();- values.mockResolvedValue(undefined);publishMessage.mockResolvedValue("message-id");publishes.length = 0;});@@ -580,10 +558,10 @@ describe("logRequest", () => {it("stores the caller's external_request_id verbatim", async () => {await logRequest(makeRequest("op_integration_42"));- expect(insert).toHaveBeenCalledWith(schema.requests);- expect(values.mock.calls[0][0].external_request_id).toBe(- "op_integration_42",+ const published = JSON.parse(+ publishMessage.mock.calls[0][0].data.toString(),);+ expect(published.external_request_id).toBe("op_integration_42");});it("prefixes the Pub/Sub topic name with PUBSUB_TOPIC_PREFIX", async () => {@@ -607,10 +585,9 @@ describe("logRequest", () => {);});- it("writes the request to the database and its Pub/Sub topic", async () => {+ it("publishes the request to its Pub/Sub topic", async () => {await logRequest(makeRequest("op_integration_42"));- expect(insert).toHaveBeenCalledWith(schema.requests);expect(publishes[0].name).toBe("requests");const gaxOpts = publishes[0].options.gaxOpts;// A bare `timeout` would collapse the retry budget to one attempt.@@ -631,9 +608,6 @@ describe("logRequest", () => {expect(new Date(published.created_at).toISOString()).toBe(published.created_at,);- expect(values.mock.calls[0][0].created_at.toISOString()).toBe(- published.created_at,- );expect(writeApiJobAccess).toHaveBeenCalledWith(expect.objectContaining({id: "019e6f45-7778-727d-adf0-0abe9d5062b6",@@ -645,7 +619,7 @@ describe("logRequest", () => {);});- it("durably queues ZDR cleanup before writing the request", async () => {+ it("durably queues ZDR cleanup before publishing the request", async () => {await logRequest({...makeRequest("op_integration_42"),zeroDataRetention: true,@@ -655,12 +629,15 @@ describe("logRequest", () => {"019e6f45-7778-727d-adf0-0abe9d5062b6",);expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(- values.mock.invocationCallOrder[0],+ publishMessage.mock.invocationCallOrder[0],+ );+ const published = JSON.parse(+ publishMessage.mock.calls[0][0].data.toString(),);- expect(values.mock.calls[0][0]).not.toHaveProperty("dr_clean_by");+ expect(published).not.toHaveProperty("dr_clean_by");});- it("does not write a ZDR request unless its cleanup job was confirmed", async () => {+ it("does not publish a ZDR request unless its cleanup job was confirmed", async () => {enqueueZdrCleanupJob.mockRejectedValueOnce(new Error("RabbitMQ unavailable"),);@@ -668,17 +645,16 @@ describe("logRequest", () => {await expect(logRequest({ ...makeRequest(null), zeroDataRetention: true }),).rejects.toThrow("RabbitMQ unavailable");- expect(values).not.toHaveBeenCalled();+ expect(publishMessage).not.toHaveBeenCalled();});- it("fails the log call and skips PostgreSQL when Pub/Sub fails", async () => {+ it("fails the log call when Pub/Sub fails", async () => {publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));await expect(logRequest(makeRequest(null))).rejects.toThrow("Pub/Sub unavailable",);- expect(values).not.toHaveBeenCalled();expect(logger.error).toHaveBeenCalledWith("Failed to publish log to Pub/Sub",expect.objectContaining({ error: expect.any(Error) }),@@ -690,7 +666,7 @@ describe("logRequest", () => {await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();- expect(values).toHaveBeenCalled();+ expect(publishMessage).toHaveBeenCalled();expect(logger.error).toHaveBeenCalledWith("Failed to write API job access to Bigtable",expect.objectContaining({ error: expect.any(Error), kind: "scrape" }),@@ -701,7 +677,7 @@ describe("logRequest", () => {await logRequest({ ...makeRequest(null), jobAccess: false });expect(writeApiJobAccess).not.toHaveBeenCalled();- expect(values).toHaveBeenCalledOnce();+ expect(publishMessage).toHaveBeenCalledOnce();});it("uses a job's explicit operational expiry", async () => {@@ -728,44 +704,27 @@ describe("logRequest", () => {await new Promise(resolve => setImmediate(resolve));expect(publishMessage).toHaveBeenCalledOnce();- expect(values).not.toHaveBeenCalled();expect(finished).toBe(false);publication.resolve("message-id");await logging;- expect(values).toHaveBeenCalled();expect(metricInc).toHaveBeenCalledWith({table: "requests",outcome: "published",});});- it("waits for PostgreSQL when publication finishes first", async () => {- const insertion = deferred<void>();- values.mockReturnValueOnce(insertion.promise);- let finished = false;- const logging = logRequest(makeRequest(null)).then(() => {- finished = true;- });-- await new Promise(resolve => setImmediate(resolve));- expect(publishMessage).toHaveBeenCalledOnce();- expect(finished).toBe(false);-- insertion.resolve();- await logging;- expect(finished).toBe(true);- });-it("stores null, not a truncation, when the id exceeds the byte cap", async () => {// The header helper already drops these; this asserts the bound holds at- // the insert boundary too, for any writer that bypasses the helper. Null- // rather than a DB constraint, which would fail the whole requests row+ // the log boundary too, for any writer that bypasses the helper. Null+ // rather than a rejected row, which would lose the whole requests row// (and its scrapes/crawls children) over a telemetry field — and null// rather than truncation, which would hand a wrong id back downstream.await logRequest(makeRequest("x".repeat(2049)));- const inserted = values.mock.calls[0][0];+ const inserted = JSON.parse(+ publishMessage.mock.calls[0][0].data.toString(),+ );expect(inserted.external_request_id).toBeNull();expect(inserted.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");expect(logger.warn).toHaveBeenCalled();@@ -774,18 +733,19 @@ describe("logRequest", () => {it("counts the cap in bytes, not characters", async () => {// 1025 two-byte characters: 1025 chars, 2050 bytes — over.await logRequest(makeRequest("é".repeat(1025)));- expect(values.mock.calls[0][0].external_request_id).toBeNull();+ const first = JSON.parse(publishMessage.mock.calls[0][0].data.toString());+ expect(first.external_request_id).toBeNull();// 1024 two-byte characters: 2048 bytes exactly — allowed.await logRequest(makeRequest("é".repeat(1024)));- expect(values.mock.calls[1][0].external_request_id).toBe("é".repeat(1024));+ const second = JSON.parse(publishMessage.mock.calls[1][0].data.toString());+ expect(second.external_request_id).toBe("é".repeat(1024));});- it("cleans NUL bytes and unpaired surrogates for both stores", async () => {+ it("cleans NUL bytes and unpaired surrogates before publishing", async () => {// "Łódź" mis-decoded by a client arrives as a lone low surrogate, which// JSON.stringify would emit as "\udc81" and ClickPipes would reject as- // invalid JSON; PostgreSQL's driver stores it as U+FFFD. The row must- // reach both stores already cleaned, and identical.+ // invalid JSON. The row must be published already cleaned.const replacement = String.fromCharCode(0xfffd);await logRequest({...makeRequest(null),@@ -793,15 +753,11 @@ describe("logRequest", () => {origin: "api" + String.fromCharCode(0),});- const inserted = values.mock.calls[0][0];- expect(inserted.target_hint).toBe("wyciek Å" + replacement + "ódź!");- expect(inserted.origin).toBe("api");-const raw = publishMessage.mock.calls[0][0].data.toString("utf8");expect(raw).not.toMatch(/\\u[dD][89a-fA-F]/);expect(raw).not.toMatch(/\\u0{4}/);const published = JSON.parse(raw);- expect(published.target_hint).toBe(inserted.target_hint);+ expect(published.target_hint).toBe("wyciek Å" + replacement + "ódź!");expect(published.origin).toBe("api");});@@ -826,8 +782,6 @@ describe("logRequest", () => {await fresh.shutdownPubSubLogging();}- // The refused row never reaches PostgreSQL either.- expect(values).toHaveBeenCalledTimes(2);expect(publishMessage).toHaveBeenCalledTimes(2);expect(metricInc).toHaveBeenCalledWith({table: "requests",@@ -900,7 +854,6 @@ describe("logRequest", () => {describe("shutdownPubSubLogging deadline", () => {beforeEach(() => {vi.clearAllMocks();- values.mockResolvedValue(undefined);publishMessage.mockResolvedValue("message-id");publishes.length = 0;});@@ -1007,7 +960,6 @@ describe("shutdownPubSubLogging deadline", () => {"shutting down",);expect(publishMessage).toHaveBeenCalledOnce();- expect(values).toHaveBeenCalledTimes(1);expect(metricInc).toHaveBeenCalledWith({table: "searches",outcome: "failed",diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 2087f536b..9f0f332ea 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -1,5 +1,3 @@-import { db } from "../../db/connection";-import * as schema from "../../db/schema";import { changeTrackingInsertScrape } from "../../lib/change-tracking-store";import { config } from "../../config";import { enqueueZdrCleanupJob } from "../../lib/zdr-queue";@@ -7,7 +5,6 @@ import "dotenv/config";import { logger as _logger } from "../../lib/logger";import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";import { configDotenv } from "dotenv";-import type { PgTable } from "drizzle-orm/pg-core";import {saveDeepResearchToGCS,saveExtractToGCS,@@ -55,7 +52,6 @@ async function withLogSpan<T>(table: string;id: string;requestId?: string;- force?: boolean;zeroDataRetention?: boolean;},fn: () => Promise<T>,@@ -67,7 +63,6 @@ async function withLogSpan<T>("log_job.table": params.table,"log_job.id": params.id,"log_job.request_id": params.requestId,- "log_job.force": params.force,});return fn();},@@ -119,26 +114,6 @@ function sanitizeString(value: string | null | undefined): string | null {return sanitizeText(value);}-const tableMap: Record<string, PgTable> = {- requests: schema.requests,- scrapes: schema.scrapes,- parses: schema.parses,- crawls: schema.crawls,- batch_scrapes: schema.batch_scrapes,- searches: schema.searches,- research_paper_searches: schema.research_paper_searches,- research_paper_inspects: schema.research_paper_inspects,- research_paper_reads: schema.research_paper_reads,- research_related_papers: schema.research_related_papers,- research_github_searches: schema.research_github_searches,- code_searches: schema.code_searches,- gov_searches: schema.gov_searches,- extracts: schema.extracts,- maps: schema.maps,- llmstxts: schema.llmstxts,- deep_researches: schema.deep_researches,-};-let pubSubClient: PubSub | null | undefined;const pubSubTopics = new Map<string, Topic>();let pubSubShutdown: Promise<void> | undefined;@@ -399,12 +374,7 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {}}-async function robustInsert(- table: string,- data: any,- force: boolean,- _logger: Logger,-) {+async function robustInsert(table: string, data: any, _logger: Logger) {const logger = _logger.child({module: "log_job",method: "robustInsert",@@ -413,84 +383,19 @@ async function robustInsert(});if (config.USE_DB_AUTHENTICATION !== true) {- logger.info(- "Skipping database insertion due to USE_DB_AUTHENTICATION being off",- );+ logger.info("Skipping job log due to USE_DB_AUTHENTICATION being off");return;}- // The single point where a row leaves for both stores: clean it once so- // PostgreSQL and ClickHouse receive identical, accepted values.+ // Clean the row once so the log receives accepted values.data = sanitizeLogData({...data,created_at: data.created_at ?? new Date(),});- // Pub/Sub is the store of record: publish first and wait for it. A failed- // publish fails the log call, and the PostgreSQL copy is not attempted.+ // Pub/Sub is the store of record: publish and wait for it. A failed+ // publish fails the log call.await publishLog(table, data, logger);-- // The PostgreSQL copy is on its way out; its failure is logged, not thrown.- const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];- try {- await withSpan("log_job.postgres.insert", async span => {- setSpanAttributes(span, {- "db.system": "postgresql",- "log_job.table": table,- "log_job.id": data.id,- "log_job.force": force,- "log_job.postgres.enabled": true,- });- const target = tableMap[table];-- const maxAttempts = force ? 10 : 1;- for (let i = 0; i < maxAttempts; i++) {- const backoffMs = i === 0 ? 0 : 75;- const start = Date.now();- try {- await db.insert(target).values(data);- attempts.push({- error: null,- timeMs: Date.now() - start,- backoffMs,- });- break;- } catch (error) {- attempts.push({- error,- timeMs: Date.now() - start,- backoffMs,- });- if (force) {- await new Promise(resolve => setTimeout(resolve, 75));- }- }- }-- const lastAttempt = attempts.at(-1);- setSpanAttributes(span, {- "log_job.postgres.attempts": attempts.length,- "log_job.postgres.retries": Math.max(0, attempts.length - 1),- "log_job.postgres.outcome":- lastAttempt?.error === null ? "inserted" : "failed",- });- if (lastAttempt?.error !== null) {- throw (- lastAttempt?.error ?? new Error("Database insert was not attempted")- );- }- });-- if (attempts.length === 1) {- logger.debug("Inserted into database successfully", { attempts });- } else {- logger.warn("Inserted into database successfully with retries", {- attempts,- });- }- } catch {- logger.error("Failed to insert into database", { attempts });- }}type LoggedRequest = {@@ -662,7 +567,6 @@ async function logRequestInternal(request: LoggedRequest) {logger,),},- true,logger,);}@@ -707,29 +611,20 @@ type LogScrapeHooks = {onStateWritten?: (outcome: ScrapeStateOutcome) => void;};-export async function logScrape(- scrape: LoggedScrape,- force: boolean = false,- hooks?: LogScrapeHooks,-) {+export async function logScrape(scrape: LoggedScrape, hooks?: LogScrapeHooks) {return withLogSpan({operation: scrape.is_parse ? "parse" : "scrape",table: scrape.is_parse ? "parses" : "scrapes",id: scrape.id,requestId: scrape.request_id,- force,zeroDataRetention: scrape.zeroDataRetention,},- () => logScrapeInternal(scrape, force, hooks),+ () => logScrapeInternal(scrape, hooks),);}-async function logScrapeInternal(- scrape: LoggedScrape,- force: boolean = false,- hooks?: LogScrapeHooks,-) {+async function logScrapeInternal(scrape: LoggedScrape, hooks?: LogScrapeHooks) {const logger = _logger.child({module: "log_job",method: "logScrape",@@ -829,7 +724,6 @@ async function logScrapeInternal(content_type: scrape.content_type ?? null,}),},- force,logger,);@@ -963,7 +857,6 @@ async function logProviderScrapeInternal(scrape: LoggedProviderScrape) {monitor_check_id: null,content_type: null,},- false,logger,);}@@ -982,21 +875,20 @@ type LoggedCrawl = {monitor_check_id?: string | null;};-export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {+export async function logCrawl(crawl: LoggedCrawl) {return withLogSpan({operation: "crawl",table: "crawls",id: crawl.id,requestId: crawl.request_id,- force,zeroDataRetention: crawl.zeroDataRetention,},- () => logCrawlInternal(crawl, force),+ () => logCrawlInternal(crawl),);}-async function logCrawlInternal(crawl: LoggedCrawl, force: boolean = false) {+async function logCrawlInternal(crawl: LoggedCrawl) {const logger = _logger.child({module: "log_job",method: "logCrawl",@@ -1025,7 +917,6 @@ async function logCrawlInternal(crawl: LoggedCrawl, force: boolean = false) {monitor_id: crawl.monitor_id ?? null,monitor_check_id: crawl.monitor_check_id ?? null,},- force,logger,);}@@ -1040,27 +931,20 @@ type LoggedBatchScrape = {cancelled: boolean;};-export async function logBatchScrape(- batchScrape: LoggedBatchScrape,- force: boolean = false,-) {+export async function logBatchScrape(batchScrape: LoggedBatchScrape) {return withLogSpan({operation: "batch_scrape",table: "batch_scrapes",id: batchScrape.id,requestId: batchScrape.request_id,- force,zeroDataRetention: batchScrape.zeroDataRetention,},- () => logBatchScrapeInternal(batchScrape, force),+ () => logBatchScrapeInternal(batchScrape),);}-async function logBatchScrapeInternal(- batchScrape: LoggedBatchScrape,- force: boolean = false,-) {+async function logBatchScrapeInternal(batchScrape: LoggedBatchScrape) {const logger = _logger.child({module: "log_job",method: "logBatchScrape",@@ -1084,7 +968,6 @@ async function logBatchScrapeInternal(credits_cost: batchScrape.credits_cost,cancelled: batchScrape.cancelled,},- force,logger,);}@@ -1104,21 +987,20 @@ export type LoggedSearch = {zeroDataRetention: boolean;};-export async function logSearch(search: LoggedSearch, force: boolean = false) {+export async function logSearch(search: LoggedSearch) {return withLogSpan({operation: "search",table: "searches",id: search.id,requestId: search.request_id,- force,zeroDataRetention: search.zeroDataRetention,},- () => logSearchInternal(search, force),+ () => logSearchInternal(search),);}-async function logSearchInternal(search: LoggedSearch, force: boolean = false) {+async function logSearchInternal(search: LoggedSearch) {const logger = _logger.child({module: "log_job",method: "logSearch",@@ -1168,7 +1050,6 @@ async function logSearchInternal(search: LoggedSearch, force: boolean = false) {num_results: search.num_results,time_taken: search.time_taken,},- force,logger,);@@ -1222,27 +1103,20 @@ type LoggedResearchEndpoint = {zeroDataRetention: boolean;};-export async function logResearchEndpoint(- research: LoggedResearchEndpoint,- force: boolean = false,-) {+export async function logResearchEndpoint(research: LoggedResearchEndpoint) {return withLogSpan({operation: "research",table: research.table,id: research.id,requestId: research.request_id,- force,zeroDataRetention: research.zeroDataRetention,},- () => logResearchEndpointInternal(research, force),+ () => logResearchEndpointInternal(research),);}-async function logResearchEndpointInternal(- research: LoggedResearchEndpoint,- force: boolean = false,-) {+async function logResearchEndpointInternal(research: LoggedResearchEndpoint) {const logger = _logger.child({module: "log_job",method: "logResearchEndpoint",@@ -1274,7 +1148,6 @@ async function logResearchEndpointInternal(is_successful: research.is_successful,error: research.zeroDataRetention ? null : (research.error ?? null),},- force,logger,);}@@ -1293,26 +1166,19 @@ export type LoggedExtract = {cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;};-export async function logExtract(- extract: LoggedExtract,- force: boolean = false,-) {+export async function logExtract(extract: LoggedExtract) {return withLogSpan({operation: "extract",table: "extracts",id: extract.id,requestId: extract.request_id,- force,},- () => logExtractInternal(extract, force),+ () => logExtractInternal(extract),);}-async function logExtractInternal(- extract: LoggedExtract,- force: boolean = false,-) {+async function logExtractInternal(extract: LoggedExtract) {const logger = _logger.child({module: "log_job",method: "logExtract",@@ -1338,7 +1204,6 @@ async function logExtractInternal(error: extract.error ?? null,cost_tracking: extract.cost_tracking ?? null,},- force,logger,);@@ -1374,21 +1239,20 @@ export type LoggedMap = {zeroDataRetention: boolean;};-export async function logMap(map: LoggedMap, force: boolean = false) {+export async function logMap(map: LoggedMap) {return withLogSpan({operation: "map",table: "maps",id: map.id,requestId: map.request_id,- force,zeroDataRetention: map.zeroDataRetention,},- () => logMapInternal(map, force),+ () => logMapInternal(map),);}-async function logMapInternal(map: LoggedMap, force: boolean = false) {+async function logMapInternal(map: LoggedMap) {const logger = _logger.child({module: "log_job",method: "logMap",@@ -1428,7 +1292,6 @@ async function logMapInternal(map: LoggedMap, force: boolean = false) {num_results: map.results.length,credits_cost: map.credits_cost,},- force,logger,);@@ -1449,26 +1312,19 @@ export type LoggedLlmsTxt = {result: { llmstxt: string; llmsfulltxt: string };};-export async function logLlmsTxt(- llmsTxt: LoggedLlmsTxt,- force: boolean = false,-) {+export async function logLlmsTxt(llmsTxt: LoggedLlmsTxt) {return withLogSpan({operation: "llmstxt",table: "llmstxts",id: llmsTxt.id,requestId: llmsTxt.request_id,- force,},- () => logLlmsTxtInternal(llmsTxt, force),+ () => logLlmsTxtInternal(llmsTxt),);}-async function logLlmsTxtInternal(- llmsTxt: LoggedLlmsTxt,- force: boolean = false,-) {+async function logLlmsTxtInternal(llmsTxt: LoggedLlmsTxt) {const logger = _logger.child({module: "log_job",method: "logLlmsTxt",@@ -1492,7 +1348,6 @@ async function logLlmsTxtInternal(credits_cost: llmsTxt.credits_cost,cost_tracking: llmsTxt.cost_tracking ?? null,},- force,logger,);@@ -1513,26 +1368,19 @@ export type LoggedDeepResearch = {cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;};-export async function logDeepResearch(- deepResearch: LoggedDeepResearch,- force: boolean = false,-) {+export async function logDeepResearch(deepResearch: LoggedDeepResearch) {return withLogSpan({operation: "deep_research",table: "deep_researches",id: deepResearch.id,requestId: deepResearch.request_id,- force,},- () => logDeepResearchInternal(deepResearch, force),+ () => logDeepResearchInternal(deepResearch),);}-async function logDeepResearchInternal(- deepResearch: LoggedDeepResearch,- force: boolean = false,-) {+async function logDeepResearchInternal(deepResearch: LoggedDeepResearch) {const logger = _logger.child({module: "log_job",method: "logDeepResearch",@@ -1557,7 +1405,6 @@ async function logDeepResearchInternal(credits_cost: deepResearch.credits_cost,cost_tracking: deepResearch.cost_tracking ?? null,},- force,logger,);diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 86c5ab3b7..9da643a2a 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -83,41 +83,35 @@ export async function finishCrawlSuper(job: NuQJob<any>) {.filter(x => x !== null);if (sc.crawlerOptions !== null) {- await logCrawl(- {- id: crawlId,- request_id: requestId,- url: sc.originUrl!,- team_id: teamId,- options: sc.crawlerOptions,- num_docs: fullDocs.length,- credits_cost: fullDocs.reduce(- (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),- 0,- ),- zeroDataRetention,- cancelled: sc.cancelled ?? false,- monitor_id: monitoring?.monitorId,- monitor_check_id: monitoring?.checkId,- },- false,- );+ await logCrawl({+ id: crawlId,+ request_id: requestId,+ url: sc.originUrl!,+ team_id: teamId,+ options: sc.crawlerOptions,+ num_docs: fullDocs.length,+ credits_cost: fullDocs.reduce(+ (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),+ 0,+ ),+ zeroDataRetention,+ cancelled: sc.cancelled ?? false,+ monitor_id: monitoring?.monitorId,+ monitor_check_id: monitoring?.checkId,+ });} else {- await logBatchScrape(- {- id: crawlId,- request_id: requestId,- team_id: teamId,- num_docs: fullDocs.length,- credits_cost: fullDocs.reduce(- (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),- 0,- ),- zeroDataRetention,- cancelled: sc.cancelled ?? false,- },- false,- );+ await logBatchScrape({+ id: crawlId,+ request_id: requestId,+ team_id: teamId,+ num_docs: fullDocs.length,+ credits_cost: fullDocs.reduce(+ (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),+ 0,+ ),+ zeroDataRetention,+ cancelled: sc.cancelled ?? false,+ });}// v0 web hooks, call when done with all the data@@ -196,35 +190,29 @@ export async function finishCrawlSuper(job: NuQJob<any>) {}if (sc.crawlerOptions !== null) {- await logCrawl(- {- id: crawlId,- request_id: requestId,- url: sc.originUrl!,- team_id: teamId,- options: sc.crawlerOptions,- num_docs: num_docs,- credits_cost: credits_billed ?? 0,- zeroDataRetention,- cancelled: sc.cancelled ?? false,- monitor_id: monitoring?.monitorId,- monitor_check_id: monitoring?.checkId,- },- false,- );+ await logCrawl({+ id: crawlId,+ request_id: requestId,+ url: sc.originUrl!,+ team_id: teamId,+ options: sc.crawlerOptions,+ num_docs: num_docs,+ credits_cost: credits_billed ?? 0,+ zeroDataRetention,+ cancelled: sc.cancelled ?? false,+ monitor_id: monitoring?.monitorId,+ monitor_check_id: monitoring?.checkId,+ });} else {- await logBatchScrape(- {- id: crawlId,- request_id: requestId,- team_id: teamId,- num_docs: num_docs,- credits_cost: credits_billed ?? 0,- zeroDataRetention,- cancelled: sc.cancelled ?? false,- },- false,- );+ await logBatchScrape({+ id: crawlId,+ request_id: requestId,+ team_id: teamId,+ num_docs: num_docs,+ credits_cost: credits_billed ?? 0,+ zeroDataRetention,+ cancelled: sc.cancelled ?? false,+ });}// The origin is client-supplied, so this only scopes which of the caller'sdiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 52c41891b..64b4b3602 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -881,28 +881,25 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {doc.metadata.creditsUsed = credits_billed ?? undefined;logger.debug("Logging job to DB...");- await logScrape(- {- id: job.id,- request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,- url: job.data.url,- is_successful: true,- doc,- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- options: job.data.scrapeOptions,- cost_tracking: costTracking.toJSON(),- pdf_num_pages: doc.metadata.numPages,- content_type: doc.metadata.contentType,- credits_cost: credits_billed ?? 0,- zeroDataRetention: job.data.zeroDataRetention,- skipNuq: job.data.skipNuq ?? false,- is_parse: Boolean(job.data.internalOptions?.isParse),- monitor_id: job.data.monitoring?.monitorId,- monitor_check_id: job.data.monitoring?.checkId,- },- true,- );+ await logScrape({+ id: job.id,+ request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,+ url: job.data.url,+ is_successful: true,+ doc,+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ options: job.data.scrapeOptions,+ cost_tracking: costTracking.toJSON(),+ pdf_num_pages: doc.metadata.numPages,+ content_type: doc.metadata.contentType,+ credits_cost: credits_billed ?? 0,+ zeroDataRetention: job.data.zeroDataRetention,+ skipNuq: job.data.skipNuq ?? false,+ is_parse: Boolean(job.data.internalOptions?.isParse),+ monitor_id: job.data.monitoring?.monitorId,+ monitor_check_id: job.data.monitoring?.checkId,+ });trackScrape({scrapeId: job.id,@@ -1011,7 +1008,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {monitor_id: job.data.monitoring?.monitorId,monitor_check_id: job.data.monitoring?.checkId,},- false,{ onStateWritten: stateWritten },);// Release the barrier if logging dies before the state write settles.@@ -1252,30 +1248,27 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}logger.debug("Logging job to DB...");- await logScrape(- {- id: job.id,- request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,- url: job.data.url,- is_successful: false,- error:- typeof error === "string"- ? error- : (error.message ??- "Something went wrong... Contact [email protected]"),- time_taken: timeTakenInSeconds,- team_id: job.data.team_id,- options: job.data.scrapeOptions,- cost_tracking: costTracking.toJSON(),- credits_cost: credits_billed ?? 0,- zeroDataRetention: job.data.zeroDataRetention,- skipNuq: job.data.skipNuq ?? false,- is_parse: Boolean(job.data.internalOptions?.isParse),- monitor_id: job.data.monitoring?.monitorId,- monitor_check_id: job.data.monitoring?.checkId,- },- true,- );+ await logScrape({+ id: job.id,+ request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,+ url: job.data.url,+ is_successful: false,+ error:+ typeof error === "string"+ ? error+ : (error.message ??+ "Something went wrong... Contact [email protected]"),+ time_taken: timeTakenInSeconds,+ team_id: job.data.team_id,+ options: job.data.scrapeOptions,+ cost_tracking: costTracking.toJSON(),+ credits_cost: credits_billed ?? 0,+ zeroDataRetention: job.data.zeroDataRetention,+ skipNuq: job.data.skipNuq ?? false,+ is_parse: Boolean(job.data.internalOptions?.isParse),+ monitor_id: job.data.monitoring?.monitorId,+ monitor_check_id: job.data.monitoring?.checkId,+ });trackScrape({scrapeId: job.id,