Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): the job log is Pub/Sub only, no PostgreSQL copy (#5010)

Gergő Móricz897354b915b5
diff --git a/apps/api/src/__tests__/snips/cost-tracking-helpers.ts b/apps/api/src/__tests__/snips/cost-tracking-helpers.ts
index 686fbf858..47159980b 100644
--- a/apps/api/src/__tests__/snips/cost-tracking-helpers.ts
+++ b/apps/api/src/__tests__/snips/cost-tracking-helpers.ts
@@ -1,6 +1,4 @@
-import { eq } from "drizzle-orm";
-import { db } from "../../db/connection";
-import * as schema from "../../db/schema";
+import { jobLogJson, jobLogRows } from "./job-log";
export type CostTrackingCall = {
model: string;
@@ -9,23 +7,21 @@ export type CostTrackingCall = {
tokens?: { input: number; output: number };
};
-// The scrape row is written when the job finishes; give the insert a moment.
+// The scrape row is published when the job finishes and lands in ClickHouse a
+// few seconds later; poll until it carries parseable cost tracking.
export async function getCostTrackingCalls(
scrapeId: string,
): Promise<CostTrackingCall[]> {
- for (let attempt = 0; attempt < 10; attempt++) {
- const rows = await db
- .select({ cost_tracking: schema.scrapes.cost_tracking })
- .from(schema.scrapes)
- .where(eq(schema.scrapes.id, scrapeId))
- .limit(1);
- if (rows.length === 1) {
- const costTracking = rows[0].cost_tracking as {
- calls?: CostTrackingCall[];
- } | null;
- return costTracking?.calls ?? [];
- }
+ const deadline = Date.now() + 30000;
+ while (Date.now() < deadline) {
+ const [row] = await jobLogRows("scrapes", "id = {id: UUID}", {
+ id: scrapeId,
+ });
+ const costTracking = jobLogJson(row?.cost_tracking) as {
+ calls?: CostTrackingCall[];
+ } | null;
+ if (costTracking) return costTracking.calls ?? [];
await new Promise(resolve => setTimeout(resolve, 1000));
}
- throw new Error(`No scrapes row for ${scrapeId}`);
+ throw new Error(`No cost tracking on the scrapes row for ${scrapeId}`);
}
diff --git a/apps/api/src/__tests__/snips/job-log.ts b/apps/api/src/__tests__/snips/job-log.ts
new file mode 100644
index 000000000..19bcb08c4
--- /dev/null
+++ b/apps/api/src/__tests__/snips/job-log.ts
@@ -0,0 +1,71 @@
+import { clickhouseClient } from "../../lib/clickhouse-client";
+
+/** Tests that read the job log skip unless CLICKHOUSE_ANALYTICS_URL is set. */
+export const HAS_JOB_LOG = clickhouseClient !== null;
+
+// The job log lives in ClickHouse, fed by ClickPipes a few seconds behind the
+// publish. Tables are ReplacingMergeTree ordered by (team_id, id); FINAL keeps
+// the latest publication of an id and the day bound prunes partitions.
+export async function jobLogRows<T extends Record<string, unknown>>(
+ table: string,
+ where: string,
+ params: Record<string, string | number>,
+ options: { orderBy?: string; limit?: number } = {},
+): Promise<T[]> {
+ if (clickhouseClient === null) {
+ throw new Error("CLICKHOUSE_ANALYTICS_URL is required to read the job log");
+ }
+ const result = await clickhouseClient.query({
+ query: `SELECT * FROM ${table} FINAL WHERE ${where} AND created_at >= now() - INTERVAL 1 DAY${
+ options.orderBy ? ` ORDER BY ${options.orderBy}` : ""
+ }${options.limit ? ` LIMIT ${options.limit}` : ""}`,
+ query_params: params,
+ format: "JSONEachRow",
+ });
+ return result.json<T>();
+}
+
+/** Polls until at least `min` rows match; returns the last read, which may be short. */
+export async function waitForJobLogRows<T extends Record<string, unknown>>(
+ table: string,
+ where: string,
+ params: Record<string, string | number>,
+ options: { orderBy?: string; limit?: number; min?: number } = {},
+ timeoutMs = 30000,
+): Promise<T[]> {
+ const { min = 1, ...query } = options;
+ const deadline = Date.now() + timeoutMs;
+ let rows = await jobLogRows<T>(table, where, params, query);
+ while (rows.length < min && Date.now() < deadline) {
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ rows = await jobLogRows<T>(table, where, params, query);
+ }
+ return rows;
+}
+
+/** Polls for the newest matching row; null when none lands in time. */
+export async function waitForJobLogRow<T extends Record<string, unknown>>(
+ table: string,
+ where: string,
+ params: Record<string, string | number>,
+ timeoutMs = 30000,
+): Promise<T | null> {
+ const [row] = await waitForJobLogRows<T>(
+ table,
+ where,
+ params,
+ { orderBy: "created_at DESC", limit: 1 },
+ timeoutMs,
+ );
+ return row ?? null;
+}
+
+/** Parses a job log JSON column; empty or malformed text reads as null. */
+export function jobLogJson(value: unknown): unknown {
+ if (typeof value !== "string") return value ?? null;
+ try {
+ return JSON.parse(value);
+ } catch {
+ return null;
+ }
+}
diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.ts
index ab10290ca..9f823332f 100644
--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts
@@ -16,8 +16,9 @@ import {
expectScrapesAreFullyCleanedAfterZDRCleaner,
expectBatchScrapeIsCleanedUp,
} from "../zdr-helpers";
+import { HAS_JOB_LOG } from "../job-log";
-describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
+describeIf(TEST_PRODUCTION && HAS_JOB_LOG)("Zero Data Retention", () => {
describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {
it("should clean up a scrape immediately", async () => {
let identity = await idmux({
diff --git a/apps/api/src/__tests__/snips/v2/developer.test.ts b/apps/api/src/__tests__/snips/v2/developer.test.ts
index b5233eb4c..2466749e9 100644
--- a/apps/api/src/__tests__/snips/v2/developer.test.ts
+++ b/apps/api/src/__tests__/snips/v2/developer.test.ts
@@ -1,9 +1,7 @@
import { config } from "../../../config";
import { describeIf, TEST_PRODUCTION } from "../lib";
import { creditUsage, idmux, researchPostRaw, researchRaw } from "./lib";
-import { and, desc, eq } from "drizzle-orm";
-import { db } from "../../../db/connection";
-import * as schema from "../../../db/schema";
+import { HAS_JOB_LOG, jobLogJson, waitForJobLogRow } from "../job-log";
const HAS_RESEARCH = !!config.RESEARCH_PROXY_URL;
const KEYLESS_ENABLED =
@@ -18,20 +16,6 @@ const SERVING_PATHS = [CANONICAL_PATH, LEGACY_PATH];
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
const sleepForBilling = () => sleep(40000);
-async function waitForSingleRow<T>(
- fetcher: () => Promise<T | null>,
- timeoutMs: number = 10000,
- intervalMs: number = 250,
-): Promise<T | null> {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- const row = await fetcher();
- if (row) return row;
- await sleep(intervalMs);
- }
- return null;
-}
-
describeIf(HAS_RESEARCH)("Developer Search API", () => {
describe.each(SERVING_PATHS)("developer search on %s", path => {
it("serves developer search", async () => {
@@ -118,7 +102,7 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {
});
it("logs the developer search request kind with origin and integration", async () => {
- if (!config.USE_DB_AUTHENTICATION) return;
+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;
const identity = await idmux({
name: "developer/logs metadata",
@@ -136,27 +120,14 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {
expect(res.statusCode).toBe(200);
expect(res.body.success).toBe(true);
- const requestLog = await waitForSingleRow<{
+ const requestLog = await waitForJobLogRow<{
origin: string | null;
integration: string | null;
- }>(async () => {
- const data = await db
- .select({
- origin: schema.requests.origin,
- integration: schema.requests.integration,
- })
- .from(schema.requests)
- .where(
- and(
- eq(schema.requests.team_id, identity.teamId),
- eq(schema.requests.kind, "code_search"),
- eq(schema.requests.target_hint, query),
- ),
- )
- .orderBy(desc(schema.requests.created_at))
- .limit(1);
- return data[0] ?? null;
- });
+ }>(
+ "requests",
+ "team_id = {teamId: UUID} AND kind = 'code_search' AND target_hint = {query: String}",
+ { teamId: identity.teamId, query },
+ );
expect(requestLog).not.toBeNull();
expect(requestLog?.origin).toBe("mcp");
@@ -164,7 +135,7 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {
}, 120000);
it("redacts stored payloads for a forced-ZDR team", async () => {
- if (!config.USE_DB_AUTHENTICATION) return;
+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;
const identity = await idmux({
name: "developer/forced ZDR retention",
@@ -182,54 +153,32 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {
);
expect(res.statusCode).toBe(200);
- const requestLog = await waitForSingleRow<{
+ const requestLog = await waitForJobLogRow<{
id: string;
target_hint: string;
- dr_clean_by: string | null;
- }>(async () => {
- const data = await db
- .select({
- id: schema.requests.id,
- target_hint: schema.requests.target_hint,
- dr_clean_by: schema.requests.dr_clean_by,
- })
- .from(schema.requests)
- .where(
- and(
- eq(schema.requests.team_id, identity.teamId),
- eq(schema.requests.kind, "code_search"),
- ),
- )
- .orderBy(desc(schema.requests.created_at))
- .limit(1);
- return data[0] ?? null;
+ }>("requests", "team_id = {teamId: UUID} AND kind = 'code_search'", {
+ teamId: identity.teamId,
});
expect(requestLog).not.toBeNull();
expect(requestLog?.target_hint).toBe(
"<redacted due to zero data retention>",
);
- expect(requestLog?.dr_clean_by).not.toBeNull();
- const usageLog = await waitForSingleRow<{
+ const usageRow = await waitForJobLogRow<{
target: string;
- options: unknown;
- response: unknown;
+ options: string | null;
+ response: string | null;
error: string | null;
- }>(async () => {
- if (!requestLog) return null;
- const data = await db
- .select({
- target: schema.code_searches.target,
- options: schema.code_searches.options,
- response: schema.code_searches.response,
- error: schema.code_searches.error,
- })
- .from(schema.code_searches)
- .where(eq(schema.code_searches.request_id, requestLog.id))
- .limit(1);
- return data[0] ?? null;
+ }>("code_searches", "request_id = {requestId: UUID}", {
+ requestId: requestLog!.id,
});
+ const usageLog = usageRow && {
+ target: usageRow.target,
+ options: jobLogJson(usageRow.options),
+ response: jobLogJson(usageRow.response),
+ error: usageRow.error,
+ };
expect(usageLog).toEqual({
target: "<redacted due to zero data retention>",
@@ -240,7 +189,7 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {
}, 120000);
it("writes a usage row with the billed credits", async () => {
- if (!config.USE_DB_AUTHENTICATION) return;
+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;
const identity = await idmux({
name: "developer/logs usage",
@@ -252,23 +201,11 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {
expect(res.statusCode).toBe(200);
const expected = Math.ceil(res.body.results.length / 10) * 2;
- const usageLog = await waitForSingleRow<{
+ const usageLog = await waitForJobLogRow<{
credits_cost: number;
num_results: number;
is_successful: boolean;
- }>(async () => {
- const data = await db
- .select({
- credits_cost: schema.code_searches.credits_cost,
- num_results: schema.code_searches.num_results,
- is_successful: schema.code_searches.is_successful,
- })
- .from(schema.code_searches)
- .where(eq(schema.code_searches.target, query))
- .orderBy(desc(schema.code_searches.created_at))
- .limit(1);
- return data[0] ?? null;
- });
+ }>("code_searches", "target = {query: String}", { query });
expect(usageLog).not.toBeNull();
expect(usageLog?.is_successful).toBe(true);
diff --git a/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts b/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts
index 48ad1e233..2bb12d008 100644
--- a/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts
+++ b/apps/api/src/__tests__/snips/v2/endpoint-feedback.test.ts
@@ -10,9 +10,7 @@ import {
scrapeTimeout,
searchRawFull,
} from "./lib";
-import { and, eq } from "drizzle-orm";
-import { db } from "../../../db/connection";
-import * as schema from "../../../db/schema";
+import { writeFeedbackJob } from "../../../lib/feedback-job-store";
let identity: Identity;
let secondaryIdentity: Identity;
@@ -123,42 +121,32 @@ describeIf(TEST_PRODUCTION)("Generic endpoint feedback tests", () => {
.toString(16)
.padStart(12, "0");
- await db.insert(schema.searches).values({
- id: searchId,
- request_id: searchId,
- query: "failed generic feedback search",
- team_id: identity.teamId,
- options: { query: "failed generic feedback search" },
- time_taken: 0,
- credits_cost: 2,
- is_successful: false,
- error: "Synthetic failed search for generic feedback policy coverage.",
- num_results: 0,
- });
-
- try {
- const failed = await endpointFeedbackRaw(
- {
- endpoint: "search",
- jobId: searchId,
- rating: "bad",
- missingContent: [{ topic: "Results" }],
- },
- identity,
- );
-
- expect(failed.statusCode).toBe(409);
- expect(failed.body.success).toBe(false);
- expect(failed.body.feedbackErrorCode).toBe("SEARCH_FAILED");
- } finally {
- await db
- .delete(schema.searches)
- .where(
- and(
- eq(schema.searches.id, searchId),
- eq(schema.searches.team_id, identity.teamId),
- ),
- );
- }
+ // The feedback policy reads the job's outcome from the feedback job
+ // store, so a failed search is a failed feedback job row.
+ expect(
+ await writeFeedbackJob({
+ endpoint: "search",
+ jobId: searchId,
+ requestId: searchId,
+ teamId: identity.teamId,
+ succeeded: false,
+ creditsBilled: 2,
+ zeroDataRetention: false,
+ }),
+ ).toBe(true);
+
+ const failed = await endpointFeedbackRaw(
+ {
+ endpoint: "search",
+ jobId: searchId,
+ rating: "bad",
+ missingContent: [{ topic: "Results" }],
+ },
+ identity,
+ );
+
+ expect(failed.statusCode).toBe(409);
+ expect(failed.body.success).toBe(false);
+ expect(failed.body.feedbackErrorCode).toBe("SEARCH_FAILED");
}, 30000);
});
diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.ts
index 83d84922b..1f3ede094 100644
--- a/apps/api/src/__tests__/snips/v2/parse.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts
@@ -14,9 +14,7 @@ import request, {
scrapeTimeout,
TEST_API_URL,
} from "./lib";
-import { and, desc, eq } from "drizzle-orm";
-import { db } from "../../../db/connection";
-import * as schema from "../../../db/schema";
+import { HAS_JOB_LOG, jobLogRows, waitForJobLogRow } from "../job-log";
import { config } from "../../../config";
import { getRedisConnection } from "../../../services/queue-service";
@@ -715,7 +713,7 @@ describe("/v2/parse", () => {
it(
"logs parse metadata into the parses table",
async () => {
- if (!config.USE_DB_AUTHENTICATION) return;
+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;
const filename = `parse-log-${Date.now()}.html`;
await parse(
@@ -732,38 +730,19 @@ describe("/v2/parse", () => {
identity,
);
- const requestLog = await waitForSingleRow<{ id: string }>(async () => {
- const data = await db
- .select({ id: schema.requests.id })
- .from(schema.requests)
- .where(
- and(
- eq(schema.requests.team_id, identity.teamId),
- eq(schema.requests.kind, "parse"),
- eq(schema.requests.target_hint, filename),
- ),
- )
- .orderBy(desc(schema.requests.created_at))
- .limit(1);
- return data[0] ?? null;
- });
+ const requestLog = await waitForJobLogRow<{ id: string }>(
+ "requests",
+ "team_id = {teamId: UUID} AND kind = 'parse' AND target_hint = {filename: String}",
+ { teamId: identity.teamId, filename },
+ );
expect(requestLog).not.toBeNull();
- const parseLog = await waitForSingleRow<{
+ const parseLog = await waitForJobLogRow<{
request_id: string;
url: string;
- }>(async () => {
- const data = await db
- .select({
- request_id: schema.parses.request_id,
- url: schema.parses.url,
- })
- .from(schema.parses)
- .where(eq(schema.parses.request_id, requestLog!.id))
- .orderBy(desc(schema.parses.created_at))
- .limit(1);
- return data[0] ?? null;
+ }>("parses", "request_id = {requestId: UUID}", {
+ requestId: requestLog!.id,
});
expect(parseLog).not.toBeNull();
@@ -772,11 +751,12 @@ describe("/v2/parse", () => {
`https://parse.firecrawl.dev/uploads/${encodeURIComponent(filename)}`,
);
- const scrapeRows = await db
- .select({ id: schema.scrapes.id })
- .from(schema.scrapes)
- .where(eq(schema.scrapes.request_id, requestLog!.id))
- .limit(1);
+ const scrapeRows = await jobLogRows<{ id: string }>(
+ "scrapes",
+ "request_id = {requestId: UUID}",
+ { requestId: requestLog!.id },
+ { limit: 1 },
+ );
expect(scrapeRows).toHaveLength(0);
},
scrapeTimeout,
diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.ts
index 64a433b09..6ad20c6ea 100644
--- a/apps/api/src/__tests__/snips/v2/research.test.ts
+++ b/apps/api/src/__tests__/snips/v2/research.test.ts
@@ -3,6 +3,7 @@ import { config } from "../../../config";
import { describeIf, itIf, TEST_PRODUCTION } from "../lib";
import { creditUsage, idmux, researchRaw } from "./lib";
import { and, desc, eq, gt } from "drizzle-orm";
+import { HAS_JOB_LOG, waitForJobLogRow } from "../job-log";
import { db } from "../../../db/connection";
import * as schema from "../../../db/schema";
import { redisRateLimitClient } from "../../../services/rate-limiter";
@@ -244,7 +245,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {
});
it("logs research origin from X-Origin and integration from query", async () => {
- if (!config.USE_DB_AUTHENTICATION) return;
+ if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;
const identity = await idmux({
name: "research/logs metadata",
@@ -262,27 +263,14 @@ describeIf(HAS_RESEARCH)("Research API", () => {
expect(res.statusCode).toBe(200);
expect(res.body.success).toBe(true);
- const requestLog = await waitForSingleRow<{
+ const requestLog = await waitForJobLogRow<{
origin: string | null;
integration: string | null;
- }>(async () => {
- const data = await db
- .select({
- origin: schema.requests.origin,
- integration: schema.requests.integration,
- })
- .from(schema.requests)
- .where(
- and(
- eq(schema.requests.team_id, identity.teamId),
- eq(schema.requests.kind, "research_paper_search"),
- eq(schema.requests.target_hint, query),
- ),
- )
- .orderBy(desc(schema.requests.created_at))
- .limit(1);
- return data[0] ?? null;
- });
+ }>(
+ "requests",
+ "team_id = {teamId: UUID} AND kind = 'research_paper_search' AND target_hint = {query: String}",
+ { teamId: identity.teamId, query },
+ );
expect(requestLog).not.toBeNull();
expect(requestLog?.origin).toBe("mcp");
diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
index 3db906a95..6611568bd 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
@@ -2,6 +2,7 @@ import {
type CostTrackingCall,
getCostTrackingCalls,
} from "../cost-tracking-helpers";
+import { HAS_JOB_LOG } from "../job-log";
import {
ALLOW_TEST_SUITE_WEBSITE,
concurrentIf,
@@ -66,7 +67,7 @@ const JEV_MAY_APPLY =
Number(process.env.BRANDING_JEV_ROLLOUT_PERCENT || 0) > 0);
describe("Branding cost tracking", () => {
- concurrentIf(TEST_PRODUCTION && !JEV_MAY_APPLY)(
+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && !JEV_MAY_APPLY)(
"records the branding LLM call with its model and cost",
async () => {
const response = await scrape(
@@ -90,7 +91,7 @@ describe("Branding cost tracking", () => {
scrapeTimeout + 15000,
);
- concurrentIf(TEST_PRODUCTION)(
+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG)(
"records no branding call when branding is not requested",
async () => {
const response = await scrape(
@@ -117,7 +118,7 @@ const JEV_ON =
!!process.env.TYPESAFE_API_KEY && process.env.BRANDING_JEV === "true";
describe("Branding with Jev", () => {
- concurrentIf(TEST_PRODUCTION && JEV_ON)(
+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && JEV_ON)(
"answers branding with Jev and records its cost, not an LLM call",
async () => {
const response = await scrape(
diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
index e8fca4d52..9f4e73e66 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
@@ -1,6 +1,7 @@
-import { describeIf, TEST_PRODUCTION } from "../lib";
+import { describeIf, testIf, TEST_PRODUCTION } from "../lib";
import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";
import { expectScrapeIsCleanedUp } from "../zdr-helpers";
+import { HAS_JOB_LOG } from "../job-log";
import { getJobFromGCS } from "../../../lib/gcs-jobs";
import crypto from "crypto";
@@ -73,7 +74,7 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
scrapeTimeout,
);
- test(
+ testIf(HAS_JOB_LOG)(
"should treat lockdown as ZDR — no URL in DB, no GCS blob",
async () => {
const id = crypto.randomUUID();
diff --git a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
index 114ecae78..198e6dc14 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
@@ -1,4 +1,5 @@
import { getCostTrackingCalls } from "../cost-tracking-helpers";
+import { HAS_JOB_LOG } from "../job-log";
import { concurrentIf, HAS_XAI, TEST_PRODUCTION } from "../lib";
import { scrape, scrapeTimeout, idmux, Identity } from "./lib";
@@ -15,7 +16,7 @@ beforeAll(async () => {
// X/Twitter scrapes go through xAI's X Search, which bills per profile and
// post fetched.
describe("X/Twitter profile scrape", () => {
- concurrentIf(TEST_PRODUCTION && HAS_XAI)(
+ concurrentIf(TEST_PRODUCTION && HAS_XAI && HAS_JOB_LOG)(
"returns the profile and its latest posts from a single profile lookup",
async () => {
const response = await scrape(
diff --git a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
index 036f0b14f..7ae446d49 100644
--- a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
@@ -7,9 +7,14 @@ import {
idmux,
Identity,
} from "./lib";
-import { and, eq } from "drizzle-orm";
+import { eq } from "drizzle-orm";
import { db } from "../../../db/connection";
import * as schema from "../../../db/schema";
+import { clickhouseClient } from "../../../lib/clickhouse-client";
+import {
+ readFeedbackJob,
+ writeFeedbackJob,
+} from "../../../lib/feedback-job-store";
let identity: Identity;
let secondaryIdentity: Identity;
@@ -47,19 +52,18 @@ describeIf(TEST_PRODUCTION)("Search feedback tests", () => {
expect(typeof raw.body.id).toBe("string");
expect((raw.body.data?.web ?? []).length).toBeGreaterThan(0);
+ // The searches row lands in ClickHouse a few seconds after the publish.
let searchRow: { options: unknown } | undefined;
- for (let attempt = 0; attempt < 20 && !searchRow; attempt++) {
- [searchRow] = await db
- .select({ options: schema.searches.options })
- .from(schema.searches)
- .where(
- and(
- eq(schema.searches.id, raw.body.id),
- eq(schema.searches.team_id, identity.teamId),
- ),
- )
- .limit(1);
- if (!searchRow) await new Promise(resolve => setTimeout(resolve, 100));
+ for (let attempt = 0; attempt < 30 && !searchRow; attempt++) {
+ const result = await clickhouseClient!.query({
+ query:
+ "SELECT options FROM searches FINAL WHERE id = {id: UUID} AND team_id = {teamId: UUID} AND created_at >= now() - INTERVAL 1 DAY",
+ query_params: { id: raw.body.id, teamId: identity.teamId },
+ format: "JSONEachRow",
+ });
+ const [row] = await result.json<{ options: string | null }>();
+ if (row) searchRow = { options: JSON.parse(row.options ?? "null") };
+ else await new Promise(resolve => setTimeout(resolve, 1000));
}
expect(searchRow?.options).toMatchObject({
objective: "Find official Firecrawl information",
@@ -450,17 +454,26 @@ describeIf(TEST_PRODUCTION)("Search feedback tests", () => {
expect(raw.statusCode).toBe(200);
const searchId = raw.body.id;
- await new Promise(r => setTimeout(r, 750));
- const aged = new Date(Date.now() - 60 * 60 * 1000).toISOString();
- await db
- .update(schema.searches)
- .set({ created_at: aged })
- .where(
- and(
- eq(schema.searches.id, searchId),
- eq(schema.searches.team_id, identity.teamId),
- ),
- );
+ // Back-date the feedback job (its deadline enforces the window) rather
+ // than waiting out the configured window.
+ let job = await readFeedbackJob(searchId);
+ for (let attempt = 0; attempt < 30 && !job; attempt++) {
+ await new Promise(r => setTimeout(r, 1000));
+ job = await readFeedbackJob(searchId);
+ }
+ expect(job).not.toBeNull();
+ expect(
+ await writeFeedbackJob({
+ endpoint: "search",
+ jobId: searchId,
+ requestId: job!.requestId,
+ teamId: identity.teamId,
+ succeeded: job!.succeeded,
+ creditsBilled: job!.creditsBilled,
+ zeroDataRetention: job!.zeroDataRetention,
+ completedAt: new Date(Date.now() - 60 * 60 * 1000),
+ }),
+ ).toBe(true);
const failed = await searchFeedbackWithFailure(
searchId,
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index e9c5a66a0..ecb18b887 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -8,7 +8,7 @@ import {
idmux,
searchRaw,
} from "./lib";
-import { describeIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";
+import { describeIf, itIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";
import {
getLogs,
expectScrapeIsCleanedUp,
@@ -17,9 +17,11 @@ import {
expectScrapesAreFullyCleanedAfterZDRCleaner,
expectBatchScrapeIsCleanedUp,
} from "../zdr-helpers";
+import { HAS_JOB_LOG } from "../job-log";
describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
- describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {
+ const scopes = ["Team-scoped", "Request-scoped"] as const;
+ describeIf(HAS_JOB_LOG).each(scopes)("%s", scope => {
it("should clean up a scrape immediately", async () => {
let identity = await idmux({
name: `zdr/${scope}/scrape`,
@@ -150,7 +152,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
);
});
- it(
+ itIf(HAS_JOB_LOG)(
"should allow screenshots and clean them up",
async () => {
const identity = await idmux({
@@ -178,7 +180,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
scrapeTimeout,
);
- it(
+ itIf(HAS_JOB_LOG)(
"should allow pdf actions and clean them up",
async () => {
const identity = await idmux({
diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.ts
index 35da88409..419cbf948 100644
--- a/apps/api/src/__tests__/snips/zdr-helpers.ts
+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts
@@ -1,8 +1,6 @@
import { readFile, stat } from "node:fs/promises";
-import { eq } from "drizzle-orm";
-import { db } from "../../db/connection";
-import * as schema from "../../db/schema";
import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { waitForJobLogRows } from "./job-log";
type Identity = { apiKey: string; teamId: string };
type ScrapeStatusRawFn = (
@@ -87,72 +85,52 @@ export async function getLogs() {
);
}
+type JobRow = Record<string, unknown>;
+
+// Scrape and crawl options are stored as a JSON string; ZDR rows carry none.
+function expectNoStoredContent(row: JobRow) {
+ expect(String(row.url)).not.toContain("://"); // no url stored
+ expect(row.options ?? null).toBeNull();
+}
+
export async function expectScrapeIsCleanedUp(scrapeId: string) {
- const scrapeData = await db
- .select()
- .from(schema.scrapes)
- .where(eq(schema.scrapes.id, scrapeId))
- .limit(1);
+ const scrapeData = await waitForJobLogRows("scrapes", "id = {id: UUID}", {
+ id: scrapeId,
+ });
expect(scrapeData).toHaveLength(1);
-
- if (scrapeData.length === 1) {
- const record = scrapeData[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.options).toBeNull();
- }
+ expectNoStoredContent(scrapeData[0]);
}
export async function expectCrawlIsCleanedUp(crawlId: string) {
- const requestData = await db
- .select()
- .from(schema.requests)
- .where(eq(schema.requests.id, crawlId))
- .limit(1);
+ const requestData = await waitForJobLogRows("requests", "id = {id: UUID}", {
+ id: crawlId,
+ });
expect(requestData).toHaveLength(1);
+ expect(requestData[0].kind).toBe("crawl");
- if (requestData.length === 1) {
- const record = requestData[0];
- expect(record.kind).toBe("crawl");
- expect(record.dr_clean_by).not.toBeNull();
- }
-
- const crawlData = await db
- .select()
- .from(schema.crawls)
- .where(eq(schema.crawls.id, crawlId))
- .limit(1);
+ const crawlData = await waitForJobLogRows("crawls", "id = {id: UUID}", {
+ id: crawlId,
+ });
expect(crawlData).toHaveLength(1);
-
- if (crawlData.length === 1) {
- const record = crawlData[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.options).toBeNull();
- }
+ expectNoStoredContent(crawlData[0]);
}
export async function expectBatchScrapeIsCleanedUp(batchScrapeId: string) {
- const requestData = await db
- .select()
- .from(schema.requests)
- .where(eq(schema.requests.id, batchScrapeId))
- .limit(1);
+ const requestData = await waitForJobLogRows("requests", "id = {id: UUID}", {
+ id: batchScrapeId,
+ });
expect(requestData).toHaveLength(1);
+ expect(requestData[0].kind).toBe("batch_scrape");
- if (requestData.length === 1) {
- const record = requestData[0];
- expect(record.kind).toBe("batch_scrape");
- expect(record.dr_clean_by).not.toBeNull();
- }
-
- const batchScrapeData = await db
- .select()
- .from(schema.batch_scrapes)
- .where(eq(schema.batch_scrapes.id, batchScrapeId))
- .limit(1);
+ const batchScrapeData = await waitForJobLogRows(
+ "batch_scrapes",
+ "id = {id: UUID}",
+ { id: batchScrapeId },
+ );
expect(batchScrapeData).toHaveLength(1);
}
@@ -161,10 +139,12 @@ export async function expectScrapesOfRequestAreCleanedUp(
requestId: string,
expectedScrapeCount?: number,
) {
- const scrapes = await db
- .select()
- .from(schema.scrapes)
- .where(eq(schema.scrapes.request_id, requestId));
+ const scrapes = await waitForJobLogRows(
+ "scrapes",
+ "request_id = {requestId: UUID}",
+ { requestId },
+ { min: expectedScrapeCount ?? 1 },
+ );
if (expectedScrapeCount !== undefined) {
expect(scrapes.length).toBe(expectedScrapeCount);
@@ -173,11 +153,10 @@ export async function expectScrapesOfRequestAreCleanedUp(
}
for (const scrape of scrapes) {
- expect(scrape.url).not.toContain("://"); // no url stored
- expect(scrape.options).toBeNull();
+ expectNoStoredContent(scrape);
if (scrape.is_successful) {
- const gcsJob = await getJobFromGCS(scrape.id);
+ const gcsJob = await getJobFromGCS(String(scrape.id));
expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry
}
}
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 88ba299d0..708aa1c5c 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -363,27 +363,24 @@ export async function searchController(
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
- logSearch(
- {
- id: jobId,
- request_id: jobId,
- query: req.body.query,
- is_successful: true,
- error: undefined,
- results: responseData.data,
- num_results: responseData.data.length,
- time_taken: timeTakenInSeconds,
- team_id: req.auth.team_id,
- options: {
- ...req.body,
- query: undefined,
- scrapeOptions: undefined,
- },
- credits_cost: result.searchCredits,
- zeroDataRetention,
+ logSearch({
+ id: jobId,
+ request_id: jobId,
+ query: req.body.query,
+ is_successful: true,
+ error: undefined,
+ results: responseData.data,
+ num_results: responseData.data.length,
+ time_taken: timeTakenInSeconds,
+ team_id: req.auth.team_id,
+ options: {
+ ...req.body,
+ query: undefined,
+ scrapeOptions: undefined,
},
- false,
- ).catch(error => {
+ credits_cost: result.searchCredits,
+ zeroDataRetention,
+ }).catch(error => {
logger.error("Failed to log search", { error, jobId });
});
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 67714a8f5..cc9d4e17a 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -480,25 +480,22 @@ async function searchControllerInner(
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
- logSearch(
- {
- id: jobId,
- request_id: agentRequestId ?? jobId,
- query: req.body.query,
- is_successful: true,
- error: undefined,
- results: result.response as any,
- num_results: result.totalResultsCount,
- time_taken: timeTakenInSeconds,
- team_id: req.auth.team_id,
- options: req.body,
- // Don't record preview tokens as billed in the ledger — only record
- // credits when billing is actually applied.
- credits_cost: !isSearchPreview && shouldBill ? result.searchCredits : 0,
- zeroDataRetention,
- },
- false,
- ).catch(error => {
+ logSearch({
+ id: jobId,
+ request_id: agentRequestId ?? jobId,
+ query: req.body.query,
+ is_successful: true,
+ error: undefined,
+ results: result.response as any,
+ num_results: result.totalResultsCount,
+ time_taken: timeTakenInSeconds,
+ team_id: req.auth.team_id,
+ options: req.body,
+ // Don't record preview tokens as billed in the ledger — only record
+ // credits when billing is actually applied.
+ credits_cost: !isSearchPreview && shouldBill ? result.searchCredits : 0,
+ zeroDataRetention,
+ }).catch(error => {
logger.error("Failed to log search", { error, jobId });
});
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 569fdb273..74bc02da7 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -84,16 +84,6 @@ export const api_keys = pgTable(
],
);
-export const batch_scrapes = pgTable("batch_scrapes", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- team_id: uuid("team_id").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- num_docs: integer("num_docs").notNull(),
- credits_cost: integer("credits_cost").notNull(),
- cancelled: boolean("cancelled").notNull(),
-});
-
export const blocklist = pgTable("blocklist", {
id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
data: jsonb("data").notNull(),
@@ -163,72 +153,6 @@ export const browser_profiles = pgTable(
table => [primaryKey({ columns: [table.team_id, table.name] })],
);
-export const crawls = pgTable("crawls", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- url: text("url").notNull(),
- team_id: uuid("team_id").notNull(),
- options: jsonb("options"),
- created_at: ts("created_at").notNull().defaultNow(),
- num_docs: integer("num_docs").notNull(),
- credits_cost: integer("credits_cost").notNull(),
- cancelled: boolean("cancelled").notNull(),
- monitor_id: uuid("monitor_id"),
- monitor_check_id: uuid("monitor_check_id"),
-});
-
-export const deep_researches = pgTable("deep_researches", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- query: text("query").notNull(),
- team_id: uuid("team_id").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- time_taken: num("time_taken").notNull(),
- credits_cost: integer("credits_cost").notNull(),
- cost_tracking: jsonb("cost_tracking"),
- options: jsonb("options"),
-});
-
-const researchEndpointTable = (name: string) =>
- pgTable(name, {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- target: text("target").notNull(),
- team_id: uuid("team_id").notNull(),
- options: jsonb("options"),
- response: jsonb("response"),
- num_results: integer("num_results").notNull(),
- time_taken: num("time_taken").notNull(),
- credits_cost: integer("credits_cost").notNull(),
- is_successful: boolean("is_successful").notNull(),
- error: text("error"),
- created_at: ts("created_at").notNull().defaultNow(),
- });
-
-export const research_paper_searches = researchEndpointTable(
- "research_paper_searches",
-);
-
-export const research_paper_inspects = researchEndpointTable(
- "research_paper_inspects",
-);
-
-export const research_paper_reads = researchEndpointTable(
- "research_paper_reads",
-);
-
-export const research_related_papers = researchEndpointTable(
- "research_related_papers",
-);
-
-export const research_github_searches = researchEndpointTable(
- "research_github_searches",
-);
-
-export const code_searches = researchEndpointTable("code_searches");
-
-export const gov_searches = researchEndpointTable("gov_searches");
-
export const deterministic_json_scripts = pgTable(
"deterministic_json_scripts",
{
@@ -261,20 +185,6 @@ export const eb_sync = pgTable("eb-sync", {
team_id: text("team_id"),
});
-export const extracts = pgTable("extracts", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- urls: text("urls").array().notNull(),
- options: jsonb("options"),
- model_kind: text("model_kind").notNull(),
- team_id: uuid("team_id").notNull(),
- is_successful: boolean("is_successful").notNull(),
- error: text("error"),
- created_at: ts("created_at").notNull().defaultNow(),
- credits_cost: integer("credits_cost").notNull(),
- cost_tracking: jsonb("cost_tracking"),
-});
-
export const idempotency_keys = pgTable("idempotency_keys", {
key: uuid("key").notNull().defaultRandom(),
created_at: ts("created_at").notNull().defaultNow(),
@@ -340,29 +250,6 @@ export const llm_texts = pgTable("llm_texts", {
updated_at: ts("updated_at"),
});
-export const llmstxts = pgTable("llmstxts", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- url: text("url").notNull(),
- team_id: uuid("team_id").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- num_urls: integer("num_urls").notNull(),
- options: jsonb("options"),
- cost_tracking: jsonb("cost_tracking"),
- credits_cost: integer("credits_cost").notNull(),
-});
-
-export const maps = pgTable("maps", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- url: text("url").notNull(),
- options: jsonb("options"),
- team_id: uuid("team_id").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- num_results: integer("num_results").notNull(),
- credits_cost: integer("credits_cost").notNull(),
-});
-
export const monitor_check_pages = pgTable("monitor_check_pages", {
id: uuid("id").notNull(),
check_id: uuid("check_id").notNull(),
@@ -506,21 +393,6 @@ export const notification_preferences = pgTable("notification_preferences", {
unsubscribed_all: boolean("unsubscribed_all").default(false),
});
-export const parses = pgTable("parses", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- url: text("url").notNull(),
- is_successful: boolean("is_successful").notNull(),
- error: text("error"),
- time_taken: num("time_taken").notNull(),
- team_id: uuid("team_id").notNull(),
- options: jsonb("options"),
- cost_tracking: jsonb("cost_tracking"),
- pdf_num_pages: integer("pdf_num_pages"),
- credits_cost: integer("credits_cost").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
-});
-
export const prices = pgTable("prices", {
id: text("id").notNull(),
product_id: text("product_id"),
@@ -570,20 +442,6 @@ export const products = pgTable("products", {
type: text("type"),
});
-export const requests = pgTable("requests", {
- id: uuid("id").notNull(),
- kind: text("kind").notNull(),
- api_version: text("api_version").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- team_id: uuid("team_id").notNull(),
- origin: text("origin").notNull(),
- integration: text("integration"),
- target_hint: text("target_hint").notNull(),
- dr_clean_by: ts("dr_clean_by"),
- api_key_id: bigintNum("api_key_id"),
- external_request_id: text("external_request_id"),
-});
-
export const mcp_action_logs = pgTable(
"mcp_action_logs",
{
@@ -616,24 +474,6 @@ export const mcp_action_logs = pgTable(
],
);
-export const scrapes = pgTable("scrapes", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- url: text("url").notNull(),
- is_successful: boolean("is_successful").notNull(),
- error: text("error"),
- time_taken: num("time_taken").notNull(),
- team_id: uuid("team_id").notNull(),
- options: jsonb("options"),
- cost_tracking: jsonb("cost_tracking"),
- pdf_num_pages: integer("pdf_num_pages"),
- credits_cost: integer("credits_cost").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- monitor_id: uuid("monitor_id"),
- monitor_check_id: uuid("monitor_check_id"),
- content_type: text("content_type"),
-});
-
export const search_feedback = pgTable("search_feedback", {
id: uuid("id").notNull().defaultRandom(),
search_id: uuid("search_id"),
@@ -760,20 +600,6 @@ export const alexandria_feedback_capabilities = pgTable(
],
);
-export const searches = pgTable("searches", {
- id: uuid("id").notNull(),
- request_id: uuid("request_id").notNull(),
- query: text("query").notNull(),
- team_id: uuid("team_id").notNull(),
- options: jsonb("options"),
- time_taken: num("time_taken").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- credits_cost: integer("credits_cost").notNull(),
- is_successful: boolean("is_successful").notNull(),
- error: text("error"),
- num_results: integer("num_results").notNull(),
-});
-
export const subscriptions = pgTable("subscriptions", {
id: text("id").notNull(),
user_id: uuid("user_id").notNull(),
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index 59098750e..94771897e 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -4,8 +4,6 @@ import { vi } from "vitest";
// vi.hoisted() (also hoisted).
const {
logger,
- values,
- insert,
topic,
publishes,
publishMessage,
@@ -29,8 +27,6 @@ const {
debug: vi.fn(),
child: vi.fn(() => logger),
};
- const values = vi.fn<(data: any) => Promise<void>>();
- const insert = vi.fn(() => ({ values }));
const publishMessage = vi.fn(async (_message: any) => "message-id");
const flush = vi.fn(async () => {});
const close = vi.fn(async () => {});
@@ -53,8 +49,6 @@ const {
});
return {
logger,
- values,
- insert,
topic,
publishes,
publishMessage,
@@ -97,10 +91,6 @@ vi.mock("../../lib/logger", () => ({
logger,
}));
-vi.mock("../../db/connection", () => ({
- db: { insert },
-}));
-
vi.mock("../../lib/change-tracking-store", () => ({
changeTrackingInsertScrape: vi.fn(),
}));
@@ -163,7 +153,6 @@ import {
shutdownPubSubLogging,
type LoggedSearch,
} from "./log_job";
-import * as schema from "../../db/schema";
import { config } from "../../config";
function deferred<T>() {
@@ -199,7 +188,6 @@ function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
describe("logSearch", () => {
beforeEach(() => {
vi.clearAllMocks();
- values.mockResolvedValue(undefined);
publishMessage.mockResolvedValue("message-id");
publishes.length = 0;
spans.length = 0;
@@ -216,8 +204,9 @@ describe("logSearch", () => {
await logSearch(search);
- expect(insert).toHaveBeenCalledWith(schema.searches);
- const inserted = values.mock.calls[0][0];
+ const inserted = JSON.parse(
+ publishMessage.mock.calls[0][0].data.toString(),
+ );
expect(inserted.query).toBe("helloworld");
expect(inserted.options.query).toBe("nestedquery");
expect(inserted.options.sources[0].location).toBe("NewYork");
@@ -241,7 +230,6 @@ describe("logSearch", () => {
await logSearch(makeSearch({ query: options.query, options }));
- expect(values.mock.calls[0][0].options).toMatchObject(options);
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
@@ -261,8 +249,6 @@ describe("logSearch", () => {
}),
);
- expect(values.mock.calls[0][0].options).not.toHaveProperty("objective");
- expect(values.mock.calls[0][0].options).not.toHaveProperty("clientModel");
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
@@ -270,12 +256,11 @@ describe("logSearch", () => {
expect(published.options).not.toHaveProperty("clientModel");
});
- it("fails the log call on a serialization failure before touching PostgreSQL", async () => {
+ it("fails the log call on a serialization failure", async () => {
const search = makeSearch({ options: { unsupported: 1n } });
await expect(logSearch(search)).rejects.toThrow();
- expect(values).not.toHaveBeenCalled();
expect(publishMessage).not.toHaveBeenCalled();
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
@@ -292,7 +277,7 @@ describe("logSearch", () => {
await expect(logSearch(makeSearch())).resolves.toBeUndefined();
- expect(values).toHaveBeenCalledOnce();
+ expect(publishMessage).toHaveBeenCalledOnce();
expect(logger.error).toHaveBeenCalledWith(
"Failed to write feedback job to Bigtable",
expect.objectContaining({
@@ -302,13 +287,12 @@ describe("logSearch", () => {
);
});
- it("profiles the log, PostgreSQL insert, and Pub/Sub publish", async () => {
+ it("profiles the log and the Pub/Sub publish", async () => {
await logSearch(makeSearch());
expect(spans).toEqual(
expect.arrayContaining([
{ name: "log_job.search", options: { zeroDataRetention: false } },
- { name: "log_job.postgres.insert", options: undefined },
{ name: "log_job.pubsub.publish", options: undefined },
]),
);
@@ -327,7 +311,6 @@ describe("logSearch", () => {
describe("operational job state logging", () => {
beforeEach(() => {
vi.clearAllMocks();
- values.mockResolvedValue(undefined);
publishMessage.mockResolvedValue("message-id");
});
@@ -356,13 +339,13 @@ describe("operational job state logging", () => {
);
});
- it("reports the state written before the PostgreSQL insert starts", async () => {
+ it("reports the state written before the Pub/Sub publish starts", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b8";
const stateWrite = deferred<boolean>();
writeScrapeJobState.mockReturnValueOnce(stateWrite.promise);
let insertsWhenStateWritten = -1;
const onStateWritten = vi.fn(() => {
- insertsWhenStateWritten = values.mock.calls.length;
+ insertsWhenStateWritten = publishMessage.mock.calls.length;
});
const logging = logScrape(
@@ -378,21 +361,20 @@ describe("operational job state logging", () => {
skipNuq: true,
zeroDataRetention: false,
},
- false,
{ onStateWritten },
);
await Promise.resolve();
expect(writeScrapeJobState).toHaveBeenCalledTimes(1);
expect(onStateWritten).not.toHaveBeenCalled();
- expect(values).not.toHaveBeenCalled();
+ expect(publishMessage).not.toHaveBeenCalled();
stateWrite.resolve(true);
await logging;
expect(onStateWritten).toHaveBeenCalledWith("written");
expect(insertsWhenStateWritten).toBe(0);
- expect(values).toHaveBeenCalledTimes(1);
+ expect(publishMessage).toHaveBeenCalledTimes(1);
});
it("reports a failed state write and keeps logging", async () => {
@@ -413,12 +395,11 @@ describe("operational job state logging", () => {
skipNuq: true,
zeroDataRetention: false,
},
- false,
{ onStateWritten },
);
expect(onStateWritten).toHaveBeenCalledWith("failed");
- expect(values).toHaveBeenCalledTimes(1);
+ expect(publishMessage).toHaveBeenCalledTimes(1);
});
it("reports a skipped state write for a parse, which stores none", async () => {
@@ -439,7 +420,6 @@ describe("operational job state logging", () => {
zeroDataRetention: false,
is_parse: true,
},
- false,
{ onStateWritten },
);
@@ -465,7 +445,6 @@ describe("operational job state logging", () => {
skipNuq: true,
zeroDataRetention: false,
},
- false,
{ onStateWritten },
);
@@ -558,7 +537,6 @@ describe("operational job state logging", () => {
describe("logRequest", () => {
beforeEach(() => {
vi.clearAllMocks();
- values.mockResolvedValue(undefined);
publishMessage.mockResolvedValue("message-id");
publishes.length = 0;
});
@@ -580,10 +558,10 @@ describe("logRequest", () => {
it("stores the caller's external_request_id verbatim", async () => {
await logRequest(makeRequest("op_integration_42"));
- expect(insert).toHaveBeenCalledWith(schema.requests);
- expect(values.mock.calls[0][0].external_request_id).toBe(
- "op_integration_42",
+ const published = JSON.parse(
+ publishMessage.mock.calls[0][0].data.toString(),
);
+ expect(published.external_request_id).toBe("op_integration_42");
});
it("prefixes the Pub/Sub topic name with PUBSUB_TOPIC_PREFIX", async () => {
@@ -607,10 +585,9 @@ describe("logRequest", () => {
);
});
- it("writes the request to the database and its Pub/Sub topic", async () => {
+ it("publishes the request to its Pub/Sub topic", async () => {
await logRequest(makeRequest("op_integration_42"));
- expect(insert).toHaveBeenCalledWith(schema.requests);
expect(publishes[0].name).toBe("requests");
const gaxOpts = publishes[0].options.gaxOpts;
// A bare `timeout` would collapse the retry budget to one attempt.
@@ -631,9 +608,6 @@ describe("logRequest", () => {
expect(new Date(published.created_at).toISOString()).toBe(
published.created_at,
);
- expect(values.mock.calls[0][0].created_at.toISOString()).toBe(
- published.created_at,
- );
expect(writeApiJobAccess).toHaveBeenCalledWith(
expect.objectContaining({
id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
@@ -645,7 +619,7 @@ describe("logRequest", () => {
);
});
- it("durably queues ZDR cleanup before writing the request", async () => {
+ it("durably queues ZDR cleanup before publishing the request", async () => {
await logRequest({
...makeRequest("op_integration_42"),
zeroDataRetention: true,
@@ -655,12 +629,15 @@ describe("logRequest", () => {
"019e6f45-7778-727d-adf0-0abe9d5062b6",
);
expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(
- values.mock.invocationCallOrder[0],
+ publishMessage.mock.invocationCallOrder[0],
+ );
+ const published = JSON.parse(
+ publishMessage.mock.calls[0][0].data.toString(),
);
- expect(values.mock.calls[0][0]).not.toHaveProperty("dr_clean_by");
+ expect(published).not.toHaveProperty("dr_clean_by");
});
- it("does not write a ZDR request unless its cleanup job was confirmed", async () => {
+ it("does not publish a ZDR request unless its cleanup job was confirmed", async () => {
enqueueZdrCleanupJob.mockRejectedValueOnce(
new Error("RabbitMQ unavailable"),
);
@@ -668,17 +645,16 @@ describe("logRequest", () => {
await expect(
logRequest({ ...makeRequest(null), zeroDataRetention: true }),
).rejects.toThrow("RabbitMQ unavailable");
- expect(values).not.toHaveBeenCalled();
+ expect(publishMessage).not.toHaveBeenCalled();
});
- it("fails the log call and skips PostgreSQL when Pub/Sub fails", async () => {
+ it("fails the log call when Pub/Sub fails", async () => {
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
await expect(logRequest(makeRequest(null))).rejects.toThrow(
"Pub/Sub unavailable",
);
- expect(values).not.toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledWith(
"Failed to publish log to Pub/Sub",
expect.objectContaining({ error: expect.any(Error) }),
@@ -690,7 +666,7 @@ describe("logRequest", () => {
await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();
- expect(values).toHaveBeenCalled();
+ expect(publishMessage).toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledWith(
"Failed to write API job access to Bigtable",
expect.objectContaining({ error: expect.any(Error), kind: "scrape" }),
@@ -701,7 +677,7 @@ describe("logRequest", () => {
await logRequest({ ...makeRequest(null), jobAccess: false });
expect(writeApiJobAccess).not.toHaveBeenCalled();
- expect(values).toHaveBeenCalledOnce();
+ expect(publishMessage).toHaveBeenCalledOnce();
});
it("uses a job's explicit operational expiry", async () => {
@@ -728,44 +704,27 @@ describe("logRequest", () => {
await new Promise(resolve => setImmediate(resolve));
expect(publishMessage).toHaveBeenCalledOnce();
- expect(values).not.toHaveBeenCalled();
expect(finished).toBe(false);
publication.resolve("message-id");
await logging;
- expect(values).toHaveBeenCalled();
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "published",
});
});
- it("waits for PostgreSQL when publication finishes first", async () => {
- const insertion = deferred<void>();
- values.mockReturnValueOnce(insertion.promise);
- let finished = false;
- const logging = logRequest(makeRequest(null)).then(() => {
- finished = true;
- });
-
- await new Promise(resolve => setImmediate(resolve));
- expect(publishMessage).toHaveBeenCalledOnce();
- expect(finished).toBe(false);
-
- insertion.resolve();
- await logging;
- expect(finished).toBe(true);
- });
-
it("stores null, not a truncation, when the id exceeds the byte cap", async () => {
// The header helper already drops these; this asserts the bound holds at
- // the insert boundary too, for any writer that bypasses the helper. Null
- // rather than a DB constraint, which would fail the whole requests row
+ // the log boundary too, for any writer that bypasses the helper. Null
+ // rather than a rejected row, which would lose the whole requests row
// (and its scrapes/crawls children) over a telemetry field — and null
// rather than truncation, which would hand a wrong id back downstream.
await logRequest(makeRequest("x".repeat(2049)));
- const inserted = values.mock.calls[0][0];
+ const inserted = JSON.parse(
+ publishMessage.mock.calls[0][0].data.toString(),
+ );
expect(inserted.external_request_id).toBeNull();
expect(inserted.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");
expect(logger.warn).toHaveBeenCalled();
@@ -774,18 +733,19 @@ describe("logRequest", () => {
it("counts the cap in bytes, not characters", async () => {
// 1025 two-byte characters: 1025 chars, 2050 bytes — over.
await logRequest(makeRequest("é".repeat(1025)));
- expect(values.mock.calls[0][0].external_request_id).toBeNull();
+ const first = JSON.parse(publishMessage.mock.calls[0][0].data.toString());
+ expect(first.external_request_id).toBeNull();
// 1024 two-byte characters: 2048 bytes exactly — allowed.
await logRequest(makeRequest("é".repeat(1024)));
- expect(values.mock.calls[1][0].external_request_id).toBe("é".repeat(1024));
+ const second = JSON.parse(publishMessage.mock.calls[1][0].data.toString());
+ expect(second.external_request_id).toBe("é".repeat(1024));
});
- it("cleans NUL bytes and unpaired surrogates for both stores", async () => {
+ it("cleans NUL bytes and unpaired surrogates before publishing", async () => {
// "Łódź" mis-decoded by a client arrives as a lone low surrogate, which
// JSON.stringify would emit as "\udc81" and ClickPipes would reject as
- // invalid JSON; PostgreSQL's driver stores it as U+FFFD. The row must
- // reach both stores already cleaned, and identical.
+ // invalid JSON. The row must be published already cleaned.
const replacement = String.fromCharCode(0xfffd);
await logRequest({
...makeRequest(null),
@@ -793,15 +753,11 @@ describe("logRequest", () => {
origin: "api" + String.fromCharCode(0),
});
- const inserted = values.mock.calls[0][0];
- expect(inserted.target_hint).toBe("wyciek Å" + replacement + "ódź!");
- expect(inserted.origin).toBe("api");
-
const raw = publishMessage.mock.calls[0][0].data.toString("utf8");
expect(raw).not.toMatch(/\\u[dD][89a-fA-F]/);
expect(raw).not.toMatch(/\\u0{4}/);
const published = JSON.parse(raw);
- expect(published.target_hint).toBe(inserted.target_hint);
+ expect(published.target_hint).toBe("wyciek Å" + replacement + "ódź!");
expect(published.origin).toBe("api");
});
@@ -826,8 +782,6 @@ describe("logRequest", () => {
await fresh.shutdownPubSubLogging();
}
- // The refused row never reaches PostgreSQL either.
- expect(values).toHaveBeenCalledTimes(2);
expect(publishMessage).toHaveBeenCalledTimes(2);
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
@@ -900,7 +854,6 @@ describe("logRequest", () => {
describe("shutdownPubSubLogging deadline", () => {
beforeEach(() => {
vi.clearAllMocks();
- values.mockResolvedValue(undefined);
publishMessage.mockResolvedValue("message-id");
publishes.length = 0;
});
@@ -1007,7 +960,6 @@ describe("shutdownPubSubLogging deadline", () => {
"shutting down",
);
expect(publishMessage).toHaveBeenCalledOnce();
- expect(values).toHaveBeenCalledTimes(1);
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
outcome: "failed",
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 2087f536b..9f0f332ea 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -1,5 +1,3 @@
-import { db } from "../../db/connection";
-import * as schema from "../../db/schema";
import { changeTrackingInsertScrape } from "../../lib/change-tracking-store";
import { config } from "../../config";
import { enqueueZdrCleanupJob } from "../../lib/zdr-queue";
@@ -7,7 +5,6 @@ import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";
import { configDotenv } from "dotenv";
-import type { PgTable } from "drizzle-orm/pg-core";
import {
saveDeepResearchToGCS,
saveExtractToGCS,
@@ -55,7 +52,6 @@ async function withLogSpan<T>(
table: string;
id: string;
requestId?: string;
- force?: boolean;
zeroDataRetention?: boolean;
},
fn: () => Promise<T>,
@@ -67,7 +63,6 @@ async function withLogSpan<T>(
"log_job.table": params.table,
"log_job.id": params.id,
"log_job.request_id": params.requestId,
- "log_job.force": params.force,
});
return fn();
},
@@ -119,26 +114,6 @@ function sanitizeString(value: string | null | undefined): string | null {
return sanitizeText(value);
}
-const tableMap: Record<string, PgTable> = {
- requests: schema.requests,
- scrapes: schema.scrapes,
- parses: schema.parses,
- crawls: schema.crawls,
- batch_scrapes: schema.batch_scrapes,
- searches: schema.searches,
- research_paper_searches: schema.research_paper_searches,
- research_paper_inspects: schema.research_paper_inspects,
- research_paper_reads: schema.research_paper_reads,
- research_related_papers: schema.research_related_papers,
- research_github_searches: schema.research_github_searches,
- code_searches: schema.code_searches,
- gov_searches: schema.gov_searches,
- extracts: schema.extracts,
- maps: schema.maps,
- llmstxts: schema.llmstxts,
- deep_researches: schema.deep_researches,
-};
-
let pubSubClient: PubSub | null | undefined;
const pubSubTopics = new Map<string, Topic>();
let pubSubShutdown: Promise<void> | undefined;
@@ -399,12 +374,7 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {
}
}
-async function robustInsert(
- table: string,
- data: any,
- force: boolean,
- _logger: Logger,
-) {
+async function robustInsert(table: string, data: any, _logger: Logger) {
const logger = _logger.child({
module: "log_job",
method: "robustInsert",
@@ -413,84 +383,19 @@ async function robustInsert(
});
if (config.USE_DB_AUTHENTICATION !== true) {
- logger.info(
- "Skipping database insertion due to USE_DB_AUTHENTICATION being off",
- );
+ logger.info("Skipping job log due to USE_DB_AUTHENTICATION being off");
return;
}
- // The single point where a row leaves for both stores: clean it once so
- // PostgreSQL and ClickHouse receive identical, accepted values.
+ // Clean the row once so the log receives accepted values.
data = sanitizeLogData({
...data,
created_at: data.created_at ?? new Date(),
});
- // Pub/Sub is the store of record: publish first and wait for it. A failed
- // publish fails the log call, and the PostgreSQL copy is not attempted.
+ // Pub/Sub is the store of record: publish and wait for it. A failed
+ // publish fails the log call.
await publishLog(table, data, logger);
-
- // The PostgreSQL copy is on its way out; its failure is logged, not thrown.
- const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];
- try {
- await withSpan("log_job.postgres.insert", async span => {
- setSpanAttributes(span, {
- "db.system": "postgresql",
- "log_job.table": table,
- "log_job.id": data.id,
- "log_job.force": force,
- "log_job.postgres.enabled": true,
- });
- const target = tableMap[table];
-
- const maxAttempts = force ? 10 : 1;
- for (let i = 0; i < maxAttempts; i++) {
- const backoffMs = i === 0 ? 0 : 75;
- const start = Date.now();
- try {
- await db.insert(target).values(data);
- attempts.push({
- error: null,
- timeMs: Date.now() - start,
- backoffMs,
- });
- break;
- } catch (error) {
- attempts.push({
- error,
- timeMs: Date.now() - start,
- backoffMs,
- });
- if (force) {
- await new Promise(resolve => setTimeout(resolve, 75));
- }
- }
- }
-
- const lastAttempt = attempts.at(-1);
- setSpanAttributes(span, {
- "log_job.postgres.attempts": attempts.length,
- "log_job.postgres.retries": Math.max(0, attempts.length - 1),
- "log_job.postgres.outcome":
- lastAttempt?.error === null ? "inserted" : "failed",
- });
- if (lastAttempt?.error !== null) {
- throw (
- lastAttempt?.error ?? new Error("Database insert was not attempted")
- );
- }
- });
-
- if (attempts.length === 1) {
- logger.debug("Inserted into database successfully", { attempts });
- } else {
- logger.warn("Inserted into database successfully with retries", {
- attempts,
- });
- }
- } catch {
- logger.error("Failed to insert into database", { attempts });
- }
}
type LoggedRequest = {
@@ -662,7 +567,6 @@ async function logRequestInternal(request: LoggedRequest) {
logger,
),
},
- true,
logger,
);
}
@@ -707,29 +611,20 @@ type LogScrapeHooks = {
onStateWritten?: (outcome: ScrapeStateOutcome) => void;
};
-export async function logScrape(
- scrape: LoggedScrape,
- force: boolean = false,
- hooks?: LogScrapeHooks,
-) {
+export async function logScrape(scrape: LoggedScrape, hooks?: LogScrapeHooks) {
return withLogSpan(
{
operation: scrape.is_parse ? "parse" : "scrape",
table: scrape.is_parse ? "parses" : "scrapes",
id: scrape.id,
requestId: scrape.request_id,
- force,
zeroDataRetention: scrape.zeroDataRetention,
},
- () => logScrapeInternal(scrape, force, hooks),
+ () => logScrapeInternal(scrape, hooks),
);
}
-async function logScrapeInternal(
- scrape: LoggedScrape,
- force: boolean = false,
- hooks?: LogScrapeHooks,
-) {
+async function logScrapeInternal(scrape: LoggedScrape, hooks?: LogScrapeHooks) {
const logger = _logger.child({
module: "log_job",
method: "logScrape",
@@ -829,7 +724,6 @@ async function logScrapeInternal(
content_type: scrape.content_type ?? null,
}),
},
- force,
logger,
);
@@ -963,7 +857,6 @@ async function logProviderScrapeInternal(scrape: LoggedProviderScrape) {
monitor_check_id: null,
content_type: null,
},
- false,
logger,
);
}
@@ -982,21 +875,20 @@ type LoggedCrawl = {
monitor_check_id?: string | null;
};
-export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {
+export async function logCrawl(crawl: LoggedCrawl) {
return withLogSpan(
{
operation: "crawl",
table: "crawls",
id: crawl.id,
requestId: crawl.request_id,
- force,
zeroDataRetention: crawl.zeroDataRetention,
},
- () => logCrawlInternal(crawl, force),
+ () => logCrawlInternal(crawl),
);
}
-async function logCrawlInternal(crawl: LoggedCrawl, force: boolean = false) {
+async function logCrawlInternal(crawl: LoggedCrawl) {
const logger = _logger.child({
module: "log_job",
method: "logCrawl",
@@ -1025,7 +917,6 @@ async function logCrawlInternal(crawl: LoggedCrawl, force: boolean = false) {
monitor_id: crawl.monitor_id ?? null,
monitor_check_id: crawl.monitor_check_id ?? null,
},
- force,
logger,
);
}
@@ -1040,27 +931,20 @@ type LoggedBatchScrape = {
cancelled: boolean;
};
-export async function logBatchScrape(
- batchScrape: LoggedBatchScrape,
- force: boolean = false,
-) {
+export async function logBatchScrape(batchScrape: LoggedBatchScrape) {
return withLogSpan(
{
operation: "batch_scrape",
table: "batch_scrapes",
id: batchScrape.id,
requestId: batchScrape.request_id,
- force,
zeroDataRetention: batchScrape.zeroDataRetention,
},
- () => logBatchScrapeInternal(batchScrape, force),
+ () => logBatchScrapeInternal(batchScrape),
);
}
-async function logBatchScrapeInternal(
- batchScrape: LoggedBatchScrape,
- force: boolean = false,
-) {
+async function logBatchScrapeInternal(batchScrape: LoggedBatchScrape) {
const logger = _logger.child({
module: "log_job",
method: "logBatchScrape",
@@ -1084,7 +968,6 @@ async function logBatchScrapeInternal(
credits_cost: batchScrape.credits_cost,
cancelled: batchScrape.cancelled,
},
- force,
logger,
);
}
@@ -1104,21 +987,20 @@ export type LoggedSearch = {
zeroDataRetention: boolean;
};
-export async function logSearch(search: LoggedSearch, force: boolean = false) {
+export async function logSearch(search: LoggedSearch) {
return withLogSpan(
{
operation: "search",
table: "searches",
id: search.id,
requestId: search.request_id,
- force,
zeroDataRetention: search.zeroDataRetention,
},
- () => logSearchInternal(search, force),
+ () => logSearchInternal(search),
);
}
-async function logSearchInternal(search: LoggedSearch, force: boolean = false) {
+async function logSearchInternal(search: LoggedSearch) {
const logger = _logger.child({
module: "log_job",
method: "logSearch",
@@ -1168,7 +1050,6 @@ async function logSearchInternal(search: LoggedSearch, force: boolean = false) {
num_results: search.num_results,
time_taken: search.time_taken,
},
- force,
logger,
);
@@ -1222,27 +1103,20 @@ type LoggedResearchEndpoint = {
zeroDataRetention: boolean;
};
-export async function logResearchEndpoint(
- research: LoggedResearchEndpoint,
- force: boolean = false,
-) {
+export async function logResearchEndpoint(research: LoggedResearchEndpoint) {
return withLogSpan(
{
operation: "research",
table: research.table,
id: research.id,
requestId: research.request_id,
- force,
zeroDataRetention: research.zeroDataRetention,
},
- () => logResearchEndpointInternal(research, force),
+ () => logResearchEndpointInternal(research),
);
}
-async function logResearchEndpointInternal(
- research: LoggedResearchEndpoint,
- force: boolean = false,
-) {
+async function logResearchEndpointInternal(research: LoggedResearchEndpoint) {
const logger = _logger.child({
module: "log_job",
method: "logResearchEndpoint",
@@ -1274,7 +1148,6 @@ async function logResearchEndpointInternal(
is_successful: research.is_successful,
error: research.zeroDataRetention ? null : (research.error ?? null),
},
- force,
logger,
);
}
@@ -1293,26 +1166,19 @@ export type LoggedExtract = {
cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;
};
-export async function logExtract(
- extract: LoggedExtract,
- force: boolean = false,
-) {
+export async function logExtract(extract: LoggedExtract) {
return withLogSpan(
{
operation: "extract",
table: "extracts",
id: extract.id,
requestId: extract.request_id,
- force,
},
- () => logExtractInternal(extract, force),
+ () => logExtractInternal(extract),
);
}
-async function logExtractInternal(
- extract: LoggedExtract,
- force: boolean = false,
-) {
+async function logExtractInternal(extract: LoggedExtract) {
const logger = _logger.child({
module: "log_job",
method: "logExtract",
@@ -1338,7 +1204,6 @@ async function logExtractInternal(
error: extract.error ?? null,
cost_tracking: extract.cost_tracking ?? null,
},
- force,
logger,
);
@@ -1374,21 +1239,20 @@ export type LoggedMap = {
zeroDataRetention: boolean;
};
-export async function logMap(map: LoggedMap, force: boolean = false) {
+export async function logMap(map: LoggedMap) {
return withLogSpan(
{
operation: "map",
table: "maps",
id: map.id,
requestId: map.request_id,
- force,
zeroDataRetention: map.zeroDataRetention,
},
- () => logMapInternal(map, force),
+ () => logMapInternal(map),
);
}
-async function logMapInternal(map: LoggedMap, force: boolean = false) {
+async function logMapInternal(map: LoggedMap) {
const logger = _logger.child({
module: "log_job",
method: "logMap",
@@ -1428,7 +1292,6 @@ async function logMapInternal(map: LoggedMap, force: boolean = false) {
num_results: map.results.length,
credits_cost: map.credits_cost,
},
- force,
logger,
);
@@ -1449,26 +1312,19 @@ export type LoggedLlmsTxt = {
result: { llmstxt: string; llmsfulltxt: string };
};
-export async function logLlmsTxt(
- llmsTxt: LoggedLlmsTxt,
- force: boolean = false,
-) {
+export async function logLlmsTxt(llmsTxt: LoggedLlmsTxt) {
return withLogSpan(
{
operation: "llmstxt",
table: "llmstxts",
id: llmsTxt.id,
requestId: llmsTxt.request_id,
- force,
},
- () => logLlmsTxtInternal(llmsTxt, force),
+ () => logLlmsTxtInternal(llmsTxt),
);
}
-async function logLlmsTxtInternal(
- llmsTxt: LoggedLlmsTxt,
- force: boolean = false,
-) {
+async function logLlmsTxtInternal(llmsTxt: LoggedLlmsTxt) {
const logger = _logger.child({
module: "log_job",
method: "logLlmsTxt",
@@ -1492,7 +1348,6 @@ async function logLlmsTxtInternal(
credits_cost: llmsTxt.credits_cost,
cost_tracking: llmsTxt.cost_tracking ?? null,
},
- force,
logger,
);
@@ -1513,26 +1368,19 @@ export type LoggedDeepResearch = {
cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;
};
-export async function logDeepResearch(
- deepResearch: LoggedDeepResearch,
- force: boolean = false,
-) {
+export async function logDeepResearch(deepResearch: LoggedDeepResearch) {
return withLogSpan(
{
operation: "deep_research",
table: "deep_researches",
id: deepResearch.id,
requestId: deepResearch.request_id,
- force,
},
- () => logDeepResearchInternal(deepResearch, force),
+ () => logDeepResearchInternal(deepResearch),
);
}
-async function logDeepResearchInternal(
- deepResearch: LoggedDeepResearch,
- force: boolean = false,
-) {
+async function logDeepResearchInternal(deepResearch: LoggedDeepResearch) {
const logger = _logger.child({
module: "log_job",
method: "logDeepResearch",
@@ -1557,7 +1405,6 @@ async function logDeepResearchInternal(
credits_cost: deepResearch.credits_cost,
cost_tracking: deepResearch.cost_tracking ?? null,
},
- force,
logger,
);
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 86c5ab3b7..9da643a2a 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -83,41 +83,35 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
.filter(x => x !== null);
if (sc.crawlerOptions !== null) {
- await logCrawl(
- {
- id: crawlId,
- request_id: requestId,
- url: sc.originUrl!,
- team_id: teamId,
- options: sc.crawlerOptions,
- num_docs: fullDocs.length,
- credits_cost: fullDocs.reduce(
- (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),
- 0,
- ),
- zeroDataRetention,
- cancelled: sc.cancelled ?? false,
- monitor_id: monitoring?.monitorId,
- monitor_check_id: monitoring?.checkId,
- },
- false,
- );
+ await logCrawl({
+ id: crawlId,
+ request_id: requestId,
+ url: sc.originUrl!,
+ team_id: teamId,
+ options: sc.crawlerOptions,
+ num_docs: fullDocs.length,
+ credits_cost: fullDocs.reduce(
+ (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),
+ 0,
+ ),
+ zeroDataRetention,
+ cancelled: sc.cancelled ?? false,
+ monitor_id: monitoring?.monitorId,
+ monitor_check_id: monitoring?.checkId,
+ });
} else {
- await logBatchScrape(
- {
- id: crawlId,
- request_id: requestId,
- team_id: teamId,
- num_docs: fullDocs.length,
- credits_cost: fullDocs.reduce(
- (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),
- 0,
- ),
- zeroDataRetention,
- cancelled: sc.cancelled ?? false,
- },
- false,
- );
+ await logBatchScrape({
+ id: crawlId,
+ request_id: requestId,
+ team_id: teamId,
+ num_docs: fullDocs.length,
+ credits_cost: fullDocs.reduce(
+ (acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),
+ 0,
+ ),
+ zeroDataRetention,
+ cancelled: sc.cancelled ?? false,
+ });
}
// v0 web hooks, call when done with all the data
@@ -196,35 +190,29 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
}
if (sc.crawlerOptions !== null) {
- await logCrawl(
- {
- id: crawlId,
- request_id: requestId,
- url: sc.originUrl!,
- team_id: teamId,
- options: sc.crawlerOptions,
- num_docs: num_docs,
- credits_cost: credits_billed ?? 0,
- zeroDataRetention,
- cancelled: sc.cancelled ?? false,
- monitor_id: monitoring?.monitorId,
- monitor_check_id: monitoring?.checkId,
- },
- false,
- );
+ await logCrawl({
+ id: crawlId,
+ request_id: requestId,
+ url: sc.originUrl!,
+ team_id: teamId,
+ options: sc.crawlerOptions,
+ num_docs: num_docs,
+ credits_cost: credits_billed ?? 0,
+ zeroDataRetention,
+ cancelled: sc.cancelled ?? false,
+ monitor_id: monitoring?.monitorId,
+ monitor_check_id: monitoring?.checkId,
+ });
} else {
- await logBatchScrape(
- {
- id: crawlId,
- request_id: requestId,
- team_id: teamId,
- num_docs: num_docs,
- credits_cost: credits_billed ?? 0,
- zeroDataRetention,
- cancelled: sc.cancelled ?? false,
- },
- false,
- );
+ await logBatchScrape({
+ id: crawlId,
+ request_id: requestId,
+ team_id: teamId,
+ num_docs: num_docs,
+ credits_cost: credits_billed ?? 0,
+ zeroDataRetention,
+ cancelled: sc.cancelled ?? false,
+ });
}
// The origin is client-supplied, so this only scopes which of the caller's
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 52c41891b..64b4b3602 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -881,28 +881,25 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
doc.metadata.creditsUsed = credits_billed ?? undefined;
logger.debug("Logging job to DB...");
- await logScrape(
- {
- id: job.id,
- request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,
- url: job.data.url,
- is_successful: true,
- doc,
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- options: job.data.scrapeOptions,
- cost_tracking: costTracking.toJSON(),
- pdf_num_pages: doc.metadata.numPages,
- content_type: doc.metadata.contentType,
- credits_cost: credits_billed ?? 0,
- zeroDataRetention: job.data.zeroDataRetention,
- skipNuq: job.data.skipNuq ?? false,
- is_parse: Boolean(job.data.internalOptions?.isParse),
- monitor_id: job.data.monitoring?.monitorId,
- monitor_check_id: job.data.monitoring?.checkId,
- },
- true,
- );
+ await logScrape({
+ id: job.id,
+ request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,
+ url: job.data.url,
+ is_successful: true,
+ doc,
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ options: job.data.scrapeOptions,
+ cost_tracking: costTracking.toJSON(),
+ pdf_num_pages: doc.metadata.numPages,
+ content_type: doc.metadata.contentType,
+ credits_cost: credits_billed ?? 0,
+ zeroDataRetention: job.data.zeroDataRetention,
+ skipNuq: job.data.skipNuq ?? false,
+ is_parse: Boolean(job.data.internalOptions?.isParse),
+ monitor_id: job.data.monitoring?.monitorId,
+ monitor_check_id: job.data.monitoring?.checkId,
+ });
trackScrape({
scrapeId: job.id,
@@ -1011,7 +1008,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
monitor_id: job.data.monitoring?.monitorId,
monitor_check_id: job.data.monitoring?.checkId,
},
- false,
{ onStateWritten: stateWritten },
);
// Release the barrier if logging dies before the state write settles.
@@ -1252,30 +1248,27 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
logger.debug("Logging job to DB...");
- await logScrape(
- {
- id: job.id,
- request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,
- url: job.data.url,
- is_successful: false,
- error:
- typeof error === "string"
- ? error
- : (error.message ??
- "Something went wrong... Contact [email protected]"),
- time_taken: timeTakenInSeconds,
- team_id: job.data.team_id,
- options: job.data.scrapeOptions,
- cost_tracking: costTracking.toJSON(),
- credits_cost: credits_billed ?? 0,
- zeroDataRetention: job.data.zeroDataRetention,
- skipNuq: job.data.skipNuq ?? false,
- is_parse: Boolean(job.data.internalOptions?.isParse),
- monitor_id: job.data.monitoring?.monitorId,
- monitor_check_id: job.data.monitoring?.checkId,
- },
- true,
- );
+ await logScrape({
+ id: job.id,
+ request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,
+ url: job.data.url,
+ is_successful: false,
+ error:
+ typeof error === "string"
+ ? error
+ : (error.message ??
+ "Something went wrong... Contact [email protected]"),
+ time_taken: timeTakenInSeconds,
+ team_id: job.data.team_id,
+ options: job.data.scrapeOptions,
+ cost_tracking: costTracking.toJSON(),
+ credits_cost: credits_billed ?? 0,
+ zeroDataRetention: job.data.zeroDataRetention,
+ skipNuq: job.data.skipNuq ?? false,
+ is_parse: Boolean(job.data.internalOptions?.isParse),
+ monitor_id: job.data.monitoring?.monitorId,
+ monitor_check_id: job.data.monitoring?.checkId,
+ });
trackScrape({
scrapeId: job.id,