Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { vi } from "vitest";
// vi.mock is hoisted; anything its factories reference must be created in
// vi.hoisted() (also hoisted).
const {
logger,
topic,
publishes,
publishMessage,
flush,
close,
metricInc,
writeApiJobAccess,
writeFeedbackJob,
writeScrapeJobState,
writeExtractJobState,
withSpan,
setSpanAttributes,
spans,
enqueueZdrCleanupJob,
saveExtractResult,
} = vi.hoisted(() => {
const logger: any = {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
child: vi.fn(() => logger),
};
const publishMessage = vi.fn(async (_message: any) => "message-id");
const flush = vi.fn(async () => {});
const close = vi.fn(async () => {});
const publishes: { name: string; options: any }[] = [];
const spans: { name: string; options: any }[] = [];
const withSpan = vi.fn(
async (name: string, fn: (span: any) => any, options?: any) => {
spans.push({ name, options });
return fn({});
},
);
const topic = vi.fn((name: string, options: any) => {
return {
publishMessage: (message: any) => {
publishes.push({ name, options });
return publishMessage(message);
},
flush,
};
});
return {
logger,
topic,
publishes,
publishMessage,
flush,
close,
metricInc: vi.fn(),
writeApiJobAccess: vi.fn(async () => true),
writeFeedbackJob: vi.fn(async () => true),
writeScrapeJobState: vi.fn(async () => true),
writeExtractJobState: vi.fn(async () => true),
withSpan,
setSpanAttributes: vi.fn(),
spans,
enqueueZdrCleanupJob: vi.fn(async () => {}),
saveExtractResult: vi.fn(async () => {}),
};
});
vi.mock("@google-cloud/pubsub", () => ({
PubSub: class {
topic = topic;
close = close;
},
}));
vi.mock("../../config", () => ({
config: {
GCS_BUCKET_NAME: undefined,
PUBSUB_CREDENTIALS: Buffer.from(
JSON.stringify({ project_id: "firecrawl" }),
).toString("base64"),
USE_DB_AUTHENTICATION: true,
PUBSUB_MAX_OUTSTANDING_MESSAGES: 10_000,
PUBSUB_MAX_OUTSTANDING_BYTES: 64 * 1024 * 1024,
PUBSUB_TOPIC_PREFIX: "",
},
}));
vi.mock("../../lib/logger", () => ({
logger,
}));
vi.mock("../../lib/change-tracking-store", () => ({
changeTrackingInsertScrape: vi.fn(),
}));
vi.mock("../../lib/job-access-store", async () => {
const actual = await vi.importActual<
typeof import("../../lib/job-access-store")
>("../../lib/job-access-store");
return { ...actual, writeApiJobAccess };
});
vi.mock("../../lib/feedback-job-store", () => ({
writeFeedbackJob,
}));
vi.mock("../../lib/job-state-store", () => ({
writeScrapeJobState,
writeExtractJobState,
}));
vi.mock("../../lib/keyless", () => ({
keylessTeamUuid: vi.fn(() => null),
}));
vi.mock("../../lib/gcs-jobs", () => ({
saveDeepResearchToGCS: vi.fn(),
saveExtractToGCS: vi.fn(),
saveLlmsTxtToGCS: vi.fn(),
saveMapToGCS: vi.fn(),
saveScrapeToGCS: vi.fn(),
saveSearchToGCS: vi.fn(),
}));
vi.mock("../../lib/zdr-queue", () => ({
enqueueZdrCleanupJob,
}));
vi.mock("../../lib/extract/extract-redis", () => ({
saveExtractResult,
}));
vi.mock("../posthog", () => ({
trackFirstSurfaceUse: vi.fn(),
}));
vi.mock("../../lib/pubsub-log-metrics", () => ({
pubsubLogPublishTotal: { inc: metricInc },
}));
vi.mock("../../lib/otel-tracer", () => ({
withSpan,
setSpanAttributes,
}));
import {
logRequest,
logScrape,
logExtract,
logSearch,
shutdownPubSubLogging,
type LoggedSearch,
} from "./log_job";
import { config } from "../../config";
function deferred<T>() {
let resolve!: (value: T | PromiseLike<T>) => void;
let reject!: (reason: unknown) => void;
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
resolve = resolvePromise;
reject = rejectPromise;
});
return { promise, resolve, reject };
}
function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
return {
id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
request_id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
query: "hello",
team_id: "team-id",
options: {
query: "hello",
sources: [{ type: "web", location: "Boston" }],
},
time_taken: 100,
credits_cost: 1,
is_successful: true,
num_results: 0,
results: null,
zeroDataRetention: false,
...overrides,
};
}
describe("logSearch", () => {
beforeEach(() => {
vi.clearAllMocks();
publishMessage.mockResolvedValue("message-id");
publishes.length = 0;
spans.length = 0;
});
it("removes null bytes from search query log fields", async () => {
const search = makeSearch({
query: "hello\u0000world",
options: {
query: "nested\u0000query",
sources: [{ type: "web", location: "New\u0000York" }],
},
});
await logSearch(search);
const inserted = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
expect(inserted.query).toBe("helloworld");
expect(inserted.options.query).toBe("nestedquery");
expect(inserted.options.sources[0].location).toBe("NewYork");
expect(search.options.query).toBe("nested\u0000query");
expect(writeFeedbackJob).toHaveBeenCalledWith(
expect.objectContaining({
jobId: search.id,
endpoint: "search",
succeeded: true,
creditsBilled: 1,
}),
);
});
it("stores optional task context with the search options", async () => {
const options = {
query: "React memo docs",
objective: "Find official rerender guidance",
clientModel: "claude-sonnet-4-6",
};
await logSearch(makeSearch({ query: options.query, options }));
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
expect(published.options).toMatchObject(options);
});
it("does not retain task context for ZDR searches", async () => {
await logSearch(
makeSearch({
options: {
query: "React memo docs",
objective: "Find official rerender guidance",
clientModel: "claude-sonnet-4-6",
enterprise: ["zdr"],
},
zeroDataRetention: true,
}),
);
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
expect(published.options).not.toHaveProperty("objective");
expect(published.options).not.toHaveProperty("clientModel");
});
it("fails the log call on a serialization failure", async () => {
const search = makeSearch({ options: { unsupported: 1n } });
await expect(logSearch(search)).rejects.toThrow();
expect(publishMessage).not.toHaveBeenCalled();
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
outcome: "failed",
});
expect(logger.error).toHaveBeenCalledWith(
"Failed to publish log to Pub/Sub",
expect.objectContaining({ logId: search.id, error: expect.any(Error) }),
);
});
it("keeps logging when the feedback Bigtable shadow write fails", async () => {
writeFeedbackJob.mockRejectedValueOnce(new Error("Bigtable unavailable"));
await expect(logSearch(makeSearch())).resolves.toBeUndefined();
expect(publishMessage).toHaveBeenCalledOnce();
expect(logger.error).toHaveBeenCalledWith(
"Failed to write feedback job to Bigtable",
expect.objectContaining({
error: expect.any(Error),
endpoint: "search",
}),
);
});
it("profiles the log and the Pub/Sub publish", async () => {
await logSearch(makeSearch());
expect(spans).toEqual(
expect.arrayContaining([
{ name: "log_job.search", options: { zeroDataRetention: false } },
{ name: "log_job.pubsub.publish", options: undefined },
]),
);
});
it("forwards zeroDataRetention to the root log span", async () => {
await logSearch(makeSearch({ zeroDataRetention: true }));
expect(spans).toContainEqual({
name: "log_job.search",
options: { zeroDataRetention: true },
});
});
});
describe("operational job state logging", () => {
beforeEach(() => {
vi.clearAllMocks();
publishMessage.mockResolvedValue("message-id");
});
it("publishes scrape options without credentials or internal fields", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062c1";
const options = {
formats: ["markdown"],
headers: { Authorization: "Bearer secret", cookie: "session=abc" },
actions: [
{ type: "click", selector: "#login" },
{ type: "write", text: "hunter2", selector: "#password" },
],
__searchPreviewToken: "preview-token",
__agentInterop: { auth: "s2s-secret", requestId: "r1", shouldBill: true },
};
await logScrape({
id,
request_id: id,
url: "https://example.com",
is_successful: true,
time_taken: 1,
team_id: "team-id",
options: options as any,
credits_cost: 1,
skipNuq: false,
zeroDataRetention: false,
});
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
expect(published.options).toEqual({
formats: ["markdown"],
headers: { Authorization: "<redacted>", cookie: "<redacted>" },
actions: [
{ type: "click", selector: "#login" },
{ type: "write", text: "<redacted>", selector: "#password" },
],
});
expect(JSON.stringify(published)).not.toContain("secret");
expect(JSON.stringify(published)).not.toContain("hunter2");
// The caller's object is left as it was.
expect(options.headers.Authorization).toBe("Bearer secret");
});
it("writes terminal standalone scrape state", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
await logScrape({
id,
request_id: id,
url: "https://example.com",
is_successful: true,
time_taken: 1,
team_id: "team-id",
options: { formats: ["markdown"] } as any,
credits_cost: 2,
skipNuq: false,
zeroDataRetention: false,
});
expect(writeScrapeJobState).toHaveBeenCalledWith(
id,
expect.objectContaining({
status: "completed",
requestId: id,
creditsBilled: 2,
}),
);
});
it("reports the state written before the Pub/Sub publish starts", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b8";
const stateWrite = deferred<boolean>();
writeScrapeJobState.mockReturnValueOnce(stateWrite.promise);
let insertsWhenStateWritten = -1;
const onStateWritten = vi.fn(() => {
insertsWhenStateWritten = publishMessage.mock.calls.length;
});
const logging = logScrape(
{
id,
request_id: id,
url: "https://example.com",
is_successful: true,
time_taken: 1,
team_id: "team-id",
options: { formats: ["markdown"] } as any,
credits_cost: 1,
skipNuq: true,
zeroDataRetention: false,
},
{ onStateWritten },
);
await Promise.resolve();
expect(writeScrapeJobState).toHaveBeenCalledTimes(1);
expect(onStateWritten).not.toHaveBeenCalled();
expect(publishMessage).not.toHaveBeenCalled();
stateWrite.resolve(true);
await logging;
expect(onStateWritten).toHaveBeenCalledWith("written");
expect(insertsWhenStateWritten).toBe(0);
expect(publishMessage).toHaveBeenCalledTimes(1);
});
it("reports a failed state write and keeps logging", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b9";
writeScrapeJobState.mockRejectedValueOnce(new Error("bigtable down"));
const onStateWritten = vi.fn();
await logScrape(
{
id,
request_id: id,
url: "https://example.com",
is_successful: true,
time_taken: 1,
team_id: "team-id",
options: { formats: ["markdown"] } as any,
credits_cost: 1,
skipNuq: true,
zeroDataRetention: false,
},
{ onStateWritten },
);
expect(onStateWritten).toHaveBeenCalledWith("failed");
expect(publishMessage).toHaveBeenCalledTimes(1);
});
it("reports a skipped state write for a parse, which stores none", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062ba";
const onStateWritten = vi.fn();
await logScrape(
{
id,
request_id: id,
url: "https://example.com/file.pdf",
is_successful: true,
time_taken: 1,
team_id: "team-id",
options: { formats: ["markdown"] } as any,
credits_cost: 1,
skipNuq: true,
zeroDataRetention: false,
is_parse: true,
},
{ onStateWritten },
);
expect(writeScrapeJobState).not.toHaveBeenCalled();
expect(onStateWritten).toHaveBeenCalledWith("skipped");
});
it("reports a skipped state write when no state table is configured", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062bb";
writeScrapeJobState.mockResolvedValueOnce(false);
const onStateWritten = vi.fn();
await logScrape(
{
id,
request_id: id,
url: "https://example.com",
is_successful: true,
time_taken: 1,
team_id: "team-id",
options: { formats: ["markdown"] } as any,
credits_cost: 1,
skipNuq: true,
zeroDataRetention: false,
},
{ onStateWritten },
);
expect(onStateWritten).toHaveBeenCalledWith("skipped");
});
it("writes job access and terminal state for a crawl child", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b7";
const crawlId = "019e6f45-7778-727d-adf0-0abe9d5062b6";
await logScrape({
id,
request_id: crawlId,
url: "https://example.com/page",
is_successful: false,
error: "boom",
time_taken: 1,
team_id: "team-id",
options: { formats: ["markdown"] } as any,
credits_cost: 1,
skipNuq: false,
zeroDataRetention: false,
});
expect(writeApiJobAccess).toHaveBeenCalledWith(
expect.objectContaining({ id, teamId: "team-id", kind: "scrape" }),
);
expect(writeScrapeJobState).toHaveBeenCalledWith(
id,
expect.objectContaining({
status: "failed",
requestId: crawlId,
creditsBilled: 1,
error: "boom",
}),
);
});
it("writes terminal extract state", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
await logExtract({
id,
request_id: id,
urls: ["https://example.com"],
team_id: "team-id",
options: {},
model_kind: "fire-1",
credits_cost: 3,
is_successful: false,
error: "failed",
});
expect(writeExtractJobState).toHaveBeenCalledWith(
id,
expect.objectContaining({
status: "failed",
creditsBilled: 3,
error: "failed",
}),
);
});
it("writes extract state before result storage fails", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
saveExtractResult.mockRejectedValueOnce(new Error("Redis unavailable"));
await expect(
logExtract({
id,
request_id: id,
urls: ["https://example.com"],
team_id: "team-id",
options: {},
model_kind: "fire-1",
credits_cost: 3,
is_successful: true,
result: { ok: true },
}),
).rejects.toThrow("Redis unavailable");
expect(writeExtractJobState).toHaveBeenCalledWith(
id,
expect.objectContaining({ status: "completed" }),
);
expect(writeExtractJobState.mock.invocationCallOrder[0]).toBeLessThan(
saveExtractResult.mock.invocationCallOrder[0],
);
});
});
describe("logRequest", () => {
beforeEach(() => {
vi.clearAllMocks();
publishMessage.mockResolvedValue("message-id");
publishes.length = 0;
});
function makeRequest(externalRequestId: string | null) {
return {
id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
kind: "scrape" as const,
api_version: "v2",
team_id: "team-id",
origin: "api",
target_hint: "https://example.com",
zeroDataRetention: false,
api_key_id: null,
external_request_id: externalRequestId,
};
}
it("stores the caller's external_request_id verbatim", async () => {
await logRequest(makeRequest("op_integration_42"));
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
expect(published.external_request_id).toBe("op_integration_42");
});
it("prefixes the Pub/Sub topic name with PUBSUB_TOPIC_PREFIX", async () => {
config.PUBSUB_TOPIC_PREFIX = "staging-";
try {
await logRequest(makeRequest("op_integration_42"));
} finally {
config.PUBSUB_TOPIC_PREFIX = "";
}
expect(publishes[0].name).toBe("staging-requests");
// The metric and span keep the bare table name.
await new Promise(resolve => setImmediate(resolve));
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "published",
});
expect(setSpanAttributes).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ "log_job.table": "requests" }),
);
});
it("publishes the request to its Pub/Sub topic", async () => {
await logRequest(makeRequest("op_integration_42"));
expect(publishes[0].name).toBe("requests");
const gaxOpts = publishes[0].options.gaxOpts;
// A bare `timeout` would collapse the retry budget to one attempt.
expect(gaxOpts.timeout).toBeUndefined();
// The caller waits on the publish, so the whole budget is 30 s.
expect(gaxOpts.retry.backoffSettings).toMatchObject({
initialRpcTimeoutMillis: 10_000,
maxRpcTimeoutMillis: 10_000,
totalTimeoutMillis: 30_000,
});
expect(gaxOpts.retry.retryCodes).toBeUndefined();
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString("utf8"),
);
expect(published.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");
expect(published.external_request_id).toBe("op_integration_42");
expect(new Date(published.created_at).toISOString()).toBe(
published.created_at,
);
expect(writeApiJobAccess).toHaveBeenCalledWith(
expect.objectContaining({
id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
teamId: "team-id",
kind: "scrape",
clientOrigin: "api",
expiresAt: expect.any(Date),
}),
);
});
it("durably queues ZDR cleanup before publishing the request", async () => {
await logRequest({
...makeRequest("op_integration_42"),
zeroDataRetention: true,
});
expect(enqueueZdrCleanupJob).toHaveBeenCalledWith({
requestId: "019e6f45-7778-727d-adf0-0abe9d5062b6",
});
expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(
publishMessage.mock.invocationCallOrder[0],
);
const published = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
expect(published).not.toHaveProperty("dr_clean_by");
});
it("does not publish a ZDR request unless its cleanup job was confirmed", async () => {
enqueueZdrCleanupJob.mockRejectedValueOnce(
new Error("RabbitMQ unavailable"),
);
await expect(
logRequest({ ...makeRequest(null), zeroDataRetention: true }),
).rejects.toThrow("RabbitMQ unavailable");
expect(publishMessage).not.toHaveBeenCalled();
});
it("fails the log call when Pub/Sub fails", async () => {
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
await expect(logRequest(makeRequest(null))).rejects.toThrow(
"Pub/Sub unavailable",
);
expect(logger.error).toHaveBeenCalledWith(
"Failed to publish log to Pub/Sub",
expect.objectContaining({ error: expect.any(Error) }),
);
});
it("keeps logging when the job access Bigtable shadow write fails", async () => {
writeApiJobAccess.mockRejectedValueOnce(new Error("Bigtable unavailable"));
await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();
expect(publishMessage).toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledWith(
"Failed to write API job access to Bigtable",
expect.objectContaining({ error: expect.any(Error), kind: "scrape" }),
);
});
it("does not write access metadata for a non-operational request id", async () => {
await logRequest({ ...makeRequest(null), jobAccess: false });
expect(writeApiJobAccess).not.toHaveBeenCalled();
expect(publishMessage).toHaveBeenCalledOnce();
});
it("uses a job's explicit operational expiry", async () => {
const expiresAt = new Date("2026-09-15T18:00:00.000Z");
await logRequest({
...makeRequest(null),
kind: "deep_research",
jobAccessExpiresAt: expiresAt,
});
expect(writeApiJobAccess).toHaveBeenCalledWith(
expect.objectContaining({ kind: "deep_research", expiresAt }),
);
});
it("records an agent run's ZDR retention on its job access row", async () => {
await logRequest({
...makeRequest(null),
kind: "agent",
zeroDataRetention: true,
jobAccessZeroDataRetentionMinutes: 15,
});
expect(writeApiJobAccess).toHaveBeenCalledWith(
expect.objectContaining({
kind: "agent",
zeroDataRetention: true,
zeroDataRetentionMinutes: 15,
}),
);
});
it("holds the caller until the publish is acknowledged", async () => {
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
let finished = false;
const logging = logRequest(makeRequest(null)).then(() => {
finished = true;
});
await new Promise(resolve => setImmediate(resolve));
expect(publishMessage).toHaveBeenCalledOnce();
expect(finished).toBe(false);
publication.resolve("message-id");
await logging;
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "published",
});
});
it("stores null, not a truncation, when the id exceeds the byte cap", async () => {
// The header helper already drops these; this asserts the bound holds at
// the log boundary too, for any writer that bypasses the helper. Null
// rather than a rejected row, which would lose the whole requests row
// (and its scrapes/crawls children) over a telemetry field — and null
// rather than truncation, which would hand a wrong id back downstream.
await logRequest(makeRequest("x".repeat(2049)));
const inserted = JSON.parse(
publishMessage.mock.calls[0][0].data.toString(),
);
expect(inserted.external_request_id).toBeNull();
expect(inserted.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");
expect(logger.warn).toHaveBeenCalled();
});
it("counts the cap in bytes, not characters", async () => {
// 1025 two-byte characters: 1025 chars, 2050 bytes — over.
await logRequest(makeRequest("é".repeat(1025)));
const first = JSON.parse(publishMessage.mock.calls[0][0].data.toString());
expect(first.external_request_id).toBeNull();
// 1024 two-byte characters: 2048 bytes exactly — allowed.
await logRequest(makeRequest("é".repeat(1024)));
const second = JSON.parse(publishMessage.mock.calls[1][0].data.toString());
expect(second.external_request_id).toBe("é".repeat(1024));
});
it("cleans NUL bytes and unpaired surrogates before publishing", async () => {
// "Łódź" mis-decoded by a client arrives as a lone low surrogate, which
// JSON.stringify would emit as "\udc81" and ClickPipes would reject as
// invalid JSON. The row must be published already cleaned.
const replacement = String.fromCharCode(0xfffd);
await logRequest({
...makeRequest(null),
target_hint: "wyciek Å\udc81ódź" + String.fromCharCode(0) + "!",
origin: "api" + String.fromCharCode(0),
});
const raw = publishMessage.mock.calls[0][0].data.toString("utf8");
expect(raw).not.toMatch(/\\u[dD][89a-fA-F]/);
expect(raw).not.toMatch(/\\u0{4}/);
const published = JSON.parse(raw);
expect(published.target_hint).toBe("wyciek Å" + replacement + "ódź!");
expect(published.origin).toBe("api");
});
it("refuses a publish beyond the outstanding cap and fails that log call", async () => {
vi.resetModules();
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockImplementation(async () => publication.promise);
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 2;
try {
const first = fresh.logRequest(makeRequest(null));
const second = fresh.logRequest(makeRequest(null));
await new Promise(resolve => setImmediate(resolve));
await expect(fresh.logRequest(makeRequest(null))).rejects.toThrow(
"backlog is full",
);
publication.resolve("message-id");
await Promise.all([first, second]);
} finally {
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 10_000;
publication.resolve("message-id");
await fresh.shutdownPubSubLogging();
}
expect(publishMessage).toHaveBeenCalledTimes(2);
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "dropped",
});
// A refusal is the rate-limited warning, not a per-row error.
expect(logger.error).not.toHaveBeenCalled();
expect(metricInc).not.toHaveBeenCalledWith({
table: "requests",
outcome: "failed",
});
expect(logger.warn).toHaveBeenCalledWith(
"Refusing Pub/Sub log: publisher backlog is full",
expect.objectContaining({
table: "requests",
logId: makeRequest(null).id,
outstandingMessages: 2,
droppedTotal: 1,
}),
);
});
it("counts a published row and a failed row separately", async () => {
await logRequest(makeRequest(null));
await new Promise(resolve => setImmediate(resolve));
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "published",
});
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
await expect(logRequest(makeRequest(null))).rejects.toThrow(
"Pub/Sub unavailable",
);
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "failed",
});
});
it("releases the backlog capacity after a failed publication", async () => {
const originalCap = config.PUBSUB_MAX_OUTSTANDING_MESSAGES;
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 1;
try {
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
await expect(logRequest(makeRequest(null))).rejects.toThrow(
"Pub/Sub unavailable",
);
await logRequest(makeRequest(null));
expect(publishMessage).toHaveBeenCalledTimes(2);
expect(metricInc).not.toHaveBeenCalledWith({
table: "requests",
outcome: "dropped",
});
} finally {
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = originalCap;
}
});
it("flushes Pub/Sub messages during shutdown", async () => {
await logRequest(makeRequest(null));
await Promise.all([shutdownPubSubLogging(), shutdownPubSubLogging()]);
expect(flush).toHaveBeenCalled();
expect(close).toHaveBeenCalledOnce();
});
});
describe("shutdownPubSubLogging deadline", () => {
beforeEach(() => {
vi.clearAllMocks();
publishMessage.mockResolvedValue("message-id");
publishes.length = 0;
});
afterEach(() => {
vi.useRealTimers();
});
it("waits for pending publications even when flush returns early", async () => {
vi.resetModules();
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
const logging = fresh.logSearch(makeSearch());
await new Promise(resolve => setImmediate(resolve));
const shutdown = fresh.shutdownPubSubLogging();
await new Promise(resolve => setImmediate(resolve));
expect(flush).toHaveBeenCalled();
expect(close).not.toHaveBeenCalled();
publication.resolve("message-id");
await shutdown;
await logging;
expect(close).toHaveBeenCalledOnce();
});
it("closes at the deadline when publication stays pending after flush", async () => {
vi.resetModules();
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
const logging = fresh.logSearch(makeSearch());
await new Promise(resolve => setImmediate(resolve));
vi.useFakeTimers();
const shutdown = fresh.shutdownPubSubLogging();
await vi.advanceTimersByTimeAsync(39_999);
expect(close).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
await shutdown;
expect(close).toHaveBeenCalledOnce();
expect(logger.warn).toHaveBeenCalledWith(
"Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
expect.objectContaining({
outstandingMessages: 1,
pendingLogSample: [
expect.objectContaining({
table: "searches",
logId: makeSearch().id,
}),
],
pendingLogSampleTruncated: false,
}),
);
publication.resolve("message-id");
await logging;
});
it("reports publication failures during shutdown and still closes", async () => {
vi.resetModules();
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
const logging = fresh.logSearch(makeSearch());
await new Promise(resolve => setImmediate(resolve));
const shutdown = fresh.shutdownPubSubLogging();
publication.reject(new Error("Pub/Sub unavailable"));
await shutdown;
await expect(logging).rejects.toThrow("Pub/Sub unavailable");
expect(close).toHaveBeenCalledOnce();
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
outcome: "failed",
});
});
it("bounds client close after draining so shutdown can finish", async () => {
vi.resetModules();
const fresh = await import("./log_job.js");
await fresh.logSearch(makeSearch());
close.mockReturnValueOnce(new Promise(() => {}));
vi.useFakeTimers();
const shutdown = fresh.shutdownPubSubLogging();
await vi.advanceTimersByTimeAsync(5_000);
await shutdown;
expect(logger.error).toHaveBeenCalledWith(
"Failed to close Pub/Sub log publisher",
expect.objectContaining({ error: expect.any(Error) }),
);
});
it("reports late logs instead of publishing after shutdown starts", async () => {
vi.resetModules();
const fresh = await import("./log_job.js");
await fresh.logSearch(makeSearch());
const flushing = deferred<void>();
flush.mockReturnValueOnce(flushing.promise);
const shutdown = fresh.shutdownPubSubLogging();
await expect(fresh.logSearch(makeSearch())).rejects.toThrow(
"shutting down",
);
expect(publishMessage).toHaveBeenCalledOnce();
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
outcome: "failed",
});
flushing.resolve();
await shutdown;
});
it("closes the client when a flush outlives the shutdown deadline", async () => {
// A fresh module instance: shutdown is memoized per process.
vi.resetModules();
const fresh = await import("./log_job.js");
await fresh.logRequest({
id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
kind: "scrape",
api_version: "v2",
team_id: "team-id",
origin: "api",
target_hint: "https://example.com",
zeroDataRetention: false,
api_key_id: null,
external_request_id: null,
});
vi.useFakeTimers();
flush.mockReturnValueOnce(new Promise(() => {}));
const shutdown = fresh.shutdownPubSubLogging();
await vi.advanceTimersByTimeAsync(40_000);
await shutdown;
expect(close).toHaveBeenCalledOnce();
expect(logger.warn).toHaveBeenCalledWith(
"Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
expect.objectContaining({ timeoutMs: 40_000 }),
);
});
});