Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

31.5 KB
Raw Blame
import { vi } from "vitest";

// vi.mock is hoisted; anything its factories reference must be created in
// vi.hoisted() (also hoisted).
const {
  logger,
  topic,
  publishes,
  publishMessage,
  flush,
  close,
  metricInc,
  writeApiJobAccess,
  writeFeedbackJob,
  writeScrapeJobState,
  writeExtractJobState,
  withSpan,
  setSpanAttributes,
  spans,
  enqueueZdrCleanupJob,
  saveExtractResult,
} = vi.hoisted(() => {
  const logger: any = {
    info: vi.fn(),
    warn: vi.fn(),
    error: vi.fn(),
    debug: vi.fn(),
    child: vi.fn(() => logger),
  };
  const publishMessage = vi.fn(async (_message: any) => "message-id");
  const flush = vi.fn(async () => {});
  const close = vi.fn(async () => {});
  const publishes: { name: string; options: any }[] = [];
  const spans: { name: string; options: any }[] = [];
  const withSpan = vi.fn(
    async (name: string, fn: (span: any) => any, options?: any) => {
      spans.push({ name, options });
      return fn({});
    },
  );
  const topic = vi.fn((name: string, options: any) => {
    return {
      publishMessage: (message: any) => {
        publishes.push({ name, options });
        return publishMessage(message);
      },
      flush,
    };
  });
  return {
    logger,
    topic,
    publishes,
    publishMessage,
    flush,
    close,
    metricInc: vi.fn(),
    writeApiJobAccess: vi.fn(async () => true),
    writeFeedbackJob: vi.fn(async () => true),
    writeScrapeJobState: vi.fn(async () => true),
    writeExtractJobState: vi.fn(async () => true),
    withSpan,
    setSpanAttributes: vi.fn(),
    spans,
    enqueueZdrCleanupJob: vi.fn(async () => {}),
    saveExtractResult: vi.fn(async () => {}),
  };
});

vi.mock("@google-cloud/pubsub", () => ({
  PubSub: class {
    topic = topic;
    close = close;
  },
}));

vi.mock("../../config", () => ({
  config: {
    GCS_BUCKET_NAME: undefined,
    PUBSUB_CREDENTIALS: Buffer.from(
      JSON.stringify({ project_id: "firecrawl" }),
    ).toString("base64"),
    USE_DB_AUTHENTICATION: true,
    PUBSUB_MAX_OUTSTANDING_MESSAGES: 10_000,
    PUBSUB_MAX_OUTSTANDING_BYTES: 64 * 1024 * 1024,
    PUBSUB_TOPIC_PREFIX: "",
  },
}));

vi.mock("../../lib/logger", () => ({
  logger,
}));

vi.mock("../../lib/change-tracking-store", () => ({
  changeTrackingInsertScrape: vi.fn(),
}));

vi.mock("../../lib/job-access-store", async () => {
  const actual = await vi.importActual<
    typeof import("../../lib/job-access-store")
  >("../../lib/job-access-store");
  return { ...actual, writeApiJobAccess };
});

vi.mock("../../lib/feedback-job-store", () => ({
  writeFeedbackJob,
}));

vi.mock("../../lib/job-state-store", () => ({
  writeScrapeJobState,
  writeExtractJobState,
}));

vi.mock("../../lib/keyless", () => ({
  keylessTeamUuid: vi.fn(() => null),
}));

vi.mock("../../lib/gcs-jobs", () => ({
  saveDeepResearchToGCS: vi.fn(),
  saveExtractToGCS: vi.fn(),
  saveLlmsTxtToGCS: vi.fn(),
  saveMapToGCS: vi.fn(),
  saveScrapeToGCS: vi.fn(),
  saveSearchToGCS: vi.fn(),
}));

vi.mock("../../lib/zdr-queue", () => ({
  enqueueZdrCleanupJob,
}));

vi.mock("../../lib/extract/extract-redis", () => ({
  saveExtractResult,
}));

vi.mock("../posthog", () => ({
  trackFirstSurfaceUse: vi.fn(),
}));

vi.mock("../../lib/pubsub-log-metrics", () => ({
  pubsubLogPublishTotal: { inc: metricInc },
}));

vi.mock("../../lib/otel-tracer", () => ({
  withSpan,
  setSpanAttributes,
}));

import {
  logRequest,
  logScrape,
  logExtract,
  logSearch,
  shutdownPubSubLogging,
  type LoggedSearch,
} from "./log_job";
import { config } from "../../config";

function deferred<T>() {
  let resolve!: (value: T | PromiseLike<T>) => void;
  let reject!: (reason: unknown) => void;
  const promise = new Promise<T>((resolvePromise, rejectPromise) => {
    resolve = resolvePromise;
    reject = rejectPromise;
  });
  return { promise, resolve, reject };
}

function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
  return {
    id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
    request_id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
    query: "hello",
    team_id: "team-id",
    options: {
      query: "hello",
      sources: [{ type: "web", location: "Boston" }],
    },
    time_taken: 100,
    credits_cost: 1,
    is_successful: true,
    num_results: 0,
    results: null,
    zeroDataRetention: false,
    ...overrides,
  };
}

describe("logSearch", () => {
  beforeEach(() => {
    vi.clearAllMocks();
    publishMessage.mockResolvedValue("message-id");
    publishes.length = 0;
    spans.length = 0;
  });

  it("removes null bytes from search query log fields", async () => {
    const search = makeSearch({
      query: "hello\u0000world",
      options: {
        query: "nested\u0000query",
        sources: [{ type: "web", location: "New\u0000York" }],
      },
    });

    await logSearch(search);

    const inserted = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString(),
    );
    expect(inserted.query).toBe("helloworld");
    expect(inserted.options.query).toBe("nestedquery");
    expect(inserted.options.sources[0].location).toBe("NewYork");
    expect(search.options.query).toBe("nested\u0000query");
    expect(writeFeedbackJob).toHaveBeenCalledWith(
      expect.objectContaining({
        jobId: search.id,
        endpoint: "search",
        succeeded: true,
        creditsBilled: 1,
      }),
    );
  });

  it("stores optional task context with the search options", async () => {
    const options = {
      query: "React memo docs",
      objective: "Find official rerender guidance",
      clientModel: "claude-sonnet-4-6",
    };

    await logSearch(makeSearch({ query: options.query, options }));

    const published = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString(),
    );
    expect(published.options).toMatchObject(options);
  });

  it("does not retain task context for ZDR searches", async () => {
    await logSearch(
      makeSearch({
        options: {
          query: "React memo docs",
          objective: "Find official rerender guidance",
          clientModel: "claude-sonnet-4-6",
          enterprise: ["zdr"],
        },
        zeroDataRetention: true,
      }),
    );

    const published = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString(),
    );
    expect(published.options).not.toHaveProperty("objective");
    expect(published.options).not.toHaveProperty("clientModel");
  });

  it("fails the log call on a serialization failure", async () => {
    const search = makeSearch({ options: { unsupported: 1n } });

    await expect(logSearch(search)).rejects.toThrow();

    expect(publishMessage).not.toHaveBeenCalled();
    expect(metricInc).toHaveBeenCalledWith({
      table: "searches",
      outcome: "failed",
    });
    expect(logger.error).toHaveBeenCalledWith(
      "Failed to publish log to Pub/Sub",
      expect.objectContaining({ logId: search.id, error: expect.any(Error) }),
    );
  });

  it("keeps logging when the feedback Bigtable shadow write fails", async () => {
    writeFeedbackJob.mockRejectedValueOnce(new Error("Bigtable unavailable"));

    await expect(logSearch(makeSearch())).resolves.toBeUndefined();

    expect(publishMessage).toHaveBeenCalledOnce();
    expect(logger.error).toHaveBeenCalledWith(
      "Failed to write feedback job to Bigtable",
      expect.objectContaining({
        error: expect.any(Error),
        endpoint: "search",
      }),
    );
  });

  it("profiles the log and the Pub/Sub publish", async () => {
    await logSearch(makeSearch());

    expect(spans).toEqual(
      expect.arrayContaining([
        { name: "log_job.search", options: { zeroDataRetention: false } },
        { name: "log_job.pubsub.publish", options: undefined },
      ]),
    );
  });

  it("forwards zeroDataRetention to the root log span", async () => {
    await logSearch(makeSearch({ zeroDataRetention: true }));

    expect(spans).toContainEqual({
      name: "log_job.search",
      options: { zeroDataRetention: true },
    });
  });
});

describe("operational job state logging", () => {
  beforeEach(() => {
    vi.clearAllMocks();
    publishMessage.mockResolvedValue("message-id");
  });

  it("publishes scrape options without credentials or internal fields", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062c1";
    const options = {
      formats: ["markdown"],
      headers: { Authorization: "Bearer secret", cookie: "session=abc" },
      actions: [
        { type: "click", selector: "#login" },
        { type: "write", text: "hunter2", selector: "#password" },
      ],
      __searchPreviewToken: "preview-token",
      __agentInterop: { auth: "s2s-secret", requestId: "r1", shouldBill: true },
    };
    await logScrape({
      id,
      request_id: id,
      url: "https://example.com",
      is_successful: true,
      time_taken: 1,
      team_id: "team-id",
      options: options as any,
      credits_cost: 1,
      skipNuq: false,
      zeroDataRetention: false,
    });

    const published = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString(),
    );
    expect(published.options).toEqual({
      formats: ["markdown"],
      headers: { Authorization: "<redacted>", cookie: "<redacted>" },
      actions: [
        { type: "click", selector: "#login" },
        { type: "write", text: "<redacted>", selector: "#password" },
      ],
    });
    expect(JSON.stringify(published)).not.toContain("secret");
    expect(JSON.stringify(published)).not.toContain("hunter2");
    // The caller's object is left as it was.
    expect(options.headers.Authorization).toBe("Bearer secret");
  });

  it("writes terminal standalone scrape state", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
    await logScrape({
      id,
      request_id: id,
      url: "https://example.com",
      is_successful: true,
      time_taken: 1,
      team_id: "team-id",
      options: { formats: ["markdown"] } as any,
      credits_cost: 2,
      skipNuq: false,
      zeroDataRetention: false,
    });

    expect(writeScrapeJobState).toHaveBeenCalledWith(
      id,
      expect.objectContaining({
        status: "completed",
        requestId: id,
        creditsBilled: 2,
      }),
    );
  });

  it("reports the state written before the Pub/Sub publish starts", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062b8";
    const stateWrite = deferred<boolean>();
    writeScrapeJobState.mockReturnValueOnce(stateWrite.promise);
    let insertsWhenStateWritten = -1;
    const onStateWritten = vi.fn(() => {
      insertsWhenStateWritten = publishMessage.mock.calls.length;
    });

    const logging = logScrape(
      {
        id,
        request_id: id,
        url: "https://example.com",
        is_successful: true,
        time_taken: 1,
        team_id: "team-id",
        options: { formats: ["markdown"] } as any,
        credits_cost: 1,
        skipNuq: true,
        zeroDataRetention: false,
      },
      { onStateWritten },
    );
    await Promise.resolve();

    expect(writeScrapeJobState).toHaveBeenCalledTimes(1);
    expect(onStateWritten).not.toHaveBeenCalled();
    expect(publishMessage).not.toHaveBeenCalled();

    stateWrite.resolve(true);
    await logging;

    expect(onStateWritten).toHaveBeenCalledWith("written");
    expect(insertsWhenStateWritten).toBe(0);
    expect(publishMessage).toHaveBeenCalledTimes(1);
  });

  it("reports a failed state write and keeps logging", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062b9";
    writeScrapeJobState.mockRejectedValueOnce(new Error("bigtable down"));
    const onStateWritten = vi.fn();

    await logScrape(
      {
        id,
        request_id: id,
        url: "https://example.com",
        is_successful: true,
        time_taken: 1,
        team_id: "team-id",
        options: { formats: ["markdown"] } as any,
        credits_cost: 1,
        skipNuq: true,
        zeroDataRetention: false,
      },
      { onStateWritten },
    );

    expect(onStateWritten).toHaveBeenCalledWith("failed");
    expect(publishMessage).toHaveBeenCalledTimes(1);
  });

  it("reports a skipped state write for a parse, which stores none", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062ba";
    const onStateWritten = vi.fn();

    await logScrape(
      {
        id,
        request_id: id,
        url: "https://example.com/file.pdf",
        is_successful: true,
        time_taken: 1,
        team_id: "team-id",
        options: { formats: ["markdown"] } as any,
        credits_cost: 1,
        skipNuq: true,
        zeroDataRetention: false,
        is_parse: true,
      },
      { onStateWritten },
    );

    expect(writeScrapeJobState).not.toHaveBeenCalled();
    expect(onStateWritten).toHaveBeenCalledWith("skipped");
  });

  it("reports a skipped state write when no state table is configured", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062bb";
    writeScrapeJobState.mockResolvedValueOnce(false);
    const onStateWritten = vi.fn();

    await logScrape(
      {
        id,
        request_id: id,
        url: "https://example.com",
        is_successful: true,
        time_taken: 1,
        team_id: "team-id",
        options: { formats: ["markdown"] } as any,
        credits_cost: 1,
        skipNuq: true,
        zeroDataRetention: false,
      },
      { onStateWritten },
    );

    expect(onStateWritten).toHaveBeenCalledWith("skipped");
  });

  it("writes job access and terminal state for a crawl child", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062b7";
    const crawlId = "019e6f45-7778-727d-adf0-0abe9d5062b6";
    await logScrape({
      id,
      request_id: crawlId,
      url: "https://example.com/page",
      is_successful: false,
      error: "boom",
      time_taken: 1,
      team_id: "team-id",
      options: { formats: ["markdown"] } as any,
      credits_cost: 1,
      skipNuq: false,
      zeroDataRetention: false,
    });

    expect(writeApiJobAccess).toHaveBeenCalledWith(
      expect.objectContaining({ id, teamId: "team-id", kind: "scrape" }),
    );
    expect(writeScrapeJobState).toHaveBeenCalledWith(
      id,
      expect.objectContaining({
        status: "failed",
        requestId: crawlId,
        creditsBilled: 1,
        error: "boom",
      }),
    );
  });

  it("writes terminal extract state", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
    await logExtract({
      id,
      request_id: id,
      urls: ["https://example.com"],
      team_id: "team-id",
      options: {},
      model_kind: "fire-1",
      credits_cost: 3,
      is_successful: false,
      error: "failed",
    });

    expect(writeExtractJobState).toHaveBeenCalledWith(
      id,
      expect.objectContaining({
        status: "failed",
        creditsBilled: 3,
        error: "failed",
      }),
    );
  });

  it("writes extract state before result storage fails", async () => {
    const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
    saveExtractResult.mockRejectedValueOnce(new Error("Redis unavailable"));

    await expect(
      logExtract({
        id,
        request_id: id,
        urls: ["https://example.com"],
        team_id: "team-id",
        options: {},
        model_kind: "fire-1",
        credits_cost: 3,
        is_successful: true,
        result: { ok: true },
      }),
    ).rejects.toThrow("Redis unavailable");

    expect(writeExtractJobState).toHaveBeenCalledWith(
      id,
      expect.objectContaining({ status: "completed" }),
    );
    expect(writeExtractJobState.mock.invocationCallOrder[0]).toBeLessThan(
      saveExtractResult.mock.invocationCallOrder[0],
    );
  });
});

describe("logRequest", () => {
  beforeEach(() => {
    vi.clearAllMocks();
    publishMessage.mockResolvedValue("message-id");
    publishes.length = 0;
  });

  function makeRequest(externalRequestId: string | null) {
    return {
      id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
      kind: "scrape" as const,
      api_version: "v2",
      team_id: "team-id",
      origin: "api",
      target_hint: "https://example.com",
      zeroDataRetention: false,
      api_key_id: null,
      external_request_id: externalRequestId,
    };
  }

  it("stores the caller's external_request_id verbatim", async () => {
    await logRequest(makeRequest("op_integration_42"));

    const published = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString(),
    );
    expect(published.external_request_id).toBe("op_integration_42");
  });

  it("prefixes the Pub/Sub topic name with PUBSUB_TOPIC_PREFIX", async () => {
    config.PUBSUB_TOPIC_PREFIX = "staging-";
    try {
      await logRequest(makeRequest("op_integration_42"));
    } finally {
      config.PUBSUB_TOPIC_PREFIX = "";
    }

    expect(publishes[0].name).toBe("staging-requests");
    // The metric and span keep the bare table name.
    await new Promise(resolve => setImmediate(resolve));
    expect(metricInc).toHaveBeenCalledWith({
      table: "requests",
      outcome: "published",
    });
    expect(setSpanAttributes).toHaveBeenCalledWith(
      expect.anything(),
      expect.objectContaining({ "log_job.table": "requests" }),
    );
  });

  it("publishes the request to its Pub/Sub topic", async () => {
    await logRequest(makeRequest("op_integration_42"));

    expect(publishes[0].name).toBe("requests");
    const gaxOpts = publishes[0].options.gaxOpts;
    // A bare `timeout` would collapse the retry budget to one attempt.
    expect(gaxOpts.timeout).toBeUndefined();
    // The caller waits on the publish, so the whole budget is 30 s.
    expect(gaxOpts.retry.backoffSettings).toMatchObject({
      initialRpcTimeoutMillis: 10_000,
      maxRpcTimeoutMillis: 10_000,
      totalTimeoutMillis: 30_000,
    });
    expect(gaxOpts.retry.retryCodes).toBeUndefined();

    const published = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString("utf8"),
    );
    expect(published.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");
    expect(published.external_request_id).toBe("op_integration_42");
    expect(new Date(published.created_at).toISOString()).toBe(
      published.created_at,
    );
    expect(writeApiJobAccess).toHaveBeenCalledWith(
      expect.objectContaining({
        id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
        teamId: "team-id",
        kind: "scrape",
        clientOrigin: "api",
        expiresAt: expect.any(Date),
      }),
    );
  });

  it("durably queues ZDR cleanup before publishing the request", async () => {
    await logRequest({
      ...makeRequest("op_integration_42"),
      zeroDataRetention: true,
    });

    expect(enqueueZdrCleanupJob).toHaveBeenCalledWith({
      requestId: "019e6f45-7778-727d-adf0-0abe9d5062b6",
    });
    expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(
      publishMessage.mock.invocationCallOrder[0],
    );
    const published = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString(),
    );
    expect(published).not.toHaveProperty("dr_clean_by");
  });

  it("does not publish a ZDR request unless its cleanup job was confirmed", async () => {
    enqueueZdrCleanupJob.mockRejectedValueOnce(
      new Error("RabbitMQ unavailable"),
    );

    await expect(
      logRequest({ ...makeRequest(null), zeroDataRetention: true }),
    ).rejects.toThrow("RabbitMQ unavailable");
    expect(publishMessage).not.toHaveBeenCalled();
  });

  it("fails the log call when Pub/Sub fails", async () => {
    publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));

    await expect(logRequest(makeRequest(null))).rejects.toThrow(
      "Pub/Sub unavailable",
    );

    expect(logger.error).toHaveBeenCalledWith(
      "Failed to publish log to Pub/Sub",
      expect.objectContaining({ error: expect.any(Error) }),
    );
  });

  it("keeps logging when the job access Bigtable shadow write fails", async () => {
    writeApiJobAccess.mockRejectedValueOnce(new Error("Bigtable unavailable"));

    await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();

    expect(publishMessage).toHaveBeenCalled();
    expect(logger.error).toHaveBeenCalledWith(
      "Failed to write API job access to Bigtable",
      expect.objectContaining({ error: expect.any(Error), kind: "scrape" }),
    );
  });

  it("does not write access metadata for a non-operational request id", async () => {
    await logRequest({ ...makeRequest(null), jobAccess: false });

    expect(writeApiJobAccess).not.toHaveBeenCalled();
    expect(publishMessage).toHaveBeenCalledOnce();
  });

  it("uses a job's explicit operational expiry", async () => {
    const expiresAt = new Date("2026-09-15T18:00:00.000Z");

    await logRequest({
      ...makeRequest(null),
      kind: "deep_research",
      jobAccessExpiresAt: expiresAt,
    });

    expect(writeApiJobAccess).toHaveBeenCalledWith(
      expect.objectContaining({ kind: "deep_research", expiresAt }),
    );
  });

  it("records an agent run's ZDR retention on its job access row", async () => {
    await logRequest({
      ...makeRequest(null),
      kind: "agent",
      zeroDataRetention: true,
      jobAccessZeroDataRetentionMinutes: 15,
    });

    expect(writeApiJobAccess).toHaveBeenCalledWith(
      expect.objectContaining({
        kind: "agent",
        zeroDataRetention: true,
        zeroDataRetentionMinutes: 15,
      }),
    );
  });

  it("holds the caller until the publish is acknowledged", async () => {
    const publication = deferred<string>();
    publishMessage.mockReturnValueOnce(publication.promise);
    let finished = false;
    const logging = logRequest(makeRequest(null)).then(() => {
      finished = true;
    });

    await new Promise(resolve => setImmediate(resolve));
    expect(publishMessage).toHaveBeenCalledOnce();
    expect(finished).toBe(false);

    publication.resolve("message-id");
    await logging;
    expect(metricInc).toHaveBeenCalledWith({
      table: "requests",
      outcome: "published",
    });
  });

  it("stores null, not a truncation, when the id exceeds the byte cap", async () => {
    // The header helper already drops these; this asserts the bound holds at
    // the log boundary too, for any writer that bypasses the helper. Null
    // rather than a rejected row, which would lose the whole requests row
    // (and its scrapes/crawls children) over a telemetry field — and null
    // rather than truncation, which would hand a wrong id back downstream.
    await logRequest(makeRequest("x".repeat(2049)));

    const inserted = JSON.parse(
      publishMessage.mock.calls[0][0].data.toString(),
    );
    expect(inserted.external_request_id).toBeNull();
    expect(inserted.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");
    expect(logger.warn).toHaveBeenCalled();
  });

  it("counts the cap in bytes, not characters", async () => {
    // 1025 two-byte characters: 1025 chars, 2050 bytes — over.
    await logRequest(makeRequest("é".repeat(1025)));
    const first = JSON.parse(publishMessage.mock.calls[0][0].data.toString());
    expect(first.external_request_id).toBeNull();

    // 1024 two-byte characters: 2048 bytes exactly — allowed.
    await logRequest(makeRequest("é".repeat(1024)));
    const second = JSON.parse(publishMessage.mock.calls[1][0].data.toString());
    expect(second.external_request_id).toBe("é".repeat(1024));
  });

  it("cleans NUL bytes and unpaired surrogates before publishing", async () => {
    // "Łódź" mis-decoded by a client arrives as a lone low surrogate, which
    // JSON.stringify would emit as "\udc81" and ClickPipes would reject as
    // invalid JSON. The row must be published already cleaned.
    const replacement = String.fromCharCode(0xfffd);
    await logRequest({
      ...makeRequest(null),
      target_hint: "wyciek Å\udc81ódź" + String.fromCharCode(0) + "!",
      origin: "api" + String.fromCharCode(0),
    });

    const raw = publishMessage.mock.calls[0][0].data.toString("utf8");
    expect(raw).not.toMatch(/\\u[dD][89a-fA-F]/);
    expect(raw).not.toMatch(/\\u0{4}/);
    const published = JSON.parse(raw);
    expect(published.target_hint).toBe("wyciek Å" + replacement + "ódź!");
    expect(published.origin).toBe("api");
  });

  it("refuses a publish beyond the outstanding cap and fails that log call", async () => {
    vi.resetModules();
    const fresh = await import("./log_job.js");
    const publication = deferred<string>();
    publishMessage.mockImplementation(async () => publication.promise);
    config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 2;
    try {
      const first = fresh.logRequest(makeRequest(null));
      const second = fresh.logRequest(makeRequest(null));
      await new Promise(resolve => setImmediate(resolve));
      await expect(fresh.logRequest(makeRequest(null))).rejects.toThrow(
        "backlog is full",
      );
      publication.resolve("message-id");
      await Promise.all([first, second]);
    } finally {
      config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 10_000;
      publication.resolve("message-id");
      await fresh.shutdownPubSubLogging();
    }

    expect(publishMessage).toHaveBeenCalledTimes(2);
    expect(metricInc).toHaveBeenCalledWith({
      table: "requests",
      outcome: "dropped",
    });
    // A refusal is the rate-limited warning, not a per-row error.
    expect(logger.error).not.toHaveBeenCalled();
    expect(metricInc).not.toHaveBeenCalledWith({
      table: "requests",
      outcome: "failed",
    });
    expect(logger.warn).toHaveBeenCalledWith(
      "Refusing Pub/Sub log: publisher backlog is full",
      expect.objectContaining({
        table: "requests",
        logId: makeRequest(null).id,
        outstandingMessages: 2,
        droppedTotal: 1,
      }),
    );
  });

  it("counts a published row and a failed row separately", async () => {
    await logRequest(makeRequest(null));
    await new Promise(resolve => setImmediate(resolve));
    expect(metricInc).toHaveBeenCalledWith({
      table: "requests",
      outcome: "published",
    });

    publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
    await expect(logRequest(makeRequest(null))).rejects.toThrow(
      "Pub/Sub unavailable",
    );
    expect(metricInc).toHaveBeenCalledWith({
      table: "requests",
      outcome: "failed",
    });
  });

  it("releases the backlog capacity after a failed publication", async () => {
    const originalCap = config.PUBSUB_MAX_OUTSTANDING_MESSAGES;
    config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 1;
    try {
      publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
      await expect(logRequest(makeRequest(null))).rejects.toThrow(
        "Pub/Sub unavailable",
      );
      await logRequest(makeRequest(null));
      expect(publishMessage).toHaveBeenCalledTimes(2);
      expect(metricInc).not.toHaveBeenCalledWith({
        table: "requests",
        outcome: "dropped",
      });
    } finally {
      config.PUBSUB_MAX_OUTSTANDING_MESSAGES = originalCap;
    }
  });

  it("flushes Pub/Sub messages during shutdown", async () => {
    await logRequest(makeRequest(null));

    await Promise.all([shutdownPubSubLogging(), shutdownPubSubLogging()]);

    expect(flush).toHaveBeenCalled();
    expect(close).toHaveBeenCalledOnce();
  });
});

describe("shutdownPubSubLogging deadline", () => {
  beforeEach(() => {
    vi.clearAllMocks();
    publishMessage.mockResolvedValue("message-id");
    publishes.length = 0;
  });

  afterEach(() => {
    vi.useRealTimers();
  });

  it("waits for pending publications even when flush returns early", async () => {
    vi.resetModules();
    const fresh = await import("./log_job.js");
    const publication = deferred<string>();
    publishMessage.mockReturnValueOnce(publication.promise);
    const logging = fresh.logSearch(makeSearch());
    await new Promise(resolve => setImmediate(resolve));

    const shutdown = fresh.shutdownPubSubLogging();
    await new Promise(resolve => setImmediate(resolve));
    expect(flush).toHaveBeenCalled();
    expect(close).not.toHaveBeenCalled();

    publication.resolve("message-id");
    await shutdown;
    await logging;
    expect(close).toHaveBeenCalledOnce();
  });

  it("closes at the deadline when publication stays pending after flush", async () => {
    vi.resetModules();
    const fresh = await import("./log_job.js");
    const publication = deferred<string>();
    publishMessage.mockReturnValueOnce(publication.promise);
    const logging = fresh.logSearch(makeSearch());
    await new Promise(resolve => setImmediate(resolve));

    vi.useFakeTimers();
    const shutdown = fresh.shutdownPubSubLogging();
    await vi.advanceTimersByTimeAsync(39_999);
    expect(close).not.toHaveBeenCalled();
    await vi.advanceTimersByTimeAsync(1);
    await shutdown;
    expect(close).toHaveBeenCalledOnce();
    expect(logger.warn).toHaveBeenCalledWith(
      "Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
      expect.objectContaining({
        outstandingMessages: 1,
        pendingLogSample: [
          expect.objectContaining({
            table: "searches",
            logId: makeSearch().id,
          }),
        ],
        pendingLogSampleTruncated: false,
      }),
    );

    publication.resolve("message-id");
    await logging;
  });

  it("reports publication failures during shutdown and still closes", async () => {
    vi.resetModules();
    const fresh = await import("./log_job.js");
    const publication = deferred<string>();
    publishMessage.mockReturnValueOnce(publication.promise);
    const logging = fresh.logSearch(makeSearch());
    await new Promise(resolve => setImmediate(resolve));
    const shutdown = fresh.shutdownPubSubLogging();

    publication.reject(new Error("Pub/Sub unavailable"));
    await shutdown;
    await expect(logging).rejects.toThrow("Pub/Sub unavailable");
    expect(close).toHaveBeenCalledOnce();
    expect(metricInc).toHaveBeenCalledWith({
      table: "searches",
      outcome: "failed",
    });
  });

  it("bounds client close after draining so shutdown can finish", async () => {
    vi.resetModules();
    const fresh = await import("./log_job.js");
    await fresh.logSearch(makeSearch());
    close.mockReturnValueOnce(new Promise(() => {}));
    vi.useFakeTimers();
    const shutdown = fresh.shutdownPubSubLogging();
    await vi.advanceTimersByTimeAsync(5_000);
    await shutdown;
    expect(logger.error).toHaveBeenCalledWith(
      "Failed to close Pub/Sub log publisher",
      expect.objectContaining({ error: expect.any(Error) }),
    );
  });

  it("reports late logs instead of publishing after shutdown starts", async () => {
    vi.resetModules();
    const fresh = await import("./log_job.js");
    await fresh.logSearch(makeSearch());
    const flushing = deferred<void>();
    flush.mockReturnValueOnce(flushing.promise);
    const shutdown = fresh.shutdownPubSubLogging();

    await expect(fresh.logSearch(makeSearch())).rejects.toThrow(
      "shutting down",
    );
    expect(publishMessage).toHaveBeenCalledOnce();
    expect(metricInc).toHaveBeenCalledWith({
      table: "searches",
      outcome: "failed",
    });

    flushing.resolve();
    await shutdown;
  });

  it("closes the client when a flush outlives the shutdown deadline", async () => {
    // A fresh module instance: shutdown is memoized per process.
    vi.resetModules();
    const fresh = await import("./log_job.js");
    await fresh.logRequest({
      id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
      kind: "scrape",
      api_version: "v2",
      team_id: "team-id",
      origin: "api",
      target_hint: "https://example.com",
      zeroDataRetention: false,
      api_key_id: null,
      external_request_id: null,
    });

    vi.useFakeTimers();
    flush.mockReturnValueOnce(new Promise(() => {}));
    const shutdown = fresh.shutdownPubSubLogging();
    await vi.advanceTimersByTimeAsync(40_000);
    await shutdown;

    expect(close).toHaveBeenCalledOnce();
    expect(logger.warn).toHaveBeenCalledWith(
      "Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
      expect.objectContaining({ timeoutMs: 40_000 }),
    );
  });
});