Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

43.8 KB
Raw Blame
import { changeTrackingInsertScrape } from "../../lib/change-tracking-store";
import { config } from "../../config";
import { enqueueZdrCleanupJob } from "../../lib/zdr-queue";
import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";
import { configDotenv } from "dotenv";
import {
  saveDeepResearchToGCS,
  saveExtractToGCS,
  saveLlmsTxtToGCS,
  saveMapToGCS,
  saveScrapeToGCS,
  saveSearchToGCS,
} from "../../lib/gcs-jobs";
import { hasFormatOfType } from "../../lib/format-utils";
import { keylessTeamUuid } from "../../lib/keyless";
import type { Document, ScrapeOptions } from "../../controllers/v2/types";
import type {
  SearchResultCountsBySource,
  SearchResultCategoriesBySource,
} from "../../lib/entities";
import type { CostTracking } from "../../lib/cost-tracking";
import type { ErrorCodes } from "../../lib/error";
import type { Logger } from "winston";
import { saveExtractResult } from "../../lib/extract/extract-redis";
import { trackFirstSurfaceUse } from "../posthog";
import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";
import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";
import { sanitizeLogData, sanitizeText } from "./sanitize";
import { redactLogData } from "./redact";
import {
  JOB_ACCESS_TTL_MS,
  isApiJobKind,
  normalizeJobAccessTeamId,
  writeApiJobAccess,
} from "../../lib/job-access-store";
import { writeFeedbackJob } from "../../lib/feedback-job-store";
import { setSpanAttributes, withSpan } from "../../lib/otel-tracer";
import {
  writeExtractJobState,
  writeScrapeJobState,
} from "../../lib/job-state-store";
import { buildReplayContextFromScrape } from "../../lib/scrape-interact/scrape-replay";
import { browserOptionsFromScrape } from "../../lib/browser-options";
import {
  initializeRequestCredits,
  recordRequestCredits,
} from "../../lib/request-credits-store";
configDotenv();

const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
const DEFAULT_JOB_ACCESS_TTL_MS = JOB_ACCESS_TTL_MS;

async function withLogSpan<T>(
  params: {
    operation: string;
    table: string;
    id: string;
    requestId?: string;
    zeroDataRetention?: boolean;
  },
  fn: () => Promise<T>,
): Promise<T> {
  return withSpan(
    `log_job.${params.operation}`,
    async span => {
      setSpanAttributes(span, {
        "log_job.table": params.table,
        "log_job.id": params.id,
        "log_job.request_id": params.requestId,
      });
      return fn();
    },
    { zeroDataRetention: params.zeroDataRetention },
  );
}

async function writeFeedbackJobSafely(
  params: Parameters<typeof writeFeedbackJob>[0],
  logger: Logger,
): Promise<void> {
  try {
    await writeFeedbackJob(params);
  } catch (error) {
    logger.error("Failed to write feedback job to Bigtable", {
      error,
      jobId: params.jobId,
      endpoint: params.endpoint,
    });
  }
}

async function recordRequestCreditsSafely(
  params: Parameters<typeof recordRequestCredits>[0],
  logger: Logger,
): Promise<void> {
  try {
    await recordRequestCredits(params);
  } catch (error) {
    logger.error("Failed to record request credits in Bigtable", {
      error,
      requestId: params.requestId,
      jobId: params.jobId,
      credits: params.credits,
    });
  }
}

/**
 * Null-aware wrapper around the shared text sanitizer, kept where a cleaned
 * value feeds a later decision (the external_request_id byte cap). Every row
 * is deep-sanitized again in robustInsert before it reaches either store, so
 * nothing else depends on this being called; see ./sanitize.ts for what gets
 * cleaned and why.
 */
function sanitizeString(value: string | null | undefined): string | null {
  if (value === null || value === undefined) return null;

  return sanitizeText(value);
}

let pubSubClient: PubSub | null | undefined;
const pubSubTopics = new Map<string, Topic>();
let pubSubShutdown: Promise<void> | undefined;

// Publish retry policy. Passing only `timeout` makes google-gax collapse the
// whole retry budget to that one value (CallSettings.merge), so a stalled RPC
// used to be a single 60 s attempt and then a lost row. An explicit `retry`
// is applied after that override and replaces the backoff settings wholesale.
// Short attempts detect a stalled RPC quickly. The total budget allows
// retries across a longer connection disruption.
// Retry codes stay the client's defaults for Publish (DEADLINE_EXCEEDED,
// UNAVAILABLE, INTERNAL, UNKNOWN, ABORTED, CANCELLED, RESOURCE_EXHAUSTED).
// A retry can deliver a batch twice when the first attempt was persisted but
// its response was lost; the ClickHouse tables dedupe on row id, not message
// id, so those copies collapse.
//
// The log call waits for the publish, so the budget is what a caller can be
// held for while Pub/Sub is unreachable before the log call fails.
const PUBSUB_PUBLISH_OPTIONS: PublishOptions = {
  gaxOpts: {
    retry: {
      backoffSettings: {
        initialRetryDelayMillis: 250,
        retryDelayMultiplier: 2,
        maxRetryDelayMillis: 5_000,
        initialRpcTimeoutMillis: 10_000,
        rpcTimeoutMultiplier: 1,
        maxRpcTimeoutMillis: 10_000,
        totalTimeoutMillis: 30_000,
      },
    },
  },
};

// Shutdown waits this long for in-flight publishes before closing the client.
// The drain shares the existing pod grace period with active work and exit.
// Memory kills and work that exceeds the pod grace period can still lose logs.
const PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS = 40_000;

// Track publication promises because topic.flush() can finish before an active RPC.
const pendingPublications = new Map<
  Promise<string>,
  { table: string; logId: string; startedAt: number }
>();
let outstandingBytes = 0;
let droppedTotal = 0;
let lastDropWarningAt = 0;

function getPubSubClient(logger: Logger): PubSub | null {
  if (pubSubClient !== undefined) return pubSubClient;
  if (!config.PUBSUB_CREDENTIALS?.trim()) {
    return (pubSubClient = config.PUBSUB_PROJECT_ID
      ? new PubSub({ projectId: config.PUBSUB_PROJECT_ID })
      : null);
  }

  try {
    const credentials = JSON.parse(
      Buffer.from(config.PUBSUB_CREDENTIALS, "base64").toString("utf8"),
    );
    return (pubSubClient = new PubSub({
      projectId: credentials.project_id,
      credentials,
    }));
  } catch (error) {
    pubSubClient = null;
    logger.error("Failed to initialize Pub/Sub log publisher", { error });
    return null;
  }
}

// One Topic per table so publishes share a batch. The topic is named after
// the table, behind the optional environment prefix (see PUBSUB_TOPIC_PREFIX).
function getTopic(client: PubSub, table: string): Topic {
  const name = `${config.PUBSUB_TOPIC_PREFIX}${table}`;
  let topic = pubSubTopics.get(name);
  if (!topic) {
    topic = client.topic(name, PUBSUB_PUBLISH_OPTIONS);
    pubSubTopics.set(name, topic);
  }
  return topic;
}

/** The publisher already holds as many unacknowledged rows as it may. */
class PubSubBacklogFullError extends Error {
  constructor() {
    super("Pub/Sub log publisher backlog is full");
    this.name = "PubSubBacklogFullError";
  }
}

/**
 * Publishes one job-log row to its Pub/Sub topic and waits for the
 * acknowledgement. Pub/Sub is the log's store of record, so a publish that
 * fails, is refused by a full backlog, or arrives during shutdown throws to
 * the caller; nothing else is written for that row.
 */
async function publishLog(table: string, data: any, logger: Logger) {
  const startedAt = Date.now();
  try {
    await withSpan("log_job.pubsub.publish", async span => {
      setSpanAttributes(span, {
        "log_job.table": table,
        "log_job.id": data.id,
        "messaging.system": "gcp_pubsub",
      });

      if (pubSubShutdown) {
        throw new Error("Pub/Sub log publisher is shutting down");
      }
      const client = getPubSubClient(logger);
      if (!client) {
        if (config.PUBSUB_CREDENTIALS?.trim() || config.PUBSUB_PROJECT_ID) {
          throw new Error("Pub/Sub log publisher initialization failed");
        }
        setSpanAttributes(span, {
          "log_job.pubsub.enabled": false,
          "log_job.pubsub.outcome": "skipped",
        });
        return;
      }

      const payload = Buffer.from(JSON.stringify(data));
      setSpanAttributes(span, {
        "log_job.pubsub.enabled": true,
        "log_job.pubsub.payload_bytes": payload.length,
      });
      if (
        pendingPublications.size >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||
        outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES
      ) {
        droppedTotal++;
        pubsubLogPublishTotal.inc({ table, outcome: "dropped" });
        setSpanAttributes(span, { "log_job.pubsub.outcome": "dropped" });
        const now = Date.now();
        if (now - lastDropWarningAt >= 60_000) {
          lastDropWarningAt = now;
          logger.warn("Refusing Pub/Sub log: publisher backlog is full", {
            table,
            logId: data.id,
            payloadBytes: payload.length,
            outstandingMessages: pendingPublications.size,
            outstandingBytes,
            droppedTotal,
          });
        }
        throw new PubSubBacklogFullError();
      }

      const publication = getTopic(client, table).publishMessage({
        data: payload,
      });
      pendingPublications.set(publication, {
        table,
        logId: data.id,
        startedAt,
      });
      outstandingBytes += payload.length;

      try {
        await publication;
        pubsubLogPublishTotal.inc({ table, outcome: "published" });
        setSpanAttributes(span, { "log_job.pubsub.outcome": "published" });
      } finally {
        pendingPublications.delete(publication);
        outstandingBytes -= payload.length;
      }
    });
  } catch (error) {
    // A backlog refusal counted itself as "dropped" and warned, rate-limited,
    // above; only a genuine publish failure is an error here.
    if (!(error instanceof PubSubBacklogFullError)) {
      pubsubLogPublishTotal.inc({ table, outcome: "failed" });
      logger.error("Failed to publish log to Pub/Sub", {
        error,
        table,
        logId: data.id,
        durationMs: Date.now() - startedAt,
      });
    }
    throw error;
  }
}

export function shutdownPubSubLogging(): Promise<void> {
  if (pubSubShutdown) return pubSubShutdown;

  pubSubShutdown = shutdownPubSubLoggingOnce();
  return pubSubShutdown;
}

async function shutdownPubSubLoggingOnce(): Promise<void> {
  const client = pubSubClient;
  if (!client) return;

  const logger = _logger.child({
    module: "log_job",
    method: "shutdownPubSubLogging",
  });
  const startedAt = Date.now();
  logger.info("Draining Pub/Sub log publisher", {
    outstandingMessages: pendingPublications.size,
    outstandingBytes,
    timeoutMs: PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,
  });
  // Callers stop accepting work before shutdown. Reject new publications so
  // this snapshot includes every publication that can still use the client.
  const flushed = Promise.allSettled([
    ...[...pubSubTopics.values()].map(async topic => topic.flush()),
    ...pendingPublications.keys(),
  ]);
  let deadline: NodeJS.Timeout | undefined;
  const timedOut = new Promise<"timeout">(resolve => {
    deadline = setTimeout(
      () => resolve("timeout"),
      PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,
    );
  });
  const results = await Promise.race([flushed, timedOut]);
  clearTimeout(deadline);

  if (results === "timeout") {
    logger.warn(
      "Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
      {
        timeoutMs: PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,
        outstandingMessages: pendingPublications.size,
        outstandingBytes,
        pendingLogSample: [...pendingPublications.values()].slice(0, 50),
        pendingLogSampleTruncated: pendingPublications.size > 50,
      },
    );
  } else {
    const errors = results.flatMap(result =>
      result.status === "rejected" ? [result.reason] : [],
    );

    if (errors.length > 0) {
      logger.error("Failed to drain Pub/Sub log publisher", { errors });
    } else {
      logger.info("Pub/Sub log publisher drained", {
        durationMs: Date.now() - startedAt,
      });
    }
  }

  let closeDeadline: NodeJS.Timeout | undefined;
  try {
    await Promise.race([
      client.close(),
      new Promise<never>((_, reject) => {
        closeDeadline = setTimeout(
          () => reject(new Error("Pub/Sub client close exceeded 5 seconds")),
          5_000,
        );
      }),
    ]);
  } catch (error) {
    logger.error("Failed to close Pub/Sub log publisher", { error });
  } finally {
    clearTimeout(closeDeadline);
  }
}

async function robustInsert(table: string, data: any, _logger: Logger) {
  const logger = _logger.child({
    module: "log_job",
    method: "robustInsert",
    table,
    canonicalLog: "log_job/robustInsert",
  });

  if (config.USE_DB_AUTHENTICATION !== true) {
    logger.info("Skipping job log due to USE_DB_AUTHENTICATION being off");
    return;
  }

  // Redact credentials and internal fields, then clean the row once so the
  // log receives accepted values.
  data = sanitizeLogData(
    redactLogData({
      ...data,
      created_at: data.created_at ?? new Date(),
    }),
  );

  // Pub/Sub is the store of record: publish and wait for it. A failed
  // publish fails the log call.
  await publishLog(table, data, logger);
}

type LoggedRequest = {
  id: string;
  kind:
    | "alexandria"
    | "scrape"
    | "crawl"
    | "batch_scrape"
    | "search"
    | "extract"
    | "llmstxt"
    | "deep_research"
    | "map"
    | "parse"
    | "agent"
    | "browser"
    | "interact"
    | "research_paper_search"
    | "research_paper_inspect"
    | "research_paper_read"
    | "research_related_papers"
    | "code_search"
    | "gov_search"
    | "fresh_feed";
  api_version: string;
  team_id: string;
  origin?: string;
  integration?: string | null;
  target_hint: string;
  zeroDataRetention: boolean;
  api_key_id?: number | null;
  /**
   * Opaque per-operation id a caller sent as `External-Request-Id` (see
   * `lib/external-request-id.ts`), stored for internal billing attribution
   * and read back off this row by the request id.
   */
  external_request_id?: string | null;
  jobAccess?: boolean;
  jobAccessExpiresAt?: Date;
  /** An agent run's ZDR retention, recorded on its job access row. */
  jobAccessZeroDataRetentionMinutes?: number;
  creditsShards?: number;
};

/**
 * The 2048-byte cap, re-checked at the one place the column is written.
 *
 * Belt and braces: the header helper (`lib/external-request-id.ts`) already
 * drops oversized ids, but the bound must hold even for a future writer that
 * bypasses it — and it cannot live in the database, where a length constraint
 * would fail the whole `requests` insert (and the `scrapes`/`crawls` rows that
 * FK into it) over a telemetry field. Oversized means null, never truncation:
 * a truncated opaque id handed back downstream would be actively wrong, where
 * an absent one is an honest reporting gap.
 */
function boundedExternalRequestId(
  value: string | null,
  logger: Logger,
): string | null {
  if (value === null) return null;
  if (Buffer.byteLength(value) <= EXTERNAL_REQUEST_ID_MAX_BYTES) return value;
  logger.warn(
    "external_request_id exceeds the cap at the insert boundary; storing null",
    {
      bytes: Buffer.byteLength(value),
      max: EXTERNAL_REQUEST_ID_MAX_BYTES,
    },
  );
  return null;
}

const inFlightZdrRequestLogs = new Set<Promise<void>>();

export async function logRequest(request: LoggedRequest) {
  const log = withLogSpan(
    {
      operation: "request",
      table: "requests",
      id: request.id,
      zeroDataRetention: request.zeroDataRetention,
    },
    () => logRequestInternal(request),
  );
  if (!request.zeroDataRetention) return log;
  inFlightZdrRequestLogs.add(log);
  try {
    return await log;
  } finally {
    inFlightZdrRequestLogs.delete(log);
  }
}

/** Waits for in-flight ZDR request logs, which enqueue their cleanup job. */
export async function drainZdrRequestLogs(): Promise<void> {
  await Promise.allSettled([...inFlightZdrRequestLogs]);
}

async function logRequestInternal(request: LoggedRequest) {
  const logger = _logger.child({
    module: "log_job",
    method: "logRequest",
    requestId: request.id,
    teamId: request.team_id,
    zeroDataRetention: request.zeroDataRetention,
  });

  // Emit a one-time PostHog milestone the first time this team uses each
  // surface (playground / sdk / mcp / cli / api / ...). Fire-and-forget.
  // Skip zero-data-retention requests — don't send their metadata to PostHog.
  if (!request.zeroDataRetention) {
    trackFirstSurfaceUse({
      teamId: request.team_id,
      origin: request.origin,
      integration: request.integration,
      kind: request.kind,
      apiVersion: request.api_version,
      apiKeyId: request.api_key_id,
    });
  }

  // Sanitize user-provided fields (most likely sources of null bytes)
  const sanitizedOrigin = sanitizeString(request.origin);
  const sanitizedIntegration = sanitizeString(request.integration ?? null);
  const sanitizedTargetHint = request.zeroDataRetention
    ? "<redacted due to zero data retention>"
    : sanitizeString(request.target_hint);
  const storedTeamId =
    request.team_id === "preview" || request.team_id?.startsWith("preview_")
      ? previewTeamId
      : request.team_id;
  const jobAccessTeamId = normalizeJobAccessTeamId(request.team_id);

  if (request.jobAccess !== false && isApiJobKind(request.kind)) {
    try {
      await writeApiJobAccess({
        id: request.id,
        teamId: jobAccessTeamId,
        kind: request.kind,
        expiresAt:
          request.jobAccessExpiresAt ??
          new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),
        clientOrigin: sanitizedOrigin,
        zeroDataRetention: request.zeroDataRetention,
        zeroDataRetentionMinutes: request.jobAccessZeroDataRetentionMinutes,
      });
    } catch (error) {
      logger.error("Failed to write API job access to Bigtable", {
        error,
        kind: request.kind,
      });
    }
  }

  if (request.creditsShards !== undefined) {
    try {
      await initializeRequestCredits(request.id, request.creditsShards);
    } catch (error) {
      logger.error("Failed to initialize request credits in Bigtable", {
        error,
        shards: request.creditsShards,
      });
    }
  }

  if (request.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {
    await enqueueZdrCleanupJob({ requestId: request.id });
  }

  await robustInsert(
    "requests",
    {
      id: request.id,
      kind: request.kind,
      api_version: request.api_version,
      team_id: storedTeamId,
      origin: sanitizedOrigin,
      integration: sanitizedIntegration,
      target_hint: sanitizedTargetHint,
      api_key_id: request.api_key_id ?? null,
      // Not redacted under zero data retention: it is the caller's own
      // operation id (attribution it asked for), not customer content — and
      // the row's blobs are cleaned by the ZDR queue regardless.
      external_request_id: boundedExternalRequestId(
        sanitizeString(request.external_request_id ?? null),
        logger,
      ),
    },
    logger,
  );
}

export type LoggedScrape = {
  id: string;
  request_id: string;
  url: string;
  is_successful: boolean;
  error?: string;
  /** Code of a failed scrape's error; the only error detail kept under zero data retention. */
  error_code?: ErrorCodes;
  doc?: Document;
  time_taken: number;
  team_id: string;
  options: ScrapeOptions;
  cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;
  pdf_num_pages?: number;
  content_type?: string | null;
  credits_cost: number;
  skipNuq: boolean;
  zeroDataRetention: boolean;
  is_parse?: boolean;
  monitor_id?: string | null;
  monitor_check_id?: string | null;
  /** False for jobs that must not be fetchable by id (defaults to true). */
  jobAccess?: boolean;
};

export type ScrapeStateOutcome =
  /** The terminal state is readable from Bigtable. */
  | "written"
  /** Nothing to write: a parse job, or no state table configured. */
  | "skipped"
  /** The Bigtable write threw; the error is logged here. */
  | "failed";

type LogScrapeHooks = {
  /**
   * Called once the terminal state write has settled, before anything else
   * is logged. The sync scrape path waits on this before answering, so an
   * interact call that follows the response finds its replay context.
   */
  onStateWritten?: (outcome: ScrapeStateOutcome) => void;
};

export async function logScrape(scrape: LoggedScrape, hooks?: LogScrapeHooks) {
  return withLogSpan(
    {
      operation: scrape.is_parse ? "parse" : "scrape",
      table: scrape.is_parse ? "parses" : "scrapes",
      id: scrape.id,
      requestId: scrape.request_id,
      zeroDataRetention: scrape.zeroDataRetention,
    },
    () => logScrapeInternal(scrape, hooks),
  );
}

async function logScrapeInternal(scrape: LoggedScrape, hooks?: LogScrapeHooks) {
  const logger = _logger.child({
    module: "log_job",
    method: "logScrape",
    scrapeId: scrape.id,
    requestId: scrape.request_id,
    teamId: scrape.team_id,
    zeroDataRetention: scrape.zeroDataRetention,
  });

  const tableName = scrape.is_parse ? "parses" : "scrapes";
  const storedTeamId =
    keylessTeamUuid(scrape.team_id) ??
    (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
      ? previewTeamId
      : scrape.team_id);

  // Terminal state for every scrape job, standalone or child, so status reads
  // never need the PostgreSQL row. It goes first: the sync scrape response
  // waits for it, and nothing else here has to be readable that early.
  let stateOutcome: ScrapeStateOutcome = "skipped";
  try {
    if (!scrape.is_parse) {
      const replay = scrape.zeroDataRetention
        ? undefined
        : buildReplayContextFromScrape({
            id: scrape.id,
            team_id: storedTeamId,
            url: scrape.url,
            options: scrape.options,
          }).context;
      const written = await writeScrapeJobState(scrape.id, {
        ...(scrape.zeroDataRetention ? { zeroDataRetention: true } : {}),
        status: scrape.is_successful ? "completed" : "failed",
        requestId: scrape.request_id,
        completedAtMs: Date.now(),
        creditsBilled: scrape.credits_cost,
        ...(scrape.zeroDataRetention
          ? {}
          : {
              ...(scrape.error ? { error: scrape.error } : {}),
              ...(replay ? { replay } : {}),
              browser: browserOptionsFromScrape(scrape.options),
              ...(typeof (scrape.options as any).origin === "string"
                ? { origin: (scrape.options as any).origin }
                : {}),
            }),
      });
      stateOutcome = written ? "written" : "skipped";
    }
  } catch (error) {
    stateOutcome = "failed";
    logger.error("Failed to write scrape state to Bigtable", { error });
  } finally {
    hooks?.onStateWritten?.(stateOutcome);
  }

  const feedbackJob = {
    jobId: scrape.id,
    requestId: scrape.request_id,
    teamId: storedTeamId,
    succeeded: scrape.is_successful,
    creditsBilled: scrape.credits_cost,
    zeroDataRetention: scrape.zeroDataRetention,
  };
  await writeFeedbackJobSafely(
    scrape.is_parse
      ? { ...feedbackJob, endpoint: "parse" }
      : { ...feedbackJob, endpoint: "scrape", scrapeOptions: scrape.options },
    logger,
  );

  await robustInsert(
    tableName,
    {
      id: scrape.id,
      request_id: scrape.request_id,
      url: scrape.zeroDataRetention
        ? "<redacted due to zero data retention>"
        : scrape.url,
      is_successful: scrape.is_successful,
      error: scrape.zeroDataRetention
        ? scrape.is_successful
          ? null
          : (scrape.error_code ?? "<redacted due to zero data retention>")
        : (scrape.error ?? null),
      time_taken: scrape.time_taken,
      team_id: storedTeamId,
      options: scrape.zeroDataRetention ? null : scrape.options,
      cost_tracking: scrape.zeroDataRetention
        ? null
        : (scrape.cost_tracking ?? null),
      pdf_num_pages: scrape.zeroDataRetention
        ? null
        : (scrape.pdf_num_pages ?? null),
      credits_cost: scrape.credits_cost,
      ...(scrape.is_parse
        ? {}
        : {
            monitor_id: scrape.monitor_id ?? null,
            monitor_check_id: scrape.monitor_check_id ?? null,
            content_type: scrape.content_type ?? null,
          }),
    },
    logger,
  );

  if (scrape.id !== scrape.request_id) {
    await recordRequestCreditsSafely(
      {
        requestId: scrape.request_id,
        jobId: scrape.id,
        credits: scrape.credits_cost,
      },
      logger,
    );

    // A crawl or batch child is fetched by its own id (`GET /scrape/{id}`,
    // typically off the crawl `page` webhook), so it needs its own access row.
    // Standalone scrapes get theirs from logRequest.
    if (!scrape.is_parse && scrape.jobAccess !== false) {
      try {
        await writeApiJobAccess({
          id: scrape.id,
          teamId: normalizeJobAccessTeamId(scrape.team_id),
          kind: "scrape",
          expiresAt: new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),
          zeroDataRetention: scrape.zeroDataRetention,
        });
      } catch (error) {
        logger.error("Failed to write child scrape job access to Bigtable", {
          error,
        });
      }
    }
  }

  if (
    !scrape.is_parse &&
    scrape.doc &&
    config.GCS_BUCKET_NAME &&
    !(scrape.skipNuq && scrape.zeroDataRetention)
  ) {
    await saveScrapeToGCS(scrape, logger);
  }

  if (
    !scrape.is_parse &&
    scrape.is_successful &&
    !scrape.zeroDataRetention &&
    config.USE_DB_AUTHENTICATION &&
    !scrape.team_id.startsWith("preview_")
  ) {
    const hasMarkdown = hasFormatOfType(scrape.options.formats, "markdown");
    const hasChangeTracking = hasFormatOfType(
      scrape.options.formats,
      "changeTracking",
    );

    if (hasMarkdown || hasChangeTracking) {
      try {
        await changeTrackingInsertScrape({
          team_id: scrape.team_id,
          url: scrape.url,
          job_id: scrape.id,
          tag: hasChangeTracking ? hasChangeTracking.tag : null,
          date_added: new Date(),
        });
        _logger.debug("Change tracking record inserted successfully");
      } catch (error) {
        _logger.warn("Error inserting into change_tracking_scrapes", {
          error,
          scrapeId: scrape.id,
          teamId: scrape.team_id,
        });
      }
    }
  }
}

type LoggedProviderScrape = {
  id: string;
  request_id: string;
  target: string;
  team_id: string;
  options: unknown;
  time_taken: number;
  credits_cost: number;
  is_successful: boolean;
  error?: string;
  zeroDataRetention: boolean;
};

export async function logProviderScrape(scrape: LoggedProviderScrape) {
  return withLogSpan(
    {
      operation: "provider_scrape",
      table: "scrapes",
      id: scrape.id,
      requestId: scrape.request_id,
      zeroDataRetention: scrape.zeroDataRetention,
    },
    () => logProviderScrapeInternal(scrape),
  );
}

async function logProviderScrapeInternal(scrape: LoggedProviderScrape) {
  const logger = _logger.child({
    module: "log_job",
    method: "logProviderScrape",
    scrapeId: scrape.id,
    requestId: scrape.request_id,
    teamId: scrape.team_id,
  });
  const storedTeamId =
    keylessTeamUuid(scrape.team_id) ??
    (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
      ? previewTeamId
      : scrape.team_id);

  await robustInsert(
    "scrapes",
    {
      id: scrape.id,
      request_id: scrape.request_id,
      url: scrape.zeroDataRetention
        ? "<redacted due to zero data retention>"
        : scrape.target,
      is_successful: scrape.is_successful,
      error: scrape.zeroDataRetention
        ? scrape.error === undefined
          ? null
          : "<redacted due to zero data retention>"
        : (scrape.error ?? null),
      time_taken: scrape.time_taken,
      team_id: storedTeamId,
      options: scrape.zeroDataRetention ? null : scrape.options,
      cost_tracking: null,
      pdf_num_pages: null,
      credits_cost: scrape.credits_cost,
      monitor_id: null,
      monitor_check_id: null,
      content_type: null,
    },
    logger,
  );
}

type LoggedCrawl = {
  id: string;
  request_id: string;
  url: string;
  team_id: string;
  options: any;
  num_docs: number;
  credits_cost: number;
  zeroDataRetention: boolean;
  cancelled: boolean;
  monitor_id?: string | null;
  monitor_check_id?: string | null;
};

export async function logCrawl(crawl: LoggedCrawl) {
  return withLogSpan(
    {
      operation: "crawl",
      table: "crawls",
      id: crawl.id,
      requestId: crawl.request_id,
      zeroDataRetention: crawl.zeroDataRetention,
    },
    () => logCrawlInternal(crawl),
  );
}

async function logCrawlInternal(crawl: LoggedCrawl) {
  const logger = _logger.child({
    module: "log_job",
    method: "logCrawl",
    crawlId: crawl.id,
    requestId: crawl.request_id,
    teamId: crawl.team_id,
    zeroDataRetention: crawl.zeroDataRetention,
  });

  await robustInsert(
    "crawls",
    {
      id: crawl.id,
      request_id: crawl.request_id,
      url: crawl.zeroDataRetention
        ? "<redacted due to zero data retention>"
        : crawl.url,
      team_id:
        crawl.team_id === "preview" || crawl.team_id?.startsWith("preview_")
          ? previewTeamId
          : crawl.team_id,
      options: crawl.zeroDataRetention ? null : crawl.options,
      num_docs: crawl.num_docs,
      credits_cost: crawl.credits_cost,
      cancelled: crawl.cancelled,
      monitor_id: crawl.monitor_id ?? null,
      monitor_check_id: crawl.monitor_check_id ?? null,
    },
    logger,
  );
}

type LoggedBatchScrape = {
  id: string;
  request_id: string;
  team_id: string;
  num_docs: number;
  credits_cost: number;
  zeroDataRetention: boolean;
  cancelled: boolean;
};

export async function logBatchScrape(batchScrape: LoggedBatchScrape) {
  return withLogSpan(
    {
      operation: "batch_scrape",
      table: "batch_scrapes",
      id: batchScrape.id,
      requestId: batchScrape.request_id,
      zeroDataRetention: batchScrape.zeroDataRetention,
    },
    () => logBatchScrapeInternal(batchScrape),
  );
}

async function logBatchScrapeInternal(batchScrape: LoggedBatchScrape) {
  const logger = _logger.child({
    module: "log_job",
    method: "logBatchScrape",
    batchScrapeId: batchScrape.id,
    requestId: batchScrape.request_id,
    teamId: batchScrape.team_id,
    zeroDataRetention: batchScrape.zeroDataRetention,
  });

  await robustInsert(
    "batch_scrapes",
    {
      id: batchScrape.id,
      request_id: batchScrape.request_id,
      team_id:
        batchScrape.team_id === "preview" ||
        batchScrape.team_id?.startsWith("preview_")
          ? previewTeamId
          : batchScrape.team_id,
      num_docs: batchScrape.num_docs,
      credits_cost: batchScrape.credits_cost,
      cancelled: batchScrape.cancelled,
    },
    logger,
  );
}

export type LoggedSearch = {
  id: string;
  request_id: string;
  query: string;
  team_id: string;
  options: any;
  time_taken: number;
  credits_cost: number;
  is_successful: boolean;
  error?: string;
  num_results: number;
  num_results_by_source?: SearchResultCountsBySource;
  result_categories?: SearchResultCategoriesBySource;
  results: any;
  zeroDataRetention: boolean;
};

export async function logSearch(search: LoggedSearch) {
  return withLogSpan(
    {
      operation: "search",
      table: "searches",
      id: search.id,
      requestId: search.request_id,
      zeroDataRetention: search.zeroDataRetention,
    },
    () => logSearchInternal(search),
  );
}

async function logSearchInternal(search: LoggedSearch) {
  const logger = _logger.child({
    module: "log_job",
    method: "logSearch",
    searchId: search.id,
    requestId: search.request_id,
    teamId: search.team_id,
    zeroDataRetention: search.zeroDataRetention,
  });

  const options =
    search.zeroDataRetention || typeof search.options?.query !== "string"
      ? search.options
      : { ...search.options, query: sanitizeString(search.options.query) };
  const storedTeamId =
    search.team_id === "preview" || search.team_id?.startsWith("preview_")
      ? previewTeamId
      : search.team_id;

  await writeFeedbackJobSafely(
    {
      jobId: search.id,
      requestId: search.request_id,
      teamId: storedTeamId,
      endpoint: "search",
      succeeded: search.is_successful,
      creditsBilled: search.credits_cost,
      zeroDataRetention: search.zeroDataRetention,
      // Position feedback bounds each `{source, position}` against the group
      // it addresses, and attributes it to the vertical that served it. This
      // is the record feedback reads, and the one the feedback window is
      // enforced from, so the counts travel with it. `writeFeedbackJob`
      // redacts both under zero data retention.
      //
      // Only v2 search reports counts. A v0/v1 search returns a flat list
      // whose positions do not map to `{source, position}`, so it records `{}`
      // and every position is rejected. Leaving it absent would read as a job
      // that predates the field and accept any position unbounded.
      numResultsBySource: search.num_results_by_source ?? {},
      resultCategories: search.result_categories,
    },
    logger,
  );

  await robustInsert(
    "searches",
    {
      id: search.id,
      request_id: search.request_id,
      query: search.zeroDataRetention
        ? "<redacted due to zero data retention>"
        : sanitizeString(search.query),
      team_id: storedTeamId,
      options: search.zeroDataRetention
        ? { enterprise: search.options?.enterprise }
        : options,
      credits_cost: search.credits_cost,
      is_successful: search.is_successful,
      error: search.zeroDataRetention ? null : (search.error ?? null),
      num_results: search.num_results,
      // Both are response-derived and both are redacted under zero data
      // retention: the per-source split says how the result set was composed
      // ({"web":3,"news":10} characterises the query's coverage in a way the
      // combined total does not), and the category map says which vertical
      // served each position. Feedback on a ZDR search is not persisted
      // anyway, so neither column has a reader for those rows.
      num_results_by_source: search.zeroDataRetention
        ? null
        : (search.num_results_by_source ?? null),
      result_categories: search.zeroDataRetention
        ? null
        : (search.result_categories ?? null),
      time_taken: search.time_taken,
    },
    logger,
  );

  if (search.id !== search.request_id) {
    await recordRequestCreditsSafely(
      {
        requestId: search.request_id,
        jobId: search.id,
        credits: search.credits_cost,
      },
      logger,
    );
  }

  if (search.results && !search.zeroDataRetention) {
    await saveSearchToGCS(search, logger);
  }
}

export type ResearchRequestKind =
  | "research_paper_search"
  | "research_paper_inspect"
  | "research_paper_read"
  | "research_related_papers"
  | "code_search"
  | "gov_search";

export type ResearchTableName =
  | "research_paper_searches"
  | "research_paper_inspects"
  | "research_paper_reads"
  | "research_related_papers"
  | "code_searches"
  | "gov_searches";

type LoggedResearchEndpoint = {
  table: ResearchTableName;
  id: string;
  request_id: string;
  target: string;
  team_id: string;
  options: any;
  response: any;
  num_results: number;
  time_taken: number;
  credits_cost: number;
  is_successful: boolean;
  error?: string;
  zeroDataRetention: boolean;
};

export async function logResearchEndpoint(research: LoggedResearchEndpoint) {
  return withLogSpan(
    {
      operation: "research",
      table: research.table,
      id: research.id,
      requestId: research.request_id,
      zeroDataRetention: research.zeroDataRetention,
    },
    () => logResearchEndpointInternal(research),
  );
}

async function logResearchEndpointInternal(research: LoggedResearchEndpoint) {
  const logger = _logger.child({
    module: "log_job",
    method: "logResearchEndpoint",
    researchId: research.id,
    requestId: research.request_id,
    teamId: research.team_id,
    zeroDataRetention: research.zeroDataRetention,
  });

  await robustInsert(
    research.table,
    {
      id: research.id,
      request_id: research.request_id,
      target: research.zeroDataRetention
        ? "<redacted due to zero data retention>"
        : (sanitizeString(research.target) ?? ""),
      team_id:
        keylessTeamUuid(research.team_id) ??
        (research.team_id === "preview" ||
        research.team_id?.startsWith("preview_")
          ? previewTeamId
          : research.team_id),
      options: research.zeroDataRetention ? null : research.options,
      response: research.zeroDataRetention ? null : research.response,
      num_results: research.num_results,
      time_taken: research.time_taken,
      credits_cost: research.credits_cost,
      is_successful: research.is_successful,
      error: research.zeroDataRetention ? null : (research.error ?? null),
    },
    logger,
  );
}

export type LoggedExtract = {
  id: string;
  request_id: string;
  urls: string[];
  team_id: string;
  options: any;
  model_kind: "fire-0" | "fire-1";
  credits_cost: number;
  is_successful: boolean;
  error?: string;
  result?: any;
  cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;
};

export async function logExtract(extract: LoggedExtract) {
  return withLogSpan(
    {
      operation: "extract",
      table: "extracts",
      id: extract.id,
      requestId: extract.request_id,
    },
    () => logExtractInternal(extract),
  );
}

async function logExtractInternal(extract: LoggedExtract) {
  const logger = _logger.child({
    module: "log_job",
    method: "logExtract",
    extractId: extract.id,
    requestId: extract.request_id,
    teamId: extract.team_id,
  });

  await robustInsert(
    "extracts",
    {
      id: extract.id,
      request_id: extract.request_id,
      urls: extract.urls,
      team_id:
        extract.team_id === "preview" || extract.team_id?.startsWith("preview_")
          ? previewTeamId
          : extract.team_id,
      options: extract.options,
      model_kind: extract.model_kind,
      credits_cost: extract.credits_cost,
      is_successful: extract.is_successful,
      error: extract.error ?? null,
      cost_tracking: extract.cost_tracking ?? null,
    },
    logger,
  );

  try {
    await writeExtractJobState(extract.id, {
      status: extract.is_successful ? "completed" : "failed",
      completedAtMs: Date.now(),
      creditsBilled: extract.credits_cost,
      ...(extract.error ? { error: extract.error } : {}),
    });
  } catch (error) {
    logger.error("Failed to write extract state to Bigtable", { error });
  }

  if (extract.result) {
    if (config.GCS_BUCKET_NAME) {
      await saveExtractToGCS(extract, logger);
    } else {
      // Fallback: save result to Redis with 24h TTL when GCS is not configured
      await saveExtractResult(extract.id, extract.result);
    }
  }
}

export type LoggedMap = {
  id: string;
  request_id: string;
  url: string;
  team_id: string;
  options: any;
  results: any[];
  credits_cost: number;
  zeroDataRetention: boolean;
};

export async function logMap(map: LoggedMap) {
  return withLogSpan(
    {
      operation: "map",
      table: "maps",
      id: map.id,
      requestId: map.request_id,
      zeroDataRetention: map.zeroDataRetention,
    },
    () => logMapInternal(map),
  );
}

async function logMapInternal(map: LoggedMap) {
  const logger = _logger.child({
    module: "log_job",
    method: "logMap",
    mapId: map.id,
    requestId: map.request_id,
    teamId: map.team_id,
    zeroDataRetention: map.zeroDataRetention,
  });
  const storedTeamId =
    map.team_id === "preview" || map.team_id?.startsWith("preview_")
      ? previewTeamId
      : map.team_id;

  await writeFeedbackJobSafely(
    {
      jobId: map.id,
      requestId: map.request_id,
      teamId: storedTeamId,
      endpoint: "map",
      succeeded: true,
      creditsBilled: map.credits_cost,
      zeroDataRetention: map.zeroDataRetention,
    },
    logger,
  );

  await robustInsert(
    "maps",
    {
      id: map.id,
      request_id: map.request_id,
      url: map.zeroDataRetention
        ? "<redacted due to zero data retention>"
        : map.url,
      team_id: storedTeamId,
      options: map.zeroDataRetention ? null : map.options,
      num_results: map.results.length,
      credits_cost: map.credits_cost,
    },
    logger,
  );

  if (map.results && !map.zeroDataRetention) {
    await saveMapToGCS(map, logger);
  }
}

export type LoggedLlmsTxt = {
  id: string;
  request_id: string;
  url: string;
  team_id: string;
  options: any;
  num_urls: number;
  cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;
  credits_cost: number;
  result: { llmstxt: string; llmsfulltxt: string };
};

export async function logLlmsTxt(llmsTxt: LoggedLlmsTxt) {
  return withLogSpan(
    {
      operation: "llmstxt",
      table: "llmstxts",
      id: llmsTxt.id,
      requestId: llmsTxt.request_id,
    },
    () => logLlmsTxtInternal(llmsTxt),
  );
}

async function logLlmsTxtInternal(llmsTxt: LoggedLlmsTxt) {
  const logger = _logger.child({
    module: "log_job",
    method: "logLlmsTxt",
    llmsTxtId: llmsTxt.id,
    requestId: llmsTxt.request_id,
    teamId: llmsTxt.team_id,
  });

  await robustInsert(
    "llmstxts",
    {
      id: llmsTxt.id,
      request_id: llmsTxt.request_id,
      url: llmsTxt.url,
      team_id:
        llmsTxt.team_id === "preview" || llmsTxt.team_id?.startsWith("preview_")
          ? previewTeamId
          : llmsTxt.team_id,
      options: llmsTxt.options,
      num_urls: llmsTxt.num_urls,
      credits_cost: llmsTxt.credits_cost,
      cost_tracking: llmsTxt.cost_tracking ?? null,
    },
    logger,
  );

  if (llmsTxt.result) {
    await saveLlmsTxtToGCS(llmsTxt, logger);
  }
}

export type LoggedDeepResearch = {
  id: string;
  request_id: string;
  query: string;
  team_id: string;
  options: any;
  time_taken: number;
  credits_cost: number;
  result: { finalAnalysis: string; sources: any; json: any };
  cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;
};

export async function logDeepResearch(deepResearch: LoggedDeepResearch) {
  return withLogSpan(
    {
      operation: "deep_research",
      table: "deep_researches",
      id: deepResearch.id,
      requestId: deepResearch.request_id,
    },
    () => logDeepResearchInternal(deepResearch),
  );
}

async function logDeepResearchInternal(deepResearch: LoggedDeepResearch) {
  const logger = _logger.child({
    module: "log_job",
    method: "logDeepResearch",
    deepResearchId: deepResearch.id,
    requestId: deepResearch.request_id,
    teamId: deepResearch.team_id,
  });

  await robustInsert(
    "deep_researches",
    {
      id: deepResearch.id,
      request_id: deepResearch.request_id,
      query: deepResearch.query,
      team_id:
        deepResearch.team_id === "preview" ||
        deepResearch.team_id?.startsWith("preview_")
          ? previewTeamId
          : deepResearch.team_id,
      options: deepResearch.options,
      time_taken: deepResearch.time_taken,
      credits_cost: deepResearch.credits_cost,
      cost_tracking: deepResearch.cost_tracking ?? null,
    },
    logger,
  );

  if (deepResearch.result) {
    await saveDeepResearchToGCS(deepResearch, logger);
  }
}

type LoggedMcpActionLog = {
  id: string;
  team_id: string;
  request_id: string;
  [field: string]: unknown;
};

/**
 * One hosted MCP tool call from the internal action-log ingest. `id` is the
 * MCP request id, so retried deliveries collapse into one row.
 */
export async function logMcpActionLog(action: LoggedMcpActionLog) {
  return withLogSpan(
    {
      operation: "mcp_action",
      table: "mcp_action_logs",
      id: action.id,
      requestId: action.request_id,
    },
    () => logMcpActionLogInternal(action),
  );
}

async function logMcpActionLogInternal(action: LoggedMcpActionLog) {
  const logger = _logger.child({
    module: "log_job",
    method: "logMcpActionLog",
    requestId: action.request_id,
    teamId: action.team_id,
  });
  await robustInsert("mcp_action_logs", action, logger);
}