Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

18.7 KB
Raw Blame
import { Response } from "express";
import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import {
  BatchScrapeRequest,
  batchScrapeRequestSchema,
  batchScrapeRequestSchemaNoURLValidation,
  URL as urlSchema,
  RequestWithAuth,
  ScrapeOptions,
  BatchScrapeResponse,
} from "./types";
import {
  addCrawlJobs,
  finishCrawlKickoff,
  getCrawl,
  lockURLs,
  markCrawlActive,
  saveCrawl,
  StoredCrawl,
} from "../../lib/crawl-redis";
import { getJobPriority } from "../../lib/job-priority";
import { addScrapeJobs } from "../../services/queue-jobs";
import { createWebhookSender, WebhookEvent } from "../../services/webhook";
import { logger as _logger } from "../../lib/logger";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { checkPermissions } from "../../lib/permissions";
import { resolveSafeMode } from "../../lib/safe-mode";
import {
  actionTypesOf,
  checkKeyFormatRestriction,
  formatTypesOf,
} from "../../lib/key-restriction";
import {
  crawlGroup,
  resolveNewGroupBackend,
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
  checkUrlsAgainstThreatPolicy,
  resolveThreatProtection,
} from "../../lib/threat-protection/request";
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
import {
  ThirdPartyDataTermsRequiredError,
  ThirdPartyDataUnsupportedOptionError,
} from "../../lib/exchange";
import { getExchangeAccessForRequestBody } from "../../lib/exchange-request";
import {
  AGENT_REQUEST_CREDITS_SHARDS,
  initializeRequestCredits,
  requestCreditsShards,
} from "../../lib/request-credits-store";

export async function batchScrapeController(
  req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
  res: Response<BatchScrapeResponse>,
) {
  const preNormalizedBody = { ...req.body };
  if (req.body?.ignoreInvalidURLs === true) {
    req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);
  } else {
    req.body = batchScrapeRequestSchema.parse(req.body);
  }

  // Batch has many URLs; per-URL allowlist is applied at the scrapeURL backstop.
  // Resolve org-level here only for the bypass/enablement decision.
  const safeMode = resolveSafeMode(req.acuc?.flags, req.body.safeMode);
  if (safeMode.error) {
    return res.status(403).json({
      success: false,
      code: safeMode.code,
      error: safeMode.error,
    });
  }

  const threatProtection = await resolveThreatProtection({
    teamId: req.auth.team_id,
    orgId: req.acuc?.org_id ?? null,
    flags: req.acuc?.flags ?? null,
    override: req.body.threatProtection,
    force: safeMode.safeMode?.domainControls === true,
  });
  if (threatProtection.error) {
    return res.status(403).json({
      success: false,
      error: threatProtection.error,
    });
  }

  const permissions = checkPermissions(req.body, req.acuc?.flags, {
    threatProtectionOrgConfig: threatProtection.orgConfig,
    safeMode: safeMode.safeMode ?? null,
  });
  if (permissions.error) {
    return res.status(403).json({
      success: false,
      code: permissions.code,
      error: permissions.error,
    });
  }

  const keyRestriction = await checkKeyFormatRestriction(
    formatTypesOf(req.body.formats),
    actionTypesOf(req.body.actions),
    req.acuc?.api_key_id,
    req.acuc?.flags ?? null,
  );
  if (!keyRestriction.allowed) {
    return res.status(keyRestriction.status).json({
      success: false,
      error: keyRestriction.error,
    });
  }

  // Safe Mode lockdown is cache-only, which implies zero data retention.
  const zeroDataRetention =
    getScrapeZDR(req.acuc?.flags) === "forced" ||
    (req.body.zeroDataRetention ?? false) ||
    (safeMode.safeMode?.lockdown ?? false);

  if (
    req.body.__agentInterop &&
    config.AGENT_INTEROP_SECRET &&
    !isAgentInteropSecretValid(req.body.__agentInterop.auth)
  ) {
    return res.status(403).json({
      success: false,
      error: "Invalid agent interop.",
    });
  } else if (req.body.__agentInterop && !config.AGENT_INTEROP_SECRET) {
    return res.status(403).json({
      success: false,
      error: "Agent interop is not enabled.",
    });
  }

  // Only batch scrapes can be appended to: crawls (including monitor crawls)
  // carry their own billing and scope.
  const appendTarget = req.body.appendToId
    ? ((await getCrawl(req.body.appendToId)) as StoredCrawl | null)
    : null;
  if (
    req.body.appendToId &&
    (!appendTarget ||
      appendTarget.team_id !== req.auth.team_id ||
      appendTarget.crawlerOptions != null)
  ) {
    return res.status(404).json({
      success: false,
      error: "Job not found",
    });
  }

  const id = req.body.appendToId ?? uuidv7();
  const billing: BillingMetadata = {
    ...(req.body.__agentInterop
      ? { endpoint: "agent" as const, jobId: id }
      : { endpoint: "batch_scrape" as const, jobId: id }),
    externalRequestId: externalRequestId(req),
    hostedMcp: req.auth.hostedMcp,
  };
  const logger = _logger.child({
    crawlId: id,
    batchScrapeId: id,
    module: "api/v2",
    method: "batchScrapeController",
    teamId: req.auth.team_id,
    zeroDataRetention,
  });

  // A blocklisted URL is admitted when the Exchange can serve it, as on
  // single scrape; null means the URL may be scraped.
  const blocklistRefusal = async (url: string) => {
    if (
      !isUrlBlocked(url, req.acuc?.flags ?? null, {
        team_id: req.auth.team_id,
        org_id: req.acuc?.org_id ?? null,
        origin: req.body.origin ?? null,
      })
    ) {
      return null;
    }
    const exchangeAccess = await getExchangeAccessForRequestBody({
      body: req.body,
      flags: req.acuc?.flags ?? null,
      url,
      blocked: true,
      zeroDataRetention,
      teamId: req.auth.team_id,
      orgId: req.acuc?.org_id ?? null,
    });
    if (exchangeAccess.allowed) {
      return null;
    }
    if (exchangeAccess.termsRequired) {
      return new ThirdPartyDataTermsRequiredError(exchangeAccess.terms);
    }
    return exchangeAccess.unsupportedOption === undefined
      ? new UnsupportedSiteError()
      : new ThirdPartyDataUnsupportedOptionError(
          exchangeAccess.unsupportedOption,
        );
  };

  let urls: string[] = req.body.urls;
  let unnormalizedURLs = preNormalizedBody.urls;
  let invalidURLs: string[] | undefined = undefined;
  const refusedURLs: {
    url: string;
    error:
      | UnsupportedSiteError
      | ThirdPartyDataTermsRequiredError
      | ThirdPartyDataUnsupportedOptionError;
  }[] = [];

  if (req.body.ignoreInvalidURLs) {
    invalidURLs = [];

    let pendingURLs = urls;
    urls = [];
    unnormalizedURLs = [];
    for (const u of pendingURLs) {
      try {
        const nu = urlSchema.parse(u);
        const refusal = await blocklistRefusal(nu);
        if (refusal === null) {
          urls.push(nu);
          unnormalizedURLs.push(u);
        } else {
          invalidURLs.push(u);
          refusedURLs.push({ url: nu, error: refusal });
        }
      } catch (_) {
        invalidURLs.push(u);
      }
    }
  } else {
    for (const url of urls) {
      const refusal = await blocklistRefusal(url);
      if (refusal !== null) {
        refusedURLs.push({ url, error: refusal });
      }
    }
  }

  emitRejectedScrapeActivityEvents(
    refusedURLs.map(({ url, error }) => ({
      scrapeId: uuidv7(),
      requestId: req.body.__agentInterop?.requestId ?? id,
      endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",
      teamId: req.auth.team_id,
      apiKeyId: req.acuc?.api_key_id ?? null,
      auditMetadata: req.body.auditMetadata,
      url,
      error,
      origin: req.body.origin ?? "api",
      integration: req.body.integration,
      zeroDataRetention,
    })),
  );

  if (!req.body.ignoreInvalidURLs && refusedURLs.length > 0) {
    // Unaccepted terms and an unsupported option are the refusals the caller
    // can fix, so they win over an unsupported site.
    const errors = refusedURLs.map(x => x.error);
    const termsRequired = errors.find(
      error => error instanceof ThirdPartyDataTermsRequiredError,
    );
    if (termsRequired) {
      return res.status(403).json(termsRequired.response());
    }
    const unsupportedOption = errors.find(
      error => error instanceof ThirdPartyDataUnsupportedOptionError,
    );
    if (unsupportedOption) {
      return res.status(400).json(unsupportedOption.response());
    }
    return res.status(403).json({
      success: false,
      error: UNSUPPORTED_SITE_MESSAGE,
    });
  }

  // Threat protection: reject/report blocked URLs at enqueue time so they
  // never consume scrape slots. Mirrors the isUrlBlocked handling above:
  // with ignoreInvalidURLs they are reported in invalidURLs; without it the
  // whole request is rejected. Any URL that slips through (e.g. via a
  // redirect) is still blocked in the scrape pipeline and its job document
  // gets the unsafe_domain_blocked error.
  if (threatProtection.policy) {
    const { blocked, decisionsByUrl } = await checkUrlsAgainstThreatPolicy(
      urls,
      threatProtection.policy,
      { teamId: req.auth.team_id },
    );
    if (blocked.length > 0) {
      // Consulted decisions bill the scan fee (+2 per unique scanned URL) —
      // the scans already happened. With ignoreInvalidURLs the allowed URLs
      // proceed to scrape jobs that bill their own scans, so only blocked
      // ones bill here; when the whole request is rejected below, no scrape
      // jobs will ever run, so every scanned URL bills here.
      const threatScanCredits = calculateThreatScanCredits(
        req.body.ignoreInvalidURLs
          ? blocked.map(x => x.decision)
          : decisionsByUrl.values(),
      );
      if (
        threatScanCredits > 0 &&
        (req.body.__agentInterop?.shouldBill ?? true)
      ) {
        billTeam(
          req.auth.team_id,
          req.acuc?.org_id ?? null,
          threatScanCredits,
          req.acuc?.api_key_id ?? null,
          {
            ...billing,
            // Appends reuse the batch id but each append's threat scans are a
            // fresh charge — a shared key would underbill them. Appends stay
            // keyless (per-request UUID in firebill).
            ...(req.body.appendToId ? {} : { chargeId: `${id}:threat` }),
          },
        ).catch(error => {
          logger.error(
            `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
          );
        });
      }
      emitRejectedScrapeActivityEvents(
        blocked.map(blockedUrl => ({
          scrapeId: uuidv7(),
          requestId: req.body.__agentInterop?.requestId ?? id,
          endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",
          teamId: req.auth.team_id,
          apiKeyId: req.acuc?.api_key_id ?? null,
          auditMetadata: req.body.auditMetadata,
          url: blockedUrl.url,
          error: new UnsafeDomainBlockedError(
            blockedUrl.url,
            blockedUrl.decision,
          ),
          threatDecisions: [blockedUrl.decision],
          origin: req.body.origin ?? "api",
          integration: req.body.integration,
          zeroDataRetention,
        })),
      );
      if (req.body.ignoreInvalidURLs) {
        const blockedSet = new Set(blocked.map(x => x.url));
        const keptUnnormalized: string[] = [];
        const keptUrls: string[] = [];
        urls.forEach((u, i) => {
          if (blockedSet.has(u)) {
            invalidURLs!.push(unnormalizedURLs[i] ?? u);
          } else {
            keptUrls.push(u);
            keptUnnormalized.push(unnormalizedURLs[i]);
          }
        });
        urls = keptUrls;
        unnormalizedURLs = keptUnnormalized;
      } else {
        const first = blocked[0];
        const error = new UnsafeDomainBlockedError(first.url, first.decision);
        return res.status(403).json({
          success: false,
          code: error.code,
          error: error.message,
        });
      }
    }
  }

  if (urls.length === 0) {
    return res.status(400).json({
      success: false,
      error: "No valid URLs provided",
    });
  }

  logger.debug("Batch scrape " + id + " starting", {
    urlsLength: urls.length,
    appendToId: req.body.appendToId,
    account: req.account,
  });

  if (!req.body.appendToId && !req.body.__agentInterop) {
    await logRequest({
      id,
      kind: "batch_scrape",
      api_version: "v2",
      external_request_id: externalRequestId(req),
      team_id: req.auth.team_id,
      origin: req.body.origin ?? "api",
      integration: req.body.integration,
      target_hint: urls[0] ?? "",
      zeroDataRetention: zeroDataRetention || false,
      api_key_id: req.acuc?.api_key_id ?? null,
      jobAccessExpiresAt: new Date(
        Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
      ),
      creditsShards: requestCreditsShards(urls.length),
    });
  } else if (!req.body.appendToId) {
    // An agent-started batch is recorded under the agent's request rather
    // than as a request of its own, so logRequest never created a credit
    // row for it. Its status is still polled: v2 reads credits under the
    // saved agent request id, v1 under the batch id, and its children record
    // credits under the agent request id. Make sure both rows exist. The
    // agent row is created once with the agent controller's shard count;
    // for an agent that came through that controller this is a no-op, for
    // one that reaches the API first through interop it is the only writer.
    const agentRequestId = req.body.__agentInterop?.requestId;
    if (agentRequestId) {
      await initializeRequestCredits(
        agentRequestId,
        AGENT_REQUEST_CREDITS_SHARDS,
      ).catch(error => {
        logger.warn("Failed to initialize Bigtable request credits", {
          error,
          requestId: agentRequestId,
          shards: AGENT_REQUEST_CREDITS_SHARDS,
        });
      });
    }
    const creditsShards = requestCreditsShards(urls.length);
    await initializeRequestCredits(id, creditsShards).catch(error => {
      logger.warn("Failed to initialize Bigtable request credits", {
        error,
        shards: creditsShards,
      });
    });
  }

  const sc: StoredCrawl = appendTarget
    ? appendTarget
    : {
        crawlerOptions: null,
        scrapeOptions: req.body,
        internalOptions: {
          disableSmartWaitCache: true,
          teamId: req.auth.team_id,
          orgId: req.acuc?.org_id ?? null,
          saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
            ? true
            : false,
          zeroDataRetention,
          bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),
          agentIndexOnly: (req as any).agentIndexOnly ?? false,
          threatProtection: threatProtection.policy ?? undefined,
          // Safe Mode rides the batch payload so each URL resolves it per-URL
          // at the scrapeURL backstop.
          teamFlags: req.acuc?.flags ?? null,
          safeModeBypassed: safeMode.bypassed === true,
        }, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
        team_id: req.auth.team_id,
        createdAt: Date.now(),
        maxConcurrency: req.body.maxConcurrency,
        zeroDataRetention,
        v1: true,
        webhook: req.body.webhook,
        requestId: req.body.__agentInterop?.requestId ?? undefined,
        origin: req.body.origin,
      };

  if (req.body.appendToId) {
    sc.internalOptions.bypassBilling = !(
      req.body.__agentInterop?.shouldBill ?? true
    );
    // Refresh Safe Mode + threat-protection context so appended jobs enforce
    // the team's current policy, not whatever was stored when the batch was
    // first created.
    sc.internalOptions.teamFlags = req.acuc?.flags ?? undefined;
    sc.internalOptions.safeModeBypassed = safeMode.bypassed === true;
    sc.internalOptions.threatProtection = threatProtection.policy ?? undefined;
  }

  if (!req.body.appendToId) {
    sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
    await crawlGroup.addGroup(
      id,
      sc.team_id,
      (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
      {
        backend: sc.queueBackend,
        maxConcurrency: sc.maxConcurrency,
        delaySeconds: sc.crawlerOptions?.delay,
      },
    );
    await saveCrawl(id, sc);
    await markCrawlActive(id);
  }

  let jobPriority = 20;

  // If it is over 1000, we need to get the job priority,
  // otherwise we can use the default priority of 20
  if (urls.length > 1000) {
    // set base to 21
    jobPriority = await getJobPriority({
      team_id: req.auth.team_id,
      acuc: req.acuc,
      basePriority: 21,
    });
  }
  logger.debug("Using job priority " + jobPriority, { jobPriority });

  const scrapeOptions: ScrapeOptions = { ...req.body };
  delete (scrapeOptions as any).urls;
  delete (scrapeOptions as any).appendToId;

  const jobs = urls.map(x => ({
    jobId: uuidv7(),
    data: {
      url: x,
      mode: "single_urls" as const,
      team_id: req.auth.team_id,
      crawlerOptions: null,
      scrapeOptions,
      origin: "api",
      integration: req.body.integration,
      billing,
      crawl_id: id,
      requestId: req.body.__agentInterop?.requestId ?? undefined,
      bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),
      sitemapped: true,
      v1: true,
      webhook: req.body.webhook,
      internalOptions: sc.internalOptions,
      zeroDataRetention,
      apiKeyId: req.acuc?.api_key_id ?? null,
    },
    priority: jobPriority,
  }));

  await finishCrawlKickoff(id);

  logger.debug("Locking URLs...");
  await lockURLs(
    id,
    sc,
    jobs.map(x => x.data.url),
    logger,
  );
  logger.debug("Adding scrape jobs to Redis...");
  await addCrawlJobs(
    id,
    jobs.map(x => x.jobId),
    logger,
  );
  logger.debug("Adding scrape jobs to BullMQ...");
  await addScrapeJobs(jobs);

  if (req.body.webhook) {
    logger.debug("Calling webhook with batch_scrape.started...", {
      webhook: req.body.webhook,
    });
    const sender = await createWebhookSender({
      teamId: req.auth.team_id,
      jobId: id,
      webhook: req.body.webhook,
      v0: false,
    });
    await sender?.send(WebhookEvent.BATCH_SCRAPE_STARTED, { success: true });
  }

  const protocol = req.protocol;

  return res.status(200).json({
    success: true,
    id,
    url: `${protocol}://${req.host}/v2/batch/scrape/${id}`,
    invalidURLs,
  });
}