Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

11.2 KB
Raw Blame
import { Response } from "express";
import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import {
  CrawlRequest,
  crawlRequestSchema,
  CrawlResponse,
  RequestWithAuth,
  toLegacyCrawlerOptions,
} from "./types";
import {
  crawlToCrawler,
  saveCrawl,
  StoredCrawl,
  markCrawlActive,
} from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
import { resolveSafeMode } from "../../lib/safe-mode";
import {
  actionTypesOf,
  checkKeyFormatRestriction,
  formatTypesOf,
} from "../../lib/key-restriction";
import {
  crawlGroup,
  resolveNewGroupBackend,
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { checkUrl } from "../../lib/threat-protection";
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";
import { requestCreditsShards } from "../../lib/request-credits-store";
import { DEFAULT_TEAM_LIMITS } from "../../services/autumn/autumn.service";

export async function crawlController(
  req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
  res: Response<CrawlResponse>,
) {
  const preNormalizedBody = req.body;
  req.body = crawlRequestSchema.parse(req.body);
  const id = uuidv7();

  const safeMode = resolveSafeMode(
    req.acuc?.flags,
    req.body.scrapeOptions?.safeMode,
    req.body.url,
  );
  if (safeMode.error) {
    return res.status(403).json({
      success: false,
      code: safeMode.code,
      error: safeMode.error,
    } as any);
  }

  // Safe Mode lockdown is cache-only, which implies zero data retention.
  const zeroDataRetention =
    getScrapeZDR(req.acuc?.flags) === "forced" ||
    req.body.zeroDataRetention ||
    (safeMode.safeMode?.lockdown ?? false);

  const threatProtection = await resolveThreatProtection({
    teamId: req.auth.team_id,
    orgId: req.acuc?.org_id ?? null,
    flags: req.acuc?.flags ?? null,
    override: req.body.scrapeOptions?.threatProtection,
    force: safeMode.safeMode?.domainControls === true,
  });
  if (threatProtection.error) {
    return res.status(403).json({
      success: false,
      error: threatProtection.error,
    });
  }

  // Scrape params live under scrapeOptions; checkPermissions reads them
  // top-level, so spread scrapeOptions while keeping the top-level fields it
  // also reads (zeroDataRetention, crawlerOptions.ignoreRobotsTxt) and the
  // nested scrapeOptions (location / threatProtection).
  const permissions = checkPermissions(
    {
      ...req.body.scrapeOptions,
      zeroDataRetention: req.body.zeroDataRetention,
      crawlerOptions: req.body,
      scrapeOptions: req.body.scrapeOptions,
    },
    req.acuc?.flags,
    {
      threatProtectionOrgConfig: threatProtection.orgConfig,
      safeMode: safeMode.safeMode ?? null,
    },
  );
  if (permissions.error) {
    return res.status(403).json({
      success: false,
      code: permissions.code,
      error: permissions.error,
    });
  }

  // Threat protection: check the seed URL before kicking off the crawl.
  if (threatProtection.policy) {
    const decision = await checkUrl(req.body.url, threatProtection.policy, {
      teamId: req.auth.team_id,
    });
    if (!decision.allowed) {
      // A blocked seed still bills the scan fee when the classifier was
      // consulted — the scan already happened. (An allowed seed is not billed
      // here: its scrape job re-checks the cached verdict and bills there.)
      const threatScanCredits = calculateThreatScanCredits([decision]);
      if (threatScanCredits > 0) {
        billTeam(
          req.auth.team_id,
          req.acuc?.org_id ?? null,
          threatScanCredits,
          req.acuc?.api_key_id ?? null,
          // No chargeId: a fresh crawl id is minted per request and the
          // rejected crawl is never persisted or queued, so there is no
          // stable per-charge identity that could dedupe a retry.
          { endpoint: "crawl", externalRequestId: externalRequestId(req) },
        ).catch(error => {
          _logger.error(
            `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
          );
        });
      }
      const error = new UnsafeDomainBlockedError(req.body.url, decision);
      emitRejectedScrapeActivityEvent({
        scrapeId: uuidv7(),
        requestId: id,
        endpoint: "crawl",
        teamId: req.auth.team_id,
        apiKeyId: req.acuc?.api_key_id ?? null,
        auditMetadata: req.body.scrapeOptions?.auditMetadata,
        url: req.body.url,
        error,
        threatDecisions: [decision],
        origin: req.body.origin ?? "api",
        integration: req.body.integration,
        zeroDataRetention: zeroDataRetention ?? false,
      });
      return res.status(403).json({
        success: false,
        code: error.code,
        error: error.message,
      });
    }
  }

  const keyRestriction = await checkKeyFormatRestriction(
    formatTypesOf(req.body.scrapeOptions?.formats),
    actionTypesOf(req.body.scrapeOptions?.actions),
    req.acuc?.api_key_id,
    req.acuc?.flags ?? null,
  );
  if (!keyRestriction.allowed) {
    return res.status(keyRestriction.status).json({
      success: false,
      error: keyRestriction.error,
    });
  }

  const logger = _logger.child({
    crawlId: id,
    module: "api/v1",
    method: "crawlController",
    teamId: req.auth.team_id,
    zeroDataRetention,
  });

  logger.debug("Crawl " + id + " starting", {
    request: req.body,
    originalRequest: preNormalizedBody,
    account: req.account,
  });

  await logRequest({
    id,
    kind: "crawl",
    api_version: "v1",
    external_request_id: externalRequestId(req),
    team_id: req.auth.team_id,
    origin: req.body.origin ?? "api",
    integration: req.body.integration,
    target_hint: req.body.url,
    zeroDataRetention: zeroDataRetention || false,
    api_key_id: req.acuc?.api_key_id ?? null,
    jobAccessExpiresAt: new Date(
      Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
    ),
    creditsShards: requestCreditsShards(req.body.limit ?? 10_000),
  });

  // checkCreditsMiddleware (always runs before this controller) is the source
  // of truth: Infinity when Autumn allows the request, the real remaining when
  // it clamps a low-credit crawl. Default to no clamp if it's somehow unset.
  let remainingCredits = req.account?.remainingCredits ?? Infinity;
  const useDbAuthentication = config.USE_DB_AUTHENTICATION;
  if (!useDbAuthentication) {
    remainingCredits = Infinity;
  }

  const crawlerOptions = {
    ...req.body,
    url: undefined,
    scrapeOptions: undefined,
  };

  const bodyScrapeOptions =
    req.body.scrapeOptions ?? ({} as typeof req.body.scrapeOptions);
  const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(
    bodyScrapeOptions,
    bodyScrapeOptions.timeout,
    req.auth.team_id,
  );
  // v1 prefaults maxAge (1 day), which would shadow the lockdown default the
  // scrape backstop applies — keep only a maxAge the request actually sent.
  if (
    safeMode.safeMode?.lockdown &&
    preNormalizedBody.scrapeOptions?.maxAge === undefined
  ) {
    scrapeOptions.maxAge = undefined;
  }

  // TODO: @rafa, is this right? copied from v0
  if (Array.isArray(crawlerOptions.includePaths)) {
    for (const x of crawlerOptions.includePaths) {
      try {
        new RegExp(x);
      } catch (e) {
        return res.status(400).json({ success: false, error: e.message });
      }
    }
  }

  if (Array.isArray(crawlerOptions.excludePaths)) {
    for (const x of crawlerOptions.excludePaths) {
      try {
        new RegExp(x);
      } catch (e) {
        return res.status(400).json({ success: false, error: e.message });
      }
    }
  }

  const originalLimit = crawlerOptions.limit;
  crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit);
  logger.debug("Determined limit: " + crawlerOptions.limit, {
    remainingCredits,
    bodyLimit: originalLimit,
    originalBodyLimit: preNormalizedBody.limit,
  });

  const sc: StoredCrawl = {
    originUrl: req.body.url,
    crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),
    scrapeOptions,
    internalOptions: {
      ...internalOptions,
      disableSmartWaitCache: true,
      teamId: req.auth.team_id,
      orgId: req.acuc?.org_id ?? null,
      saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
      zeroDataRetention,
      agentIndexOnly: (req as any).agentIndexOnly ?? false,
      threatProtection: threatProtection.policy ?? undefined,
      // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
      teamFlags: req.acuc?.flags ?? null,
      safeModeBypassed: safeMode.bypassed === true,
    }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
    team_id: req.auth.team_id,
    createdAt: Date.now(),
    maxConcurrency:
      req.body.maxConcurrency !== undefined
        ? Math.min(
            req.body.maxConcurrency,
            req.acuc?.concurrency_limit ??
              DEFAULT_TEAM_LIMITS.concurrency_limit,
          )
        : undefined,
    zeroDataRetention,
    v1: true,
    webhook: req.body.webhook,
    origin: req.body.origin,
  };

  const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);

  try {
    sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);
    // const robotsCrawlDelay = crawler.getRobotsCrawlDelay();
    // if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {
    //   sc.crawlerOptions.delay = robotsCrawlDelay;
    // }
  } catch (e) {
    logger.debug("Failed to get robots.txt (this is probably fine!)", {
      error: e,
    });
  }

  sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
  await crawlGroup.addGroup(
    id,
    sc.team_id,
    (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
    {
      backend: sc.queueBackend,
      maxConcurrency: sc.maxConcurrency,
      delaySeconds: sc.crawlerOptions?.delay,
    },
  );

  await saveCrawl(id, sc);

  await markCrawlActive(id);

  await _addScrapeJobToBullMQ(
    {
      mode: "kickoff" as const,
      url: req.body.url,
      team_id: req.auth.team_id,
      crawlerOptions,
      scrapeOptions: sc.scrapeOptions,
      internalOptions: sc.internalOptions,
      origin: req.body.origin,
      integration: req.body.integration,
      billing: {
        endpoint: "crawl",
        jobId: id,
        externalRequestId: externalRequestId(req),
      },
      crawl_id: id,
      webhook: req.body.webhook,
      v1: true,
      zeroDataRetention: zeroDataRetention || false,
      apiKeyId: req.acuc?.api_key_id ?? null,
    },
    uuidv7(),
  );

  const protocol = req.protocol;

  return res.status(200).json({
    success: true,
    id,
    url: `${protocol}://${req.host}/v1/crawl/${id}`,
  });
}