Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

15.7 KB
Raw Blame
import { Response } from "express";
import { config } from "../../config";
import { timingSafeSecretEqual } from "../../lib/secret-compare";
import { logger as _logger } from "../../lib/logger";
import {
  Document,
  RequestWithAuth,
  ScrapeRequest,
  scrapeRequestSchema,
  ScrapeResponse,
} from "./types";
import { v7 as uuidv7 } from "uuid";
import { getJobPriority } from "../../lib/job-priority";
import { fromV1ScrapeOptions } from "../v2/types";
import { TransportableError } from "../../lib/error";
import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { NuQJob } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
import {
  resolveSafeMode,
  applySafeMode,
  isLockdownZeroDataRetention,
} from "../../lib/safe-mode";
import {
  actionTypesOf,
  checkKeyFormatRestriction,
  formatTypesOf,
} from "../../lib/key-restriction";
import { includesFormat } from "../../lib/format-utils";
import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";
import { processJobInternal } from "../../services/worker/scrape-worker";
import { ScrapeJobData } from "../../types";
import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManager";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { getErrorContactMessage } from "../../lib/deployment";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
  withSpan,
  setSpanAttributes,
  recordSpanException,
  SpanKind,
  type Span,
} from "../../lib/otel-tracer";
import {
  adjustKeylessCredits,
  keylessLimitBody,
  logKeylessCreditUsage,
  reserveKeylessCredits,
} from "../../lib/keyless";
import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { DEFAULT_TEAM_LIMITS } from "../../services/autumn/autumn.service";

export async function scrapeController(
  req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
  res: Response<ScrapeResponse>,
) {
  // Resolved before the root span starts so the whole request trace stays
  // unrecorded for zero-data-retention requests (see otel-tracer). Safe Mode
  // lockdown implies ZDR, so fold it in here too — otherwise child spans could
  // export target URLs before the inner handler applies lockdown.
  const zeroDataRetentionTrace =
    getScrapeZDR(req.acuc?.flags) === "forced" ||
    req.body?.zeroDataRetention === true ||
    isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);

  return withSpan(
    "api.scrape.request",
    span => scrapeControllerInner(req, res, span),
    {
      kind: SpanKind.SERVER,
      attributes: {
        "api.version": "v1",
        "scrape.team_id": req.auth.team_id,
      },
      zeroDataRetention: zeroDataRetentionTrace,
    },
  );
}

async function scrapeControllerInner(
  req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
  res: Response<ScrapeResponse>,
  span: Span,
) {
  // Get timing data from middleware (includes all middleware processing time)
  const middlewareStartTime =
    (req as any).requestTiming?.startTime || new Date().getTime();
  const controllerStartTime = new Date().getTime();

  const jobId: string = uuidv7();
  const preNormalizedBody = { ...req.body };
  req.body = scrapeRequestSchema.parse(req.body);

  // Honor a per-request Safe Mode opt-out/affirmation, consistent with v2.
  const safeMode = resolveSafeMode(
    req.acuc?.flags,
    req.body.safeMode,
    req.body.url,
  );
  if (safeMode.error) {
    return res.status(403).json({
      success: false,
      code: safeMode.code,
      error: safeMode.error,
    } as any);
  }

  const threatProtection = await resolveThreatProtection({
    teamId: req.auth.team_id,
    orgId: req.acuc?.org_id ?? null,
    flags: req.acuc?.flags ?? null,
    override: req.body.threatProtection,
    force: safeMode.safeMode?.domainControls === true,
  });
  if (threatProtection.error) {
    return res.status(403).json({
      success: false,
      error: threatProtection.error,
    });
  }

  const permissions = checkPermissions(req.body, req.acuc?.flags, {
    threatProtectionOrgConfig: threatProtection.orgConfig,
    safeMode: safeMode.safeMode ?? null,
  });
  if (permissions.error) {
    return res.status(403).json({
      success: false,
      code: permissions.code,
      error: permissions.error,
    });
  }

  const keyRestriction = await checkKeyFormatRestriction(
    formatTypesOf(req.body.formats),
    actionTypesOf(req.body.actions),
    req.acuc?.api_key_id,
    req.acuc?.flags ?? null,
  );
  if (!keyRestriction.allowed) {
    return res.status(keyRestriction.status).json({
      success: false,
      error: keyRestriction.error,
    });
  }

  const zeroDataRetention =
    getScrapeZDR(req.acuc?.flags) === "forced" ||
    req.body.zeroDataRetention ||
    (safeMode.safeMode?.lockdown ?? false);

  const logger = _logger.child({
    method: "scrapeController",
    jobId,
    noq: true,
    scrapeId: jobId,
    teamId: req.auth.team_id,
    team_id: req.auth.team_id,
    zeroDataRetention,
  });

  const middlewareTime = controllerStartTime - middlewareStartTime;

  logger.debug("Scrape " + jobId + " starting", {
    version: "v1",
    scrapeId: jobId,
    request: req.body,
    originalRequest: preNormalizedBody,
    account: req.account,
  });

  const logRequestPromise = logRequest({
    id: jobId,
    kind: "scrape",
    api_version: "v1",
    external_request_id: externalRequestId(req),
    team_id: req.auth.team_id,
    origin: req.body.origin,
    integration: req.body.integration,
    target_hint: req.body.url,
    zeroDataRetention: zeroDataRetention || false,
    api_key_id: req.acuc?.api_key_id ?? null,
  }).catch(err =>
    logger.warn("Background request log failed", { error: err, jobId }),
  );

  const origin = req.body.origin;
  const timeout = req.body.timeout;

  // const startTime = new Date().getTime();

  const isDirectToBullMQ = timingSafeSecretEqual(
    req.body.__searchPreviewToken,
    config.SEARCH_PREVIEW_TOKEN,
  );

  const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(
    req.body,
    req.body.timeout,
    req.auth.team_id,
  );
  // v1 prefaults maxAge (1 day), which would shadow the lockdown default —
  // keep only a maxAge the request actually sent under forced lockdown.
  if (safeMode.safeMode?.lockdown && preNormalizedBody.maxAge === undefined) {
    scrapeOptions.maxAge = undefined;
  }
  applySafeMode(safeMode.safeMode, scrapeOptions);
  const projectedKeylessCredits = !isDirectToBullMQ
    ? projectScrapeCredits(
        scrapeOptions,
        req.acuc?.flags ?? null,
        zeroDataRetention ?? false,
      )
    : 0;
  let reservedKeylessCredits = 0;
  let reconciledKeylessCredits = false;

  if (projectedKeylessCredits > 0) {
    const reservation = await reserveKeylessCredits(
      req.auth.team_id,
      projectedKeylessCredits,
    );
    if (!reservation.ok) {
      applyAgentAuthDiscoveryHeader(res);
      return res
        .status(429)
        .json(await keylessLimitBody(req.auth.team_id, "v1_scrape", req));
    }
    reservedKeylessCredits = projectedKeylessCredits;
  }

  const totalWait =
    (req.body.waitFor ?? 0) +
    (req.body.actions ?? []).reduce(
      (a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,
      0,
    );

  let lockTime: number | null = null;
  let concurrencyLimited: boolean = false;

  let timeoutHandle: NodeJS.Timeout | null = null;
  let doc: Document | null = null;

  try {
    const lockStart = Date.now();
    const aborter = new AbortController();
    if (timeout) {
      timeoutHandle = setTimeout(() => {
        aborter.abort();
      }, timeout * 0.667);
    }
    req.on("close", () => aborter.abort());

    doc = await teamConcurrencySemaphore.withSemaphore(
      req.auth.team_id,
      jobId,
      req.acuc?.concurrency_limit ?? DEFAULT_TEAM_LIMITS.concurrency_limit,
      aborter.signal,
      timeout ?? 60_000,
      async limited => {
        const jobPriority = await getJobPriority({
          team_id: req.auth.team_id,
          acuc: req.acuc,
          basePriority: 10,
        });

        lockTime = Date.now() - lockStart;
        concurrencyLimited = limited;

        logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {
          teamId: req.auth.team_id,
          lockTime,
          limited,
        });

        const job: NuQJob<ScrapeJobData> = {
          id: jobId,
          status: "active",
          createdAt: new Date(),
          priority: jobPriority,
          data: {
            url: req.body.url,
            mode: "single_urls",
            team_id: req.auth.team_id,
            scrapeOptions,
            internalOptions: {
              ...internalOptions,
              teamId: req.auth.team_id,
              saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
                ? true
                : false,
              unnormalizedSourceURL: preNormalizedBody.url,
              bypassBilling: isDirectToBullMQ,
              zeroDataRetention,
              teamFlags: req.acuc?.flags ?? null,
              orgId: req.acuc?.org_id ?? null,
              agentIndexOnly: (req as any).agentIndexOnly ?? false,
              threatProtection: threatProtection.policy ?? undefined,
              safeMode: safeMode.allowlisted ? undefined : safeMode.safeMode,
              safeModeBypassed: safeMode.bypassed === true,
            },
            skipNuq: true,
            origin,
            integration: req.body.integration,
            billing: {
              endpoint: "scrape",
              jobId,
              externalRequestId: externalRequestId(req),
            },
            startTime: controllerStartTime,
            zeroDataRetention: zeroDataRetention ?? false,
            apiKeyId: req.acuc?.api_key_id ?? null,
            concurrencyLimited: limited,
            keylessReserved: reservedKeylessCredits > 0,
            logRequestPromise: logRequestPromise,
          },
        };

        const doc = await processJobInternal(job);
        return doc ?? null;
      },
    );
  } catch (e) {
    if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
      reconciledKeylessCredits = true;
      adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(
        () => {},
      );
    }

    const timeoutErr =
      e instanceof TransportableError &&
      (e.code === "SCRAPE_TIMEOUT" || e.code === "CONCURRENCY_QUEUE_TIMEOUT");

    if (e instanceof TransportableError) {
      if (!timeoutErr) {
        logger.error(`Error in scrapeController`, {
          version: "v1",
          error: e,
        });
      }
      // DNS resolution errors should return 200 with success: false
      if (e.code === "SCRAPE_DNS_RESOLUTION_ERROR") {
        return res.status(200).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (e.code === "AGENT_INDEX_ONLY") {
        return res.status(403).json({
          success: false,
          code: e.code,
          error: e.message,
          sponsor_status: "pending",
          login_url: "https://firecrawl.dev/signin",
        });
      }

      if (e.code === "SCRAPE_ACTIONS_NOT_SUPPORTED") {
        return res.status(400).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (
        e.code === "unsafe_domain_blocked" ||
        e.code === "US_ONLY_UNAVAILABLE" ||
        e.code === "RESERVED_CAPACITY_UNAVAILABLE"
      ) {
        return res.status(403).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (e.code === "UNSUPPORTED_SITE") {
        return res.status(403).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (e.code === "SCRAPE_MEDIA_ACCESS_DENIED") {
        return res.status(403).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (e.code === "SCRAPE_PROMPT_INJECTION_DETECTED") {
        return res.status(403).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (e.code === "SCRAPE_JSON_CONTENT_TOO_LARGE") {
        return res.status(400).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (e instanceof ThirdPartyDataTermsRequiredError) {
        return res.status(403).json(e.response());
      }

      if (
        e.code === "THIRD_PARTY_DATA_NOT_ENABLED" ||
        e.code === "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"
      ) {
        return res.status(403).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (e.code === "THIRD_PARTY_DATA_NOT_FOUND") {
        return res.status(404).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      if (
        e.code === "THIRD_PARTY_DATA_UNSUPPORTED_URL" ||
        e.code === "THIRD_PARTY_DATA_UNSUPPORTED_OPTION"
      ) {
        return res.status(400).json({
          success: false,
          code: e.code,
          error: e.message,
        });
      }

      return res.status(timeoutErr ? 408 : 500).json({
        success: false,
        code: e.code,
        error: e.message,
      });
    } else {
      const id = uuidv7();
      logger.error(`Error in scrapeController`, {
        version: "v1",
        error: e,
        errorId: id,
        path: req.path,
        teamId: req.auth.team_id,
      });
      recordSpanException(span, e);
      setSpanAttributes(span, {
        "scrape.status_code": 500,
        "scrape.error_id": id,
      });
      return res.status(500).json({
        success: false,
        code: "UNKNOWN_ERROR",
        error: getErrorContactMessage(id),
      });
    }
  } finally {
    if (timeoutHandle) {
      clearTimeout(timeoutHandle);
    }
  }

  logger.info("Done with waitForJob");

  logger.info("Removed job from queue");

  if (!includesFormat(req.body.formats, "rawHtml")) {
    if (doc && doc.rawHtml) {
      delete doc.rawHtml;
    }
  }

  if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
    reconciledKeylessCredits = true;
    const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;
    adjustKeylessCredits(
      req.auth.team_id,
      actualKeylessCredits - reservedKeylessCredits,
    ).catch(() => {});
    logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(
      () => {},
    );
  }

  const totalRequestTime = new Date().getTime() - middlewareStartTime;
  const controllerTime = new Date().getTime() - controllerStartTime;

  let usedLlm =
    includesFormat(req.body.formats, "json") ||
    includesFormat(req.body.formats, "summary") ||
    includesFormat(req.body.formats, "branding") ||
    includesFormat(req.body.formats, "extract");

  if (
    !usedLlm &&
    includesFormat(req.body.formats, "changeTracking") &&
    req.body.changeTrackingOptions?.modes?.includes("json")
  ) {
    usedLlm = true;
  }

  logger.info("Request metrics", {
    version: "v1",
    mode: "scrape",
    scrapeId: jobId,
    middlewareStartTime,
    controllerStartTime,
    middlewareTime,
    controllerTime,
    totalRequestTime,
    totalWait,
    usedLlm,
    formats: req.body.formats,
    concurrencyLimited,
    concurrencyQueueDurationMs: lockTime || undefined,
  });

  return res.status(200).json({
    success: true,
    data: {
      ...doc!,
      metadata: {
        ...doc!.metadata,
        concurrencyLimited,
        concurrencyQueueDurationMs: concurrencyLimited
          ? lockTime || 0
          : undefined,
      },
    },
    scrape_id: origin?.includes("website") ? jobId : undefined,
  });
}