Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

11.3 KB
Raw Blame
import { Response } from "express";
import {
  mapRequestSchema,
  RequestWithAuth,
  MapRequest,
  MapResponse,
} from "./types";
import { configDotenv } from "dotenv";
import { billTeam } from "../../services/billing/credit_billing";
import { logMap, logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { logger as _logger } from "../../lib/logger";
import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
import {
  resolveSafeMode,
  isLockdownZeroDataRetention,
} from "../../lib/safe-mode";
import { getMapResults, MapResult } from "../../lib/map-utils";
import { v7 as uuidv7 } from "uuid";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { AvgrabResolveError, resolveViaAvgrab } from "../../lib/avgrab-resolve";
import {
  checkUrlsAgainstThreatPolicy,
  resolveThreatProtection,
} from "../../lib/threat-protection/request";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";

configDotenv();

export async function mapController(
  req: RequestWithAuth<{}, MapResponse, MapRequest>,
  res: Response<MapResponse>,
) {
  // Safe Mode lockdown is cache-only, which implies zero data retention.
  const zeroDataRetention =
    getScrapeZDR(req.acuc?.flags) === "forced" ||
    isLockdownZeroDataRetention(req.acuc?.flags, undefined);
  const logger = _logger.child({
    jobId: uuidv7(),
    teamId: req.auth.team_id,
    module: "api/v2",
    method: "mapController",
    zeroDataRetention,
  });
  // Get timing data from middleware (includes all middleware processing time)
  const middlewareStartTime =
    (req as any).requestTiming?.startTime || new Date().getTime();
  const controllerStartTime = new Date().getTime();

  const originalRequest = req.body;
  req.body = mapRequestSchema.parse(req.body);

  // Map is link discovery, not content scraping, but it is outbound to the
  // target. Under Safe Mode: force domainControls so discovered links are
  // filtered, and under lockdown serve from the index only (no sitemap/robots
  // fetch to the target).
  const safeMode = resolveSafeMode(req.acuc?.flags, undefined, req.body.url);
  const lockdownIndexOnly = safeMode.safeMode?.lockdown === true;
  if (lockdownIndexOnly) {
    req.body.useIndex = true;
  }
  // Map's ignoreRobotsTxt is top-level, so checkPermissions (which reads it
  // under crawlerOptions) can't see it — enforce robots here for Safe Mode.
  if (
    safeMode.safeMode?.enforceRobots &&
    !lockdownIndexOnly &&
    req.body.ignoreRobotsTxt
  ) {
    return res.status(403).json({
      success: false,
      code: "SAFE_MODE_BLOCKED",
      error:
        "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",
    });
  }

  const threatProtection = await resolveThreatProtection({
    teamId: req.auth.team_id,
    orgId: req.acuc?.org_id ?? null,
    flags: req.acuc?.flags ?? null,
    override: req.body.threatProtection,
    force: safeMode.safeMode?.domainControls === true,
  });
  if (threatProtection.error) {
    return res.status(403).json({
      success: false,
      error: threatProtection.error,
    });
  }

  const permissions = checkPermissions(req.body, req.acuc?.flags, {
    threatProtectionOrgConfig: threatProtection.orgConfig,
    safeMode: safeMode.safeMode ?? null,
  });
  if (permissions.error) {
    return res.status(403).json({
      success: false,
      code: permissions.code,
      error: permissions.error,
    });
  }

  const middlewareTime = controllerStartTime - middlewareStartTime;

  const mapId = uuidv7();

  logger.info("Map request", {
    request: req.body,
    originalRequest,
    teamId: req.auth.team_id,
    mapId,
  });

  await logRequest({
    id: mapId,
    kind: "map",
    api_version: "v2",
    external_request_id: externalRequestId(req),
    team_id: req.auth.team_id,
    origin: req.body.origin ?? "api",
    integration: req.body.integration,
    target_hint: req.body.url,
    zeroDataRetention,
    api_key_id: req.acuc?.api_key_id ?? null,
  });

  // Short-circuit: if the URL matches avgrab's resolve pattern, delegate
  // entirely. Skipped under Safe Mode so results still flow through the
  // standard path (domain-controls filtering + lockdown index-only).
  try {
    const avgrabResults = safeMode.safeMode
      ? null
      : await resolveViaAvgrab(req.body.url, req.body.limit, logger);

    if (avgrabResults !== null) {
      const creditsCost = avgrabResults.length;

      billTeam(
        req.auth.team_id,
        req.acuc?.org_id ?? null,
        creditsCost,
        req.acuc?.api_key_id ?? null,
        {
          endpoint: "map",
          jobId: mapId,
          // Suffixed so this early-return path can never collide with the main
          // map charge below, even if both ever billed the same mapId.
          chargeId: `${mapId}:avgrab`,
          externalRequestId: externalRequestId(req),
          hostedMcp: req.auth.hostedMcp,
        },
      ).catch(error => {
        logger.error(
          `Failed to bill team ${req.auth.team_id} for ${creditsCost} credits: ${error}`,
        );
      });

      logMap({
        id: mapId,
        request_id: mapId,
        url: req.body.url,
        team_id: req.auth.team_id,
        options: {
          search: req.body.search,
          sitemap: req.body.sitemap,
          includeSubdomains: req.body.includeSubdomains,
          ignoreQueryParameters: req.body.ignoreQueryParameters,
          limit: req.body.limit,
          timeout: req.body.timeout,
          location: req.body.location,
        },
        results: avgrabResults,
        credits_cost: creditsCost,
        zeroDataRetention,
      }).catch(error => {
        logger.error(
          `Failed to log job for team ${req.auth.team_id}: ${error}`,
        );
      });

      return res.status(200).json({
        success: true,
        id: mapId,
        links: avgrabResults,
      });
    }
  } catch (error) {
    if (error instanceof AvgrabResolveError) {
      const status = [403, 404, 503].includes(error.upstreamStatus)
        ? error.upstreamStatus
        : 502;
      return res.status(status).json({
        success: false,
        id: mapId,
        code: error.code,
        error: error.message,
      });
    }
    logger.warn("avgrab resolve failed, falling back to standard map", {
      error,
    });
  }

  let result: MapResult;
  let timeoutHandle: NodeJS.Timeout | null = null;

  const abort = new AbortController();
  try {
    result = (await Promise.race([
      getMapResults({
        url: req.body.url,
        search: req.body.search,
        limit: req.body.limit,
        includeSubdomains: req.body.includeSubdomains,
        crawlerOptions: {
          ...req.body,
          sitemap: req.body.sitemap,
        },
        origin: req.body.origin,
        teamId: req.auth.team_id,
        orgId: req.acuc?.org_id ?? null,
        allowExternalLinks: req.body.allowExternalLinks,
        abort: abort.signal,
        mock: req.body.useMock,
        filterByPath: req.body.filterByPath !== false,
        flags: req.acuc?.flags ?? null,
        useIndex: req.body.useIndex,
        ignoreCache: req.body.ignoreCache,
        location: req.body.location,
        headers: req.body.headers,
        id: mapId,
      }),
      ...(req.body.timeout !== undefined
        ? [
            new Promise(
              (_resolve, reject) =>
                (timeoutHandle = setTimeout(() => {
                  abort.abort(new MapTimeoutError());
                  reject(new MapTimeoutError());
                }, req.body.timeout)),
            ),
          ]
        : []),
    ])) as any;
  } catch (error) {
    if (error instanceof MapTimeoutError) {
      return res.status(408).json({
        success: false,
        code: error.code,
        error: error.message,
      });
    } else {
      throw error;
    }
  } finally {
    if (timeoutHandle) {
      clearTimeout(timeoutHandle);
    }
  }

  // Threat protection: remove blocked links from the returned URL list
  // entirely. Checks are URL-level; scan fees bill +2 per unique scanned
  // URL (see calculateThreatScanCredits).
  //
  // "zscaler" mode evaluates map results against local rules only (org
  // lists + synced custom categories): one map can return thousands of
  // URLs, and inline classification would burn the tenant's 400/hour
  // urlLookup budget on links that may never be fetched. Every URL still
  // gets the full provider check when a scrape of it starts.
  let threatScanCredits = 0;
  if (threatProtection.policy && result.mapResults.length > 0) {
    const { decisionsByUrl } = await checkUrlsAgainstThreatPolicy(
      result.mapResults.map(x => x.url),
      threatProtection.policy,
      {
        teamId: req.auth.team_id,
        localRulesOnly: threatProtection.policy.mode === "zscaler",
      },
    );
    threatScanCredits = calculateThreatScanCredits(decisionsByUrl.values());
    result.mapResults = result.mapResults.filter(x => {
      const decision = decisionsByUrl.get(x.url);
      return decision === undefined || decision.allowed;
    });
  }

  // Bill the team
  const creditsToBill = 1 + threatScanCredits;
  billTeam(
    req.auth.team_id,
    req.acuc?.org_id ?? null,
    creditsToBill,
    req.acuc?.api_key_id ?? null,
    {
      endpoint: "map",
      externalRequestId: externalRequestId(req),
      jobId: mapId,
      chargeId: mapId,
      hostedMcp: req.auth.hostedMcp,
    },
  ).catch(error => {
    logger.error("Failed to bill team for map credits", {
      teamId: req.auth.team_id,
      creditsToBill,
      error,
    });
  });

  logMap({
    id: result.job_id,
    request_id: result.job_id,
    url: req.body.url,
    team_id: req.auth.team_id,
    options: {
      search: req.body.search,
      sitemap: req.body.sitemap,
      includeSubdomains: req.body.includeSubdomains,
      ignoreQueryParameters: req.body.ignoreQueryParameters,
      limit: req.body.limit,
      timeout: req.body.timeout,
      location: req.body.location,
    },
    results: result.mapResults,
    credits_cost: creditsToBill,
    zeroDataRetention,
  }).catch(error => {
    logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`);
  });

  // Log final timing information
  const totalRequestTime = new Date().getTime() - middlewareStartTime;
  const controllerTime = new Date().getTime() - controllerStartTime;

  logger.info("Request metrics", {
    version: "v2",
    jobId: result.job_id,
    mode: "map",
    middlewareStartTime,
    controllerStartTime,
    middlewareTime,
    controllerTime,
    totalRequestTime,
    linksCount: result.mapResults.length,
  });

  // Check if we should warn about base domain
  let warning: string | undefined;
  // Only show warning if results <= 1 AND user didn't explicitly request limit=1 AND URL is not base domain
  if (
    result.mapResults.length <= 1 &&
    req.body.limit !== 1 &&
    !isBaseDomain(req.body.url)
  ) {
    const baseDomain = extractBaseDomain(req.body.url);
    if (baseDomain) {
      warning = `Only ${result.mapResults.length} result(s) found. For broader coverage, try mapping the base domain: ${baseDomain}`;
    }
  }

  const response = {
    success: true as const,
    id: result.job_id,
    links: result.mapResults,
    ...(warning && { warning }),
  };

  return res.status(200).json(response);
}