Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

7.0 KB
Raw Blame
import { TeamFlags } from "../controllers/v2/types";
import type { ErrorCodes } from "./error";
import {
  type ResolvedSafeMode,
  SAFE_MODE_CREDENTIAL_HEADERS,
  SAFE_MODE_LOGIN_ACTIONS,
} from "./safe-mode";
import {
  getScrapeZDR,
  getIgnoreRobots,
  getCustomRobotsAgent,
  getThreatProtection,
} from "./zdr-helpers";
import {
  THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,
  THREAT_PROTECTION_NOT_ENABLED_MESSAGE,
  THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE,
} from "./threat-protection/request";
import { usOnlyLocationRefusal } from "./us-only";

type LocationOptions = { country?: string };
type ThreatProtectionOption = { mode?: string };

interface APIRequest {
  zeroDataRetention?: boolean;
  location?: LocationOptions;
  geolocation?: LocationOptions;
  scrapeOptions?: {
    location?: LocationOptions;
    geolocation?: LocationOptions;
    threatProtection?: ThreatProtectionOption;
  };
  crawlerOptions?: {
    ignoreRobotsTxt?: boolean;
    robotsUserAgent?: string;
  };
  // Per-request threat protection policy override (field-level override of
  // the org config). Presence of any value gates on the team flag.
  threatProtection?: ThreatProtectionOption;
  proxy?: string;
  profile?: unknown;
  actions?: { type: string }[];
  headers?: Record<string, string>;
}

interface PermissionOptions {
  /**
   * Org-level threat protection config (or the relevant slice of it), if
   * already loaded. When the org disables request overrides, any per-request
   * threatProtection option is rejected.
   */
  threatProtectionOrgConfig?: { allowRequestOverrides: boolean } | null;
  safeMode?: ResolvedSafeMode | null;
}

const SUPPORT_EMAIL = "[email protected]";

/** The refusal for static-IP egress ("us-whitelist") without the entitlement. */
export function ipWhitelistRefusal(
  flags: TeamFlags | null | undefined,
  ...countries: unknown[]
): string | undefined {
  const needsWhitelist = countries.some(
    value =>
      typeof value === "string" && value.toLowerCase() === "us-whitelist",
  );
  return needsWhitelist && !flags?.ipWhitelist
    ? `Static IP addresses are not enabled for your team. Contact ${SUPPORT_EMAIL} to get a dedicated set of IP addresses you can whitelist.`
    : undefined;
}

export function checkPermissions(
  request: APIRequest,
  flags?: TeamFlags,
  options?: PermissionOptions,
): { error?: string; code?: ErrorCodes } {
  // zdr perms β€” scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDR
  const scrapeMode = getScrapeZDR(flags);
  if (
    request.zeroDataRetention &&
    scrapeMode !== "allowed" &&
    scrapeMode !== "forced"
  ) {
    return {
      error: `Zero Data Retention (ZDR) is not enabled for your team. Contact ${SUPPORT_EMAIL} to enable this feature.`,
    };
  }

  // robots perms β€” ignoreRobots must be 'allowed' or 'forced'
  const robotsMode = getIgnoreRobots(flags);
  if (
    request.crawlerOptions?.ignoreRobotsTxt &&
    robotsMode !== "allowed" &&
    robotsMode !== "forced" &&
    // Under Safe Mode (outside lockdown) the robots rejection below returns
    // SAFE_MODE_BLOCKED; don't preempt it with the generic entitlement error.
    !(
      options?.safeMode &&
      !options.safeMode.lockdown &&
      options.safeMode.enforceRobots
    )
  ) {
    return {
      error: `The ignoreRobotsTxt parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
    };
  }
  // customRobotsAgent perms β€” separate flag for robotsUserAgent
  const customAgentMode = getCustomRobotsAgent(flags);
  if (
    request.crawlerOptions?.robotsUserAgent &&
    customAgentMode !== "allowed"
  ) {
    return {
      error: `The robotsUserAgent parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
    };
  }

  // threat protection perms β€” the flag must be 'allowed' or 'forced' for any
  // per-request threatProtection option, the org must not have locked down
  // request-level overrides, and a 'forced' team may never disable the
  // feature per-request.
  const threatProtectionOption =
    request.threatProtection ?? request.scrapeOptions?.threatProtection;
  if (threatProtectionOption !== undefined) {
    const effectiveThreatMode =
      options?.safeMode?.domainControls === true
        ? "forced"
        : getThreatProtection(flags);
    if (effectiveThreatMode !== "allowed" && effectiveThreatMode !== "forced") {
      return { error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE };
    }
    if (options?.threatProtectionOrgConfig?.allowRequestOverrides === false) {
      return { error: THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE };
    }
    if (
      effectiveThreatMode === "forced" &&
      threatProtectionOption.mode === "off"
    ) {
      return { error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE };
    }
  }

  const safeMode = options?.safeMode;
  if (safeMode && !safeMode.lockdown) {
    if (
      safeMode.disableStealthProxy &&
      (request.proxy === "stealth" || request.proxy === "enhanced")
    ) {
      return {
        error:
          'Safe Mode: stealth and enhanced proxies are not allowed for your organization. Remove the proxy option or use proxy: "basic".',
        code: "SAFE_MODE_BLOCKED",
      };
    }
    if (safeMode.enforceRobots && request.crawlerOptions?.ignoreRobotsTxt) {
      return {
        error:
          "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",
        code: "SAFE_MODE_BLOCKED",
      };
    }
    if (safeMode.disableAuthentication) {
      if (request.profile !== undefined) {
        return {
          error:
            "Safe Mode: browser profiles are not allowed for your organization (nothing behind a login).",
          code: "SAFE_MODE_BLOCKED",
        };
      }
      const loginAction = (request.actions ?? []).find(a =>
        SAFE_MODE_LOGIN_ACTIONS.includes(a.type),
      );
      if (loginAction) {
        return {
          error: `Safe Mode: the ${loginAction.type} action is not allowed for your organization (nothing behind a login).`,
          code: "SAFE_MODE_BLOCKED",
        };
      }
      const credentialHeader = Object.keys(request.headers ?? {}).find(h =>
        SAFE_MODE_CREDENTIAL_HEADERS.includes(h.toLowerCase()),
      );
      if (credentialHeader) {
        return {
          error: `Safe Mode: the ${credentialHeader} header is not allowed for your organization (nothing behind a login).`,
          code: "SAFE_MODE_BLOCKED",
        };
      }
    }
  }

  const usOnlyLocationError = usOnlyLocationRefusal(
    flags,
    request.location?.country,
    request.geolocation?.country,
    request.scrapeOptions?.location?.country,
    request.scrapeOptions?.geolocation?.country,
  );
  if (usOnlyLocationError) {
    return { error: usOnlyLocationError, code: "US_ONLY_UNAVAILABLE" };
  }

  const ipWhitelistError = ipWhitelistRefusal(
    flags,
    request.location?.country,
    request.scrapeOptions?.location?.country,
  );
  if (ipWhitelistError) {
    return { error: ipWhitelistError };
  }

  return {};
}