Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

11.8 KB
Raw Blame
import { v7 as uuidv7 } from "uuid";
import { AGENT_REQUEST_CREDITS_SHARDS } from "../../lib/request-credits-store";
import { Response } from "express";
import {
  AgentRequest,
  AgentResponse,
  RequestWithAuth,
  agentRequestSchema,
} from "./types";
import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { config } from "../../config";
import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";
import {
  AGENT_ZDR_DEFAULT_RETENTION_MINUTES,
  getAgentZdrRetentionMinutes,
  getScrapeZDR,
  getSearchZDR,
} from "../../lib/zdr-helpers";
import {
  checkUrlsAgainstThreatPolicy,
  resolveThreatProtection,
} from "../../lib/threat-protection/request";
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { fetchAgentThread, threadErrorFor } from "./agent-thread";
import { isUsOnly } from "../../lib/us-only";
import {
  AGENT_DEFAULT_MAX_CREDITS,
  gateAgentCredits,
} from "./agent-credit-gate";

export async function agentController(
  req: RequestWithAuth<{}, AgentResponse, AgentRequest>,
  res: Response<AgentResponse>,
) {
  const agentId = uuidv7();

  const originalRequest = { ...req.body };
  req.body = agentRequestSchema.parse(req.body);

  // An agent run scrapes and searches under ZDR, so it needs both
  // entitlements up front instead of failing on its first inner call.
  const scrapeZDR = getScrapeZDR(req.acuc?.flags);
  const searchZDR = getSearchZDR(req.acuc?.flags);
  if (
    (req.body.zeroDataRetention === true || scrapeZDR === "forced") &&
    !(
      (scrapeZDR === "allowed" || scrapeZDR === "forced") &&
      (searchZDR === "allowed" || searchZDR === "forced-zdr")
    )
  ) {
    return res.status(400).json({
      success: false,
      code: "zdr_not_enabled",
      error:
        "Zero Data Retention (ZDR) for agent runs requires both scrape ZDR and search ZDR to be enabled for your team. Contact [email protected] to enable it.",
    });
  }

  const zeroDataRetention =
    scrapeZDR === "forced" ||
    (req.body.zeroDataRetention === true && scrapeZDR === "allowed");

  const logger = _logger.child({
    agentId,
    extractId: agentId,
    jobId: agentId,
    teamId: req.auth.team_id,
    team_id: req.auth.team_id,
    module: "api/v2",
    method: "agentController",
    zeroDataRetention,
  });

  if (zeroDataRetention && req.body.threadId) {
    return res.status(400).json({
      success: false,
      code: "zdr_threads_unsupported",
      error:
        "Agent threads are not supported with zero data retention. Remove threadId to start a new run.",
    });
  }

  // Read once, here: the agent service keeps the run's retention from start.
  const zeroDataRetentionMinutes = zeroDataRetention
    ? getAgentZdrRetentionMinutes(req.acuc?.flags)
    : null;
  if (zeroDataRetentionMinutes === "invalid") {
    logger.error("Invalid agentZdrRetentionMinutes team flag", {
      agentZdrRetentionMinutes: req.acuc?.flags?.agentZdrRetentionMinutes,
    });
    return res.status(500).json({
      success: false,
      code: "zdr_retention_misconfigured",
      error:
        "Zero Data Retention is misconfigured for your team. Contact [email protected].",
    });
  }

  logger.info("Agent starting...", {
    request: req.body,
    originalRequest,
  });

  // Threat protection: check the agent's starting URLs before handing off to
  // the agent service. Content the agent fetches through the API
  // (agent-interop scrapes) is additionally enforced by the scrape pipeline's
  // org-policy resolution; in-page navigations performed by the remote
  // browser cannot be intercepted here.
  const threatProtection = await resolveThreatProtection({
    teamId: req.auth.team_id,
    orgId: req.acuc?.org_id ?? null,
    flags: req.acuc?.flags ?? null,
    override: req.body.threatProtection,
  });
  if (threatProtection.error) {
    return res.status(403).json({
      success: false,
      error: threatProtection.error,
    });
  }
  if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {
    const { blocked, decisionsByUrl } = await checkUrlsAgainstThreatPolicy(
      req.body.urls ?? [],
      threatProtection.policy,
      { teamId: req.auth.team_id },
    );
    if (blocked.length > 0) {
      // The whole request is rejected below, so no agent job will ever run
      // to bill the allowed start URLs' scans — every consulted decision
      // (allowed and blocked) bills its scan fee here (+2 per unique
      // scanned URL): the scans already happened.
      const threatScanCredits = calculateThreatScanCredits(
        decisionsByUrl.values(),
      );
      if (threatScanCredits > 0) {
        billTeam(
          req.auth.team_id,
          req.acuc?.org_id ?? null,
          threatScanCredits,
          req.acuc?.api_key_id ?? null,
          {
            endpoint: "agent",
            jobId: agentId,
            chargeId: `${agentId}:threat`,
            hostedMcp: req.auth.hostedMcp,
          },
        ).catch(error => {
          logger.error(
            `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
          );
        });
      }
      const first = blocked[0];
      const error = new UnsafeDomainBlockedError(first.url, first.decision);
      emitRejectedScrapeActivityEvents(
        blocked.map(blockedUrl => ({
          scrapeId: uuidv7(),
          requestId: agentId,
          endpoint: "agent",
          teamId: req.auth.team_id,
          apiKeyId: req.acuc?.api_key_id ?? null,
          auditMetadata: req.body.auditMetadata,
          url: blockedUrl.url,
          error: new UnsafeDomainBlockedError(
            blockedUrl.url,
            blockedUrl.decision,
          ),
          threatDecisions: [blockedUrl.decision],
          origin: req.body.origin ?? "api",
          integration: req.body.integration,
          zeroDataRetention,
        })),
      );
      return res.status(403).json({
        success: false,
        code: error.code,
        error: error.message,
      });
    }
  }

  if (!config.EXTRACT_V3_BETA_URL) {
    throw new Error("Agent beta is not enabled.");
  }

  // A follow-up is validated before the free request is consumed and before
  // logRequest, so a rejected continuation leaves no orphan request row.
  if (req.body.threadId) {
    const thread = await fetchAgentThread(
      req.body.threadId,
      req.auth.team_id,
    ).catch(error => {
      logger.error("Failed to check agent thread.", { error });
      return null;
    });

    if (thread === null) {
      return res.status(500).json({
        success: false,
        error: "Failed to check agent thread.",
      });
    }

    if (thread.status !== 200) {
      const mapped = threadErrorFor(thread.status);

      if (!mapped) {
        logger.error("Failed to check agent thread.", {
          status: thread.status,
          text: await thread.text(),
        });

        return res.status(500).json({
          success: false,
          error: "Failed to check agent thread.",
        });
      }

      const body = (await thread.json().catch(() => null)) as {
        runId?: unknown;
      } | null;

      return res.status(thread.status).json({
        success: false,
        code: mapped.code,
        error: mapped.error,
        ...(typeof body?.runId === "string" ? { runId: body.runId } : {}),
      });
    }
  }

  // A request above the free cap is always paid. Otherwise a free request is
  // consumed atomically here, and only a consumed one skips the credit gate.
  const highCreditRequest =
    req.body.maxCredits !== undefined &&
    req.body.maxCredits > AGENT_DEFAULT_MAX_CREDITS;

  let freeRequest: any;

  if (config.USE_DB_AUTHENTICATION && !highCreditRequest) {
    freeRequest = await agentConsumeFreeRequestIfLeft(req.auth.team_id);
  }

  const isFreeRequest = highCreditRequest
    ? false
    : config.USE_DB_AUTHENTICATION
      ? !!freeRequest?.[0]?.consumed
      : true;

  let maxCredits = req.body.maxCredits;
  let stopOnInsufficientCredits = false;
  if (!isFreeRequest) {
    const gate = await gateAgentCredits({
      teamId: req.auth.team_id,
      orgId: req.auth.org_id,
      apiKeyId: req.acuc?.api_key_id ?? null,
      requestedMaxCredits: req.body.maxCredits,
      hostedMcp: req.auth.hostedMcp,
    });
    if (!gate.allowed) {
      logger.warn("Insufficient credits to start agent run", {
        balance: gate.balance,
      });
      return res.status(402).json({
        success: false,
        error: `Insufficient credits to start an agent run. Your team's credit balance is ${gate.balance} credits. Add credits at https://firecrawl.dev/pricing to continue.`,
      });
    }
    maxCredits = gate.maxCredits;
    stopOnInsufficientCredits = gate.stopOnInsufficientCredits;
    if (gate.onTeamCredits) req.auth.hostedMcp = false;
  }

  await logRequest({
    id: agentId,
    kind: "agent",
    api_version: "v2",
    external_request_id: externalRequestId(req),
    team_id: req.auth.team_id,
    origin: req.body.origin ?? "api",
    integration: req.body.integration,
    target_hint: req.body.urls?.[0] ?? req.body.prompt ?? "",
    zeroDataRetention,
    api_key_id: req.acuc?.api_key_id ?? null,
    creditsShards: AGENT_REQUEST_CREDITS_SHARDS,
    ...(zeroDataRetention
      ? {
          jobAccessZeroDataRetentionMinutes:
            zeroDataRetentionMinutes ?? AGENT_ZDR_DEFAULT_RETENTION_MINUTES,
        }
      : {}),
  });

  const passthrough = await fetch(
    config.EXTRACT_V3_BETA_URL + "/internal/extracts",
    {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`,
      },
      body: JSON.stringify({
        id: agentId,
        urls: req.body.urls,
        schema: req.body.schema,
        prompt: req.body.prompt,
        apiKey: req.acuc!.api_key,
        apiKeyId: req.acuc!.api_key_id ?? undefined,
        teamId: req.auth.team_id,
        isFreeRequest,
        maxCredits: maxCredits ?? undefined,
        stopOnInsufficientCredits: stopOnInsufficientCredits || undefined,
        strictConstrainToURLs: req.body.strictConstrainToURLs ?? undefined,
        webhook: req.body.webhook ?? undefined,
        model: req.body.model,
        effort: req.body.effort,
        auditMetadata: req.body.auditMetadata,
        threadId: req.body.threadId,
        mode: req.body.mode,
        exchange: req.body.exchange,
        location: req.body.location,
        safe: req.body.safe,
        ...(req.auth.hostedMcp ? { hostedMcp: true } : {}),
        ...(zeroDataRetention ? { zeroDataRetention: true } : {}),
        ...(zeroDataRetentionMinutes !== null
          ? { zeroDataRetentionMinutes }
          : {}),
        ...(isUsOnly(req.acuc?.flags) ? { usOnly: true } : {}),
      }),
    },
  );

  if (passthrough.status !== 200) {
    const text = await passthrough.text();

    logger.error("Failed to passthrough agent request.", {
      status: passthrough.status,
      text,
    });

    // TODO: should we try to insert a failed agent row here, since a request is already created? - Mogery

    return res.status(500).json({
      success: false,
      error: "Failed to passthrough agent request.",
    });
  }

  // The agent service mints the thread id, so the response body is the only
  // place it exists at this point.
  const result = (await passthrough.json().catch(() => null)) as {
    threadId?: unknown;
    threadTurn?: unknown;
  } | null;

  return res.status(200).json({
    success: true,
    id: agentId,
    ...(typeof result?.threadId === "string"
      ? { threadId: result.threadId }
      : {}),
    ...(typeof result?.threadTurn === "number"
      ? { threadTurn: result.threadTurn }
      : {}),
  });
}