Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

10.9 KB
Raw Blame
import { v7 as uuidv7 } from "uuid";
import { Request, Response } from "express";
import {
  RequestWithAuth,
  ExtractRequest,
  extractRequestSchema,
  ExtractResponse,
} from "./types";
import { addExtractJobToQueue } from "../../services/queue-service";
import { saveExtract } from "../../lib/extract/extract-redis";
import { getTeamIdSyncB } from "../../lib/extract/team-id-sync";
import { ExtractResult } from "../../lib/extract/types";
import {
  checkExtractAvailable,
  performExtraction_F0,
} from "../../lib/extract/fire-0/extraction-service-f0";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { logger as _logger } from "../../lib/logger";
import {
  fromV1ScrapeOptions,
  ExtractRequest as V2ExtractRequest,
} from "../v2/types";
import { createWebhookSender, WebhookEvent } from "../../services/webhook";
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";

import { config } from "../../config";
import {
  checkUrlsAgainstThreatPolicy,
  resolveThreatProtection,
} from "../../lib/threat-protection/request";
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
import { ipWhitelistRefusal } from "../../lib/permissions";
import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";
import { isSdkOrigin } from "../../lib/sdk-origin";
async function oldExtract(
  req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
  res: Response<ExtractResponse>,
  extractId: string,
) {
  // Means that are in the non-queue system
  // TODO: Remove this once all teams have transitioned to the new system

  const sender = await createWebhookSender({
    teamId: req.auth.team_id,
    jobId: extractId,
    webhook: req.body.webhook,
    v0: false,
  });

  sender?.send(WebhookEvent.EXTRACT_STARTED, { success: true });

  try {
    // Convert v1 scrapeOptions to v2 format
    const scrapeOptions = req.body.scrapeOptions
      ? fromV1ScrapeOptions(
          req.body.scrapeOptions,
          req.body.scrapeOptions.timeout,
          req.auth.team_id,
        ).scrapeOptions
      : undefined;

    // Create request with converted scrapeOptions (v2 format)
    const request: V2ExtractRequest = {
      ...req.body,
      scrapeOptions,
    } as V2ExtractRequest;

    let result: ExtractResult;
    result = await performExtraction_F0(extractId, {
      request,
      teamId: req.auth.team_id,
      apiKeyId: req.acuc?.api_key_id ?? null,
      externalRequestId: externalRequestId(req),
    });

    if (sender) {
      if (result.success) {
        sender.send(WebhookEvent.EXTRACT_COMPLETED, {
          success: true,
          data: [result],
        });
      } else {
        sender.send(WebhookEvent.EXTRACT_FAILED, {
          success: false,
          error: result.error ?? "Unknown error",
        });
      }
    }

    return res.status(200).json(result);
  } catch (error) {
    sender?.send(WebhookEvent.EXTRACT_FAILED, {
      success: false,
      error: error instanceof Error ? error.message : "Unknown error",
    });

    return res.status(500).json({
      success: false,
      error: error instanceof Error ? error.message : "Unknown error",
    });
  }
}
/**
 * Extracts data from the provided URLs based on the request parameters.
 * Currently in beta.
 * @param req - The request object containing authentication and extraction details.
 * @param res - The response object to send the extraction results.
 * @returns A promise that resolves when the extraction process is complete.
 */
export async function extractController(
  req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
  res: Response<ExtractResponse>,
) {
  const selfHosted = config.USE_DB_AUTHENTICATION !== true;
  const originalRequest = { ...req.body };
  req.body = extractRequestSchema.parse(req.body);
  const extractId = uuidv7();

  if (getScrapeZDR(req.acuc?.flags) === "forced") {
    return res.status(400).json({
      success: false,
      error:
        "Your team has zero data retention enabled. This is not supported on extract. Please contact [email protected] to unblock this feature.",
    });
  }

  try {
    checkExtractAvailable(req.acuc?.flags);
  } catch (error) {
    if (!(error instanceof UsOnlyUnavailableError)) throw error;
    return res.status(403).json({
      success: false,
      code: error.code,
      error: error.message,
    });
  }

  const ipWhitelistError = ipWhitelistRefusal(
    req.acuc?.flags,
    req.body.scrapeOptions?.location?.country,
    req.body.scrapeOptions?.geolocation?.country,
  );
  if (ipWhitelistError) {
    return res.status(403).json({
      success: false,
      error: ipWhitelistError,
    });
  }

  const invalidURLs: string[] =
    req.body.urls?.filter((url: string) =>
      isUrlBlocked(url, req.acuc?.flags ?? null, {
        team_id: req.auth.team_id,
        org_id: req.acuc?.org_id ?? null,
        origin: req.body.origin ?? null,
      }),
    ) ?? [];

  emitRejectedScrapeActivityEvents(
    invalidURLs.map(url => ({
      scrapeId: uuidv7(),
      requestId: extractId,
      endpoint: "extract",
      teamId: req.auth.team_id,
      apiKeyId: req.acuc?.api_key_id ?? null,
      auditMetadata: req.body.scrapeOptions?.auditMetadata,
      url,
      error: new UnsupportedSiteError(),
      origin: req.body.origin ?? "api",
      integration: req.body.integration,
      zeroDataRetention: false,
    })),
  );

  const createdAt = Date.now();

  if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {
    if (!res.headersSent) {
      return res.status(403).json({
        success: false,
        error: UNSUPPORTED_SITE_MESSAGE,
      });
    }
  }

  // Threat protection: check target URLs before fetching. Blocked URLs are
  // reported via invalidURLs (with ignoreInvalidURLs) or reject the request.
  // Discovered URLs are enforced in the scrape pipeline via the policy
  // threaded through the extract job.
  const threatProtection = await resolveThreatProtection({
    teamId: req.auth.team_id,
    orgId: req.acuc?.org_id ?? null,
    flags: req.acuc?.flags ?? null,
    override: req.body.threatProtection,
  });
  if (threatProtection.error) {
    return res.status(403).json({
      success: false,
      error: threatProtection.error,
    });
  }
  if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {
    const { blocked, decisionsByUrl } = await checkUrlsAgainstThreatPolicy(
      req.body.urls ?? [],
      threatProtection.policy,
      { teamId: req.auth.team_id },
    );
    // Consulted decisions bill the scan fee (+2 per unique scanned URL) —
    // including when the request is rejected below: the scans already
    // happened.
    const threatScanCredits = calculateThreatScanCredits(
      decisionsByUrl.values(),
    );
    if (threatScanCredits > 0) {
      billTeam(
        req.auth.team_id,
        req.acuc?.org_id ?? null,
        threatScanCredits,
        req.acuc?.api_key_id ?? null,
        {
          endpoint: "extract",
          externalRequestId: externalRequestId(req),
          // Suffixed: the extract's MAIN charge (fire-0) uses the bare
          // extractId — a shared key would collapse the two into one charge.
          chargeId: `${extractId}:threat`,
        },
      ).catch(error => {
        _logger.error(
          `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
        );
      });
    }
    if (blocked.length > 0) {
      emitRejectedScrapeActivityEvents(
        blocked
          .filter(blockedUrl => !invalidURLs.includes(blockedUrl.url))
          .map(blockedUrl => ({
            scrapeId: uuidv7(),
            requestId: extractId,
            endpoint: "extract",
            teamId: req.auth.team_id,
            apiKeyId: req.acuc?.api_key_id ?? null,
            auditMetadata: req.body.scrapeOptions?.auditMetadata,
            url: blockedUrl.url,
            error: new UnsafeDomainBlockedError(
              blockedUrl.url,
              blockedUrl.decision,
            ),
            threatDecisions: [blockedUrl.decision],
            origin: req.body.origin ?? "api",
            integration: req.body.integration,
            zeroDataRetention: false,
          })),
      );
      if (req.body.ignoreInvalidURLs) {
        invalidURLs.push(...blocked.map(x => x.url));
      } else {
        const first = blocked[0];
        const error = new UnsafeDomainBlockedError(first.url, first.decision);
        return res.status(403).json({
          success: false,
          code: error.code,
          error: error.message,
        });
      }
    }
  }

  _logger.info("Extract starting...", {
    request: req.body,
    originalRequest,
    teamId: req.auth.team_id,
    team_id: req.auth.team_id,
    extractId,
    zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",
  });

  await logRequest({
    id: extractId,
    kind: "extract",
    api_version: "v1",
    external_request_id: externalRequestId(req),
    team_id: req.auth.team_id,
    origin: req.body.origin ?? "api",
    integration: req.body.integration,
    target_hint: req.body.urls?.[0] ?? "",
    zeroDataRetention: false, // not supported for extract
    api_key_id: req.acuc?.api_key_id ?? null,
  });

  const scrapeOptions = req.body.scrapeOptions
    ? fromV1ScrapeOptions(
        req.body.scrapeOptions,
        req.body.scrapeOptions.timeout,
        req.auth.team_id,
      ).scrapeOptions
    : undefined;

  const jobData = {
    request: {
      ...req.body,
      scrapeOptions,
    },
    teamId: req.auth.team_id,
    extractId,
    agent: req.body.agent,
    apiKeyId: req.acuc?.api_key_id ?? null,
    externalRequestId: externalRequestId(req),
    createdAt,
  };

  if (
    (await getTeamIdSyncB(req.auth.team_id)) &&
    req.body.origin !== "api-sdk" &&
    req.body.origin !== "website" &&
    !isSdkOrigin(req.body.origin)
  ) {
    return await oldExtract(req, res, extractId);
  }

  await saveExtract(extractId, {
    id: extractId,
    team_id: req.auth.team_id,
    createdAt,
    status: "processing",
    showSteps: req.body.__experimental_streamSteps,
    showLLMUsage: req.body.__experimental_llmUsage,
    showSources: req.body.__experimental_showSources || req.body.showSources,
    showCostTracking: req.body.__experimental_showCostTracking,
    zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",
  });

  await addExtractJobToQueue(extractId, jobData);

  return res.status(200).json({
    success: true,
    id: extractId,
    urlTrace: [],
    ...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs
      ? {
          invalidURLs,
        }
      : {}),
  });
}