Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

10.3 KB
Raw Blame
import { Request, Response } from "express";
import { billTeam } from "../../services/billing/credit_billing";
import { autumnService } from "../../services/autumn/autumn.service";
import { authenticateUser } from "../auth";
import { RateLimiterMode, ScrapeJobSingleUrls } from "../../types";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { PageOptions, SearchOptions } from "../../lib/entities";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
import { redisEvictConnection } from "../../../src/services/redis";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
import {
  fromLegacyScrapeOptions,
  AuthCreditUsageChunk,
  toLegacyDocument,
} from "../v1/types";
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq-router";
import { defaultOrigin } from "../../lib/default-values";
import { getSearchZDR } from "../../lib/zdr-helpers";
import {
  isThreatProtectionForced,
  THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
} from "../../lib/threat-protection/request";
import {
  getSafeMode,
  SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { isUsOnly } from "../../lib/us-only";
import { ipWhitelistRefusal } from "../../lib/permissions";
import { UsOnlyUnavailableError } from "../../scraper/scrapeURL/error";

async function searchHelper(
  jobId: string,
  req: Request,
  team_id: string,
  crawlerOptions: any,
  pageOptions: PageOptions,
  searchOptions: SearchOptions,
  acuc: AuthCreditUsageChunk | null,
): Promise<{
  success: boolean;
  error?: string;
  data?: any;
  returnCode: number;
}> {
  const flags = acuc?.flags ?? null;
  const org_id = acuc?.org_id ?? null;
  const api_key_id = acuc?.api_key_id ?? null;
  const query = req.body.query;
  const advanced = false;
  if (!query) {
    return { success: false, error: "Query is required", returnCode: 400 };
  }

  const tbs = searchOptions.tbs ?? undefined;
  const filter = searchOptions.filter ?? undefined;
  let num_results = Math.min(searchOptions.limit ?? 7, 10);

  if (team_id === "d97c4ceb-290b-4957-8432-2b2a02727d95") {
    num_results = 1;
  }

  const num_results_buffer = Math.floor(num_results * 1.5);

  let res = await search({
    query,
    logger,
    advanced,
    num_results: num_results_buffer,
    tbs,
    filter,
    lang: searchOptions.lang ?? "en",
    country: searchOptions.country ?? "us",
    location: searchOptions.location,
    usOnly: isUsOnly(flags),
  });

  let justSearch = pageOptions.fetchPageContent === false;

  const { scrapeOptions, internalOptions } = fromV0Combo(
    pageOptions,
    undefined,
    60000,
    crawlerOptions,
    team_id,
  );
  internalOptions.orgId = org_id;
  if (isUsOnly(flags)) {
    internalOptions.teamFlags = flags;
  }

  if (justSearch) {
    const searchCredits = Math.ceil(res.length / 10) * 2;
    billTeam(
      team_id,
      org_id,
      searchCredits,
      api_key_id,
      {
        endpoint: "search",
        jobId,
        chargeId: jobId,
        externalRequestId: externalRequestId(req),
      },
      logger,
    ).catch(error => {
      logger.error(
        `Failed to bill team ${team_id} for ${searchCredits} credits: ${error}`,
      );
      // Optionally, you could notify an admin or add to a retry queue here
    });
    return { success: true, data: res, returnCode: 200 };
  }

  res = res.filter(
    r =>
      !isUrlBlocked(r.url, flags, {
        team_id,
        org_id,
        origin: req.body?.origin ?? null,
      }),
  );
  if (res.length > num_results) {
    res = res.slice(0, num_results);
  }

  if (res.length === 0) {
    return { success: true, error: "No search results found", returnCode: 200 };
  }

  const jobPriority = await getJobPriority({
    team_id,
    acuc,
    basePriority: 20,
  });
  const billing = {
    endpoint: "search" as const,
    jobId,
    externalRequestId: externalRequestId(req),
  };

  // filter out social media links

  const jobDatas = res.map(x => {
    const url = x.url;
    const uuid = uuidv7();
    return {
      jobId: uuid,
      data: {
        url,
        mode: "single_urls" as const,
        team_id: team_id,
        scrapeOptions,
        internalOptions,
        startTime: Date.now(),
        zeroDataRetention: false, // not supported on v0
        apiKeyId: api_key_id,
        origin: req.body.origin ?? defaultOrigin,
        requestId: jobId,
        billing,
      } satisfies ScrapeJobSingleUrls,
    };
  });

  // TODO: addScrapeJobs
  for (const job of jobDatas) {
    await addScrapeJob(job.data, job.jobId, jobPriority, false, true);
  }

  const docs = (
    await Promise.all(jobDatas.map(x => waitForJob(x.jobId, 60000, false)))
  ).map(x => toLegacyDocument(x, internalOptions));

  if (docs.length === 0) {
    return { success: true, error: "No search results found", returnCode: 200 };
  }

  await scrapeQueue.removeJobs(jobDatas.map(x => x.jobId));

  // make sure doc.content is not empty
  const filteredDocs = docs.filter(
    (doc: any) => doc && doc.content && doc.content.trim().length > 0,
  );

  if (filteredDocs.length === 0) {
    return {
      success: true,
      error: "No page found",
      returnCode: 200,
      data: docs,
    };
  }

  return {
    success: true,
    data: filteredDocs,
    returnCode: 200,
  };
}

export async function searchController(req: Request, res: Response) {
  try {
    // make sure to authenticate user first, Bearer <token>
    const auth = await authenticateUser(req, res, RateLimiterMode.Search);
    if (!auth.success) {
      if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
      return res.status(auth.status).json({ error: auth.error });
    }
    const { team_id, chunk } = auth;

    const v0SearchZDRMode = getSearchZDR(chunk?.flags);
    if (v0SearchZDRMode === "forced-zdr" || v0SearchZDRMode === "forced-anon") {
      return res.status(400).json({
        error:
          "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",
      });
    }

    if (isThreatProtectionForced(chunk?.flags)) {
      return res.status(403).json({
        error: THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
      });
    }

    if (getSafeMode(chunk?.flags)) {
      return res.status(403).json({
        error: SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
      });
    }

    const jobId = uuidv7();

    await logRequest({
      id: jobId,
      kind: "search",
      api_version: "v0",
      external_request_id: externalRequestId(req),
      team_id,
      origin: req.body.origin ?? "api",
      integration: req.body.integration,
      target_hint: req.body.query ?? "",
      zeroDataRetention: false, // not supported on v0
      api_key_id: chunk?.api_key_id ?? null,
    });

    redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>
      logger.error("Failed to add team to teams_using_v0", {
        error,
        team_id,
      }),
    );

    redisEvictConnection
      .sadd("teams_using_v0:" + team_id, "search:" + jobId)
      .catch(error =>
        logger.error("Failed to add team to teams_using_v0 (2)", {
          error,
          team_id,
        }),
      );

    const crawlerOptions = req.body.crawlerOptions ?? {};
    const pageOptions = req.body.pageOptions ?? {
      includeHtml: req.body.pageOptions?.includeHtml ?? false,
      onlyMainContent: req.body.pageOptions?.onlyMainContent ?? false,
      fetchPageContent: req.body.pageOptions?.fetchPageContent ?? true,
      removeTags: req.body.pageOptions?.removeTags ?? [],
      fallback: req.body.pageOptions?.fallback ?? false,
    };
    const origin = req.body.origin ?? "api";

    const ipWhitelistError =
      pageOptions.fetchPageContent !== false
        ? ipWhitelistRefusal(chunk?.flags, pageOptions?.geolocation?.country)
        : undefined;
    if (ipWhitelistError) {
      return res.status(403).json({ error: ipWhitelistError });
    }

    const searchOptions = req.body.searchOptions ?? { limit: 5 };

    try {
      // No org, no Autumn customer to gate against: fail open, exactly as
      // checkCredits answered for an identity it could not name.
      const orgId = chunk?.org_id ?? null;
      const autumnResult = orgId
        ? await autumnService.checkCredits({
            teamId: team_id,
            orgId,
            value: 1,
            properties: {
              source: "v0/search",
              apiKeyId: chunk?.api_key_id ?? null,
            },
          })
        : null;
      // null = Autumn unavailable / self-hosted -> fail open, matching v1/v2.
      if (autumnResult !== null && !autumnResult.allowed) {
        return res.status(402).json({ error: "Insufficient credits" });
      }
    } catch (error) {
      logger.error(error);
      return res.status(500).json({ error: "Internal server error" });
    }
    const startTime = new Date().getTime();
    const result = await searchHelper(
      jobId,
      req,
      team_id,
      crawlerOptions,
      pageOptions,
      searchOptions,
      chunk,
    );
    const endTime = new Date().getTime();
    const timeTakenInSeconds = (endTime - startTime) / 1000;
    await logSearch({
      id: jobId,
      request_id: jobId,
      query: req.body.query,
      num_results: result.data?.length ?? 0,
      is_successful: result.success,
      error: result.error,
      results: result.data,
      time_taken: timeTakenInSeconds,
      team_id: team_id,
      options: searchOptions,
      credits_cost: pageOptions.fetchPageContent
        ? 0
        : Math.ceil((result.data?.length ?? 0) / 10) * 2,
      zeroDataRetention: false, // not supported
    });
    return res.status(result.returnCode).json(result);
  } catch (error) {
    if (error instanceof ScrapeJobTimeoutError) {
      return res.status(408).json({ error: error.message });
    }
    if (error instanceof UsOnlyUnavailableError) {
      return res.status(403).json({ code: error.code, error: error.message });
    }

    logger.error("Unhandled error occurred in search", { error });
    return res.status(500).json({ error: error.message });
  }
}