Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: init

Nicolas392d2c0fb8cd
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index 3fc74f78c..1c3e98cb4 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -1,4 +1,3 @@
-import { authMiddleware } from "../../routes/v1";
import { RateLimiterMode } from "../../types";
import { authenticateUser } from "../auth";
import {
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
new file mode 100644
index 000000000..82efbcfb3
--- /dev/null
+++ b/apps/api/src/controllers/v2/search.ts
@@ -0,0 +1,412 @@
+import { Response } from "express";
+import {
+ Document,
+ RequestWithAuth,
+ SearchRequest,
+ SearchResponse,
+ searchRequestSchema,
+ ScrapeOptions,
+ TeamFlags,
+ scrapeOptions,
+} from "./types";
+import { billTeam } from "../../services/billing/credit_billing";
+import { v4 as uuidv4 } from "uuid";
+import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
+import { logJob } from "../../services/logging/log_job";
+import { getJobPriority } from "../../lib/job-priority";
+import { Mode } from "../../types";
+import { getScrapeQueue } from "../../services/queue-service";
+import { search } from "../../search";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import * as Sentry from "@sentry/node";
+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { logger as _logger } from "../../lib/logger";
+import type { Logger } from "winston";
+import { CostTracking } from "../../lib/extract/extraction-service";
+import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
+import { supabase_service } from "../../services/supabase";
+
+interface DocumentWithCostTracking {
+ document: Document;
+ costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
+}
+
+// Used for deep research
+export async function searchAndScrapeSearchResult(
+ query: string,
+ options: {
+ teamId: string;
+ origin: string;
+ timeout: number;
+ scrapeOptions: ScrapeOptions;
+ },
+ logger: Logger,
+ flags: TeamFlags,
+): Promise<DocumentWithCostTracking[]> {
+ try {
+ const searchResults = await search({
+ query,
+ num_results: 5,
+ });
+
+ const documentsWithCostTracking = await Promise.all(
+ searchResults.map((result) =>
+ scrapeSearchResult(
+ {
+ url: result.url,
+ title: result.title,
+ description: result.description,
+ },
+ options,
+ logger,
+ flags,
+ ),
+ ),
+ );
+
+ return documentsWithCostTracking;
+ } catch (error) {
+ return [];
+ }
+}
+
+async function scrapeSearchResult(
+ searchResult: { url: string; title: string; description: string },
+ options: {
+ teamId: string;
+ origin: string;
+ timeout: number;
+ scrapeOptions: ScrapeOptions;
+ },
+ logger: Logger,
+ flags: TeamFlags,
+ directToBullMQ: boolean = false,
+ isSearchPreview: boolean = false,
+): Promise<DocumentWithCostTracking> {
+ const jobId = uuidv4();
+ const jobPriority = await getJobPriority({
+ team_id: options.teamId,
+ basePriority: 10,
+ });
+
+ const costTracking = new CostTracking();
+
+ const zeroDataRetention = flags?.forceZDR ?? false;
+
+ try {
+ if (isUrlBlocked(searchResult.url, flags)) {
+ throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE);
+ }
+ logger.info("Adding scrape job", {
+ scrapeId: jobId,
+ url: searchResult.url,
+ teamId: options.teamId,
+ origin: options.origin,
+ zeroDataRetention,
+ });
+ await addScrapeJob(
+ {
+ url: searchResult.url,
+ mode: "single_urls" as Mode,
+ team_id: options.teamId,
+ scrapeOptions: {
+ ...options.scrapeOptions,
+ maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days
+ },
+ internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },
+ origin: options.origin,
+ is_scrape: true,
+ startTime: Date.now(),
+ zeroDataRetention,
+ },
+ {},
+ jobId,
+ jobPriority,
+ directToBullMQ,
+ );
+
+ const doc: Document = await waitForJob(jobId, options.timeout);
+
+ logger.info("Scrape job completed", {
+ scrapeId: jobId,
+ url: searchResult.url,
+ teamId: options.teamId,
+ origin: options.origin,
+ });
+ await getScrapeQueue().remove(jobId);
+
+ const document = {
+ title: searchResult.title,
+ description: searchResult.description,
+ url: searchResult.url,
+ ...doc,
+ };
+
+ let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const { data: costTrackingResponse, error: costTrackingError } = await supabase_service.from("firecrawl_jobs")
+ .select("cost_tracking")
+ .eq("job_id", jobId);
+
+ if (costTrackingError) {
+ logger.error("Error getting cost tracking", { error: costTrackingError });
+ throw costTrackingError;
+ }
+
+ costTracking = costTrackingResponse?.[0]?.cost_tracking;
+ } else {
+ costTracking = new CostTracking().toJSON();
+ }
+
+ return {
+ document,
+ costTracking,
+ };
+ } catch (error) {
+ logger.error(`Error in scrapeSearchResult: ${error}`, {
+ scrapeId: jobId,
+ url: searchResult.url,
+ teamId: options.teamId,
+ });
+
+ let statusCode = 0;
+ if (error?.message?.includes("Could not scrape url")) {
+ statusCode = 403;
+ }
+
+ const document: Document = {
+ title: searchResult.title,
+ description: searchResult.description,
+ url: searchResult.url,
+ metadata: {
+ statusCode,
+ error: error.message,
+ proxyUsed: "basic",
+ },
+ };
+
+ return {
+ document,
+ costTracking: new CostTracking().toJSON(),
+ };
+ }
+}
+
+export async function searchController(
+ req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
+ res: Response<SearchResponse>,
+) {
+ const jobId = uuidv4();
+ let logger = _logger.child({
+ jobId,
+ teamId: req.auth.team_id,
+ module: "search",
+ method: "searchController",
+ zeroDataRetention: req.acuc?.flags?.forceZDR,
+ });
+
+ if (req.acuc?.flags?.forceZDR) {
+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search. Please contact [email protected] to unblock this feature." });
+ }
+
+ let responseData: SearchResponse = {
+ success: true,
+ data: [],
+ };
+ const startTime = new Date().getTime();
+ const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+
+ let credits_billed = 0;
+ let allDocsWithCostTracking: DocumentWithCostTracking[] = [];
+
+ try {
+ req.body = searchRequestSchema.parse(req.body);
+
+ logger = logger.child({
+ query: req.body.query,
+ origin: req.body.origin,
+ });
+
+ let limit = req.body.limit;
+
+ // Buffer results by 50% to account for filtered URLs
+ const num_results_buffer = Math.floor(limit * 2);
+
+ logger.info("Searching for results");
+
+ let searchResults = await search({
+ query: req.body.query,
+ advanced: false,
+ num_results: num_results_buffer,
+ tbs: req.body.tbs,
+ filter: req.body.filter,
+ lang: req.body.lang,
+ country: req.body.country,
+ location: req.body.location,
+ });
+
+ if (req.body.ignoreInvalidURLs) {
+ searchResults = searchResults.filter(
+ (result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null),
+ );
+ }
+
+ logger.info("Searching completed", {
+ num_results: searchResults.length,
+ });
+
+ // Filter blocked URLs early to avoid unnecessary billing
+ if (searchResults.length > limit) {
+ searchResults = searchResults.slice(0, limit);
+ }
+
+ if (searchResults.length === 0) {
+ logger.info("No search results found");
+ responseData.warning = "No search results found";
+ } else if (
+ !req.body.scrapeOptions.formats ||
+ req.body.scrapeOptions.formats.length === 0
+ ) {
+ responseData.data = searchResults.map((r) => ({
+ url: r.url,
+ title: r.title,
+ description: r.description,
+ })) as Document[];
+ credits_billed = responseData.data.length;
+ } else {
+ logger.info("Scraping search results");
+ const scrapePromises = searchResults.map((result) =>
+ scrapeSearchResult(
+ result,
+ {
+ teamId: req.auth.team_id,
+ origin: req.body.origin,
+ timeout: req.body.timeout,
+ scrapeOptions: req.body.scrapeOptions,
+ },
+ logger,
+ req.acuc?.flags ?? null,
+ (req.acuc?.price_credits ?? 0) <= 3000,
+ isSearchPreview,
+ ),
+ );
+
+ const docsWithCostTracking = await Promise.all(scrapePromises);
+ logger.info("Scraping completed", {
+ num_docs: docsWithCostTracking.length,
+ });
+
+ const docs = docsWithCostTracking.map(item => item.document);
+ const filteredDocs = docs.filter(
+ (doc) =>
+ doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),
+ );
+
+ logger.info("Filtering completed", {
+ num_docs: filteredDocs.length,
+ });
+
+ if (filteredDocs.length === 0) {
+ responseData.data = docs;
+ responseData.warning = "No content found in search results";
+ } else {
+ responseData.data = filteredDocs;
+ }
+
+ const finalDocsForBilling = responseData.data;
+
+ const creditPromises = finalDocsForBilling.map(async (finalDoc) => {
+ const matchingDocWithCost = docsWithCostTracking.find(item =>
+ item.document.metadata && finalDoc.metadata && item.document.metadata.scrapeId === finalDoc.metadata.scrapeId
+ );
+
+ if (matchingDocWithCost) {
+ return await calculateCreditsToBeBilled(
+ req.body.scrapeOptions,
+ { teamId: req.auth.team_id, bypassBilling: true, zeroDataRetention: false },
+ matchingDocWithCost.document,
+ matchingDocWithCost.costTracking,
+ req.acuc?.flags ?? null,
+ );
+ } else {
+ return 1;
+ }
+ });
+
+ try {
+ const individualCredits = await Promise.all(creditPromises);
+ credits_billed = individualCredits.reduce((sum, credit) => sum + credit, 0);
+ } catch (error) {
+ logger.error("Error calculating credits for billing", { error });
+ credits_billed = responseData.data.length;
+ }
+
+ allDocsWithCostTracking = docsWithCostTracking;
+ }
+
+ // Bill team once for all successful results
+ if (!isSearchPreview) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id,
+ credits_billed,
+ ).catch((error) => {
+ logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,
+ );
+ });
+ }
+
+ const endTime = new Date().getTime();
+ const timeTakenInSeconds = (endTime - startTime) / 1000;
+
+ logger.info("Logging job", {
+ num_docs: responseData.data.length,
+ time_taken: timeTakenInSeconds,
+ });
+
+ logJob(
+ {
+ job_id: jobId,
+ success: true,
+ num_docs: responseData.data.length,
+ docs: responseData.data,
+ time_taken: timeTakenInSeconds,
+ team_id: req.auth.team_id,
+ mode: "search",
+ url: req.body.query,
+ scrapeOptions: req.body.scrapeOptions,
+ crawlerOptions: {
+ ...req.body,
+ query: undefined,
+ scrapeOptions: undefined,
+ },
+ origin: req.body.origin,
+ integration: req.body.integration,
+ credits_billed,
+ zeroDataRetention: false, // not supported
+ },
+ false,
+ isSearchPreview,
+ );
+
+ return res.status(200).json(responseData);
+ } catch (error) {
+ if (
+ error instanceof Error &&
+ (error.message.startsWith("Job wait") || error.message === "timeout")
+ ) {
+ return res.status(408).json({
+ success: false,
+ error: "Request timed out",
+ });
+ }
+
+ Sentry.captureException(error);
+ logger.error("Unhandled error occurred in search", { error });
+ return res.status(500).json({
+ success: false,
+ error: error.message,
+ });
+ }
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
new file mode 100644
index 000000000..84ca8d6e7
--- /dev/null
+++ b/apps/api/src/controllers/v2/types.ts
@@ -0,0 +1,1383 @@
+import { Request, Response } from "express";
+import { z } from "zod";
+import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
+import { countries } from "../../lib/validate-country";
+import {
+ ExtractorOptions,
+ PageOptions,
+ ScrapeActionContent,
+ Document as V0Document,
+} from "../../lib/entities";
+import { InternalOptions } from "../../scraper/scrapeURL";
+import { getURLDepth } from "../../scraper/WebScraper/utils/maxDepthUtils";
+
+export enum IntegrationEnum {
+ DIFY = "dify",
+ ZAPIER = "zapier",
+ PIPEDREAM = "pipedream",
+ RAYCAST = "raycast",
+ LANGCHAIN = "langchain",
+ CREWAI = "crewai",
+ LLAMAINDEX = "llamaindex",
+ N8N = "n8n",
+ CAMELAI = "camelai",
+ MAKE = "make",
+ FLOWISE = "flowise",
+ METAGPT = "metagpt",
+ RELEVANCEAI = 'relevanceai',
+}
+
+export type Format =
+ | "markdown"
+ | "html"
+ | "rawHtml"
+ | "links"
+ | "screenshot"
+ | "screenshot@fullPage"
+ | "extract"
+ | "json"
+ | "changeTracking";
+
+export const url = z.preprocess(
+ (x) => {
+ if (!protocolIncluded(x as string)) {
+ x = `http://${x}`;
+ }
+
+ // transforming the query parameters is breaking certain sites, so we're not doing it - mogery
+ // try {
+ // const urlObj = new URL(x as string);
+ // if (urlObj.search) {
+ // const searchParams = new URLSearchParams(urlObj.search.substring(1));
+ // return `${urlObj.origin}${urlObj.pathname}?${searchParams.toString()}`;
+ // }
+ // } catch (e) {
+ // }
+
+ return x;
+ },
+ z
+ .string()
+ .url()
+ .regex(/^https?:\/\//i, "URL uses unsupported protocol")
+ .refine(
+ (x) =>
+ /(\.[a-zA-Z0-9-\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}|\.xn--[a-zA-Z0-9-]{1,})(:\d+)?([\/?#]|$)/i.test(
+ x,
+ ),
+ "URL must have a valid top-level domain or be a valid path",
+ )
+ .refine((x) => {
+ try {
+ checkUrl(x as string);
+ return true;
+ } catch (_) {
+ return false;
+ }
+ }, "Invalid URL")
+ // .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),
+);
+
+const strictMessage =
+ "Unrecognized key in body -- please review the v1 API documentation for request body changes";
+
+export const agentExtractModelValue = 'fire-1'
+export const isAgentExtractModelValid = (x: string | undefined) => x?.toLowerCase() === agentExtractModelValue;
+
+export const agentOptionsExtract = z
+ .object({
+ model: z.string().default(agentExtractModelValue),
+ })
+ .strict(strictMessage);
+
+export type AgentOptions = z.infer<typeof agentOptionsExtract>;
+
+export const extractOptions = z
+ .object({
+ mode: z.enum(["llm"]).default("llm"),
+ schema: z.any().optional(),
+ systemPrompt: z
+ .string()
+ .max(10000)
+ .default(""),
+ prompt: z.string().max(10000).optional(),
+ temperature: z.number().optional(),
+ })
+ .strict(strictMessage)
+ .transform((data) => ({
+ ...data,
+ systemPrompt: "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required."
+ }));
+
+export const extractOptionsWithAgent = z
+ .object({
+ mode: z.enum(["llm"]).default("llm"),
+ schema: z.any().optional(),
+ systemPrompt: z
+ .string()
+ .max(10000)
+ .default(""),
+ prompt: z.string().max(10000).optional(),
+ temperature: z.number().optional(),
+ agent: z
+ .object({
+ model: z.string().default(agentExtractModelValue),
+ prompt: z.string().optional(),
+ })
+ .optional(),
+ })
+ .strict(strictMessage)
+ .transform((data) => ({
+ ...data,
+ systemPrompt: isAgentExtractModelValid(data.agent?.model)
+ ? `You are an expert web data extractor. Your task is to analyze the provided markdown content from a web page and generate a JSON object based *strictly* on the provided schema.
+
+Key Instructions:
+1. **Schema Adherence:** Populate the JSON object according to the structure defined in the schema.
+2. **Content Grounding:** Extract information *only* if it is explicitly present in the provided markdown. Do NOT infer or fabricate information.
+3. **Missing Information:** If a piece of information required by the schema cannot be found in the markdown, use \`null\` for that field's value.
+4. **SmartScrape Recommendation:**
+ * Assess if the *full* required data seems unavailable in the current markdown likely because:
+ - Content requires user interaction to reveal (e.g., clicking buttons, hovering, scrolling)
+ - Content uses pagination (e.g., "Load More" buttons, numbered pagination, infinite scroll)
+ - Content is dynamically loaded after user actions
+ * If the content requires user interaction or pagination to be fully accessible, set \`shouldUseSmartscrape\` to \`true\` in your response and provide a clear \`reasoning\` and \`prompt\` for the SmartScrape tool.
+ * If the content is simply JavaScript rendered but doesn't require interaction, set \`shouldUseSmartscrape\` to \`false\`.
+5. **Output Format:** Your final output MUST be a single, valid JSON object conforming precisely to the schema. Do not include any explanatory text outside the JSON structure.`
+ : "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required."
+ }));
+
+export type ExtractOptions = z.infer<typeof extractOptions>;
+
+const ACTIONS_MAX_WAIT_TIME = 60;
+const MAX_ACTIONS = 50;
+function calculateTotalWaitTime(
+ actions: any[] = [],
+ waitFor: number = 0,
+): number {
+ const actionWaitTime = actions.reduce((acc, action) => {
+ if (action.type === "wait") {
+ if (action.milliseconds) {
+ return acc + action.milliseconds;
+ }
+ // Consider selector actions as 1 second
+ if (action.selector) {
+ return acc + 1000;
+ }
+ }
+ return acc;
+ }, 0);
+
+ return waitFor + actionWaitTime;
+}
+
+export const actionSchema = z
+ .union([
+ z
+ .object({
+ type: z.literal("wait"),
+ milliseconds: z.number().int().positive().finite().optional(),
+ selector: z.string().optional(),
+ })
+ .refine(
+ (data) =>
+ (data.milliseconds !== undefined || data.selector !== undefined) &&
+ !(data.milliseconds !== undefined && data.selector !== undefined),
+ {
+ message:
+ "Either 'milliseconds' or 'selector' must be provided, but not both.",
+ },
+ ),
+ z.object({
+ type: z.literal("click"),
+ selector: z.string(),
+ all: z.boolean().default(false),
+ }),
+ z.object({
+ type: z.literal("screenshot"),
+ fullPage: z.boolean().default(false),
+ quality: z.number().min(1).max(100).optional(),
+ }),
+ z.object({
+ type: z.literal("write"),
+ text: z.string(),
+ }),
+ z.object({
+ type: z.literal("press"),
+ key: z.string(),
+ }),
+ z.object({
+ type: z.literal("scroll"),
+ direction: z.enum(["up", "down"]).optional().default("down"),
+ selector: z.string().optional(),
+ }),
+ z.object({
+ type: z.literal("scrape"),
+ }),
+ z.object({
+ type: z.literal("executeJavascript"),
+ script: z.string(),
+ }),
+ z.object({
+ type: z.literal("pdf"),
+ landscape: z.boolean().default(false),
+ scale: z.number().default(1),
+ format: z.enum(['A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'Letter', 'Legal', 'Tabloid', 'Ledger']).default("Letter"),
+ }),
+ ]);
+
+export type Action = z.infer<typeof actionSchema>;
+
+export const actionsSchema = z
+ .array(actionSchema)
+ .refine((actions) => actions.length <= MAX_ACTIONS, {
+ message: `Number of actions cannot exceed ${MAX_ACTIONS}`,
+ })
+ .refine(
+ (actions) =>
+ calculateTotalWaitTime(actions) <= ACTIONS_MAX_WAIT_TIME * 1000,
+ {
+ message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,
+ },
+ );
+
+const baseScrapeOptions = z
+ .object({
+ formats: z
+ .enum([
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ "screenshot",
+ "screenshot@fullPage",
+ "extract",
+ "json",
+ "changeTracking",
+ ])
+ .array()
+ .optional()
+ .default(["markdown"])
+ .refine(
+ (x) => !(x.includes("screenshot") && x.includes("screenshot@fullPage")),
+ "You may only specify either screenshot or screenshot@fullPage",
+ )
+ .refine(
+ (x) => !x.includes("changeTracking") || x.includes("markdown"),
+ "The changeTracking format requires the markdown format to be specified as well",
+ ),
+ headers: z.record(z.string(), z.string()).optional(),
+ includeTags: z.string().array().optional(),
+ excludeTags: z.string().array().optional(),
+ onlyMainContent: z.boolean().default(true),
+ timeout: z.number().int().positive().finite().safe().optional(),
+ waitFor: z
+ .number()
+ .int()
+ .nonnegative()
+ .finite()
+ .safe()
+ .max(60000)
+ .default(0),
+ // Deprecate this to jsonOptions
+ extract: extractOptions.optional(),
+ // New
+ jsonOptions: extractOptions.optional(),
+ changeTrackingOptions: z
+ .object({
+ prompt: z.string().optional(),
+ schema: z.any().optional(),
+ modes: z.enum(["json", "git-diff"]).array().optional().default([]),
+ tag: z.string().or(z.null()).default(null),
+ })
+ .optional(),
+ mobile: z.boolean().default(false),
+ parsePDF: z.boolean().default(true),
+ actions: actionsSchema.optional(),
+ // New
+ location: z
+ .object({
+ country: z
+ .string()
+ .optional()
+ .refine(
+ (val) =>
+ !val ||
+ Object.keys(countries).includes(val.toUpperCase()) ||
+ val === "US-generic",
+ {
+ message:
+ "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",
+ },
+ )
+ .transform((val) => (val ? val.toUpperCase() : "US-generic")),
+ languages: z.string().array().optional(),
+ })
+ .optional(),
+
+ // Deprecated
+ geolocation: z
+ .object({
+ country: z
+ .string()
+ .optional()
+ .refine(
+ (val) => !val || Object.keys(countries).includes(val.toUpperCase()),
+ {
+ message:
+ "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",
+ },
+ )
+ .transform((val) => (val ? val.toUpperCase() : "US-generic")),
+ languages: z.string().array().optional(),
+ })
+ .optional(),
+ skipTlsVerification: z.boolean().default(false),
+ removeBase64Images: z.boolean().default(true),
+ fastMode: z.boolean().default(false),
+ useMock: z.string().optional(),
+ blockAds: z.boolean().default(true),
+ proxy: z.enum(["basic", "stealth", "auto"]).optional(),
+ maxAge: z.number().int().gte(0).safe().default(0),
+ storeInCache: z.boolean().default(true),
+ // @deprecated
+ __experimental_cache: z.boolean().default(false).optional(),
+ __searchPreviewToken: z.string().optional(),
+ __experimental_omce: z.boolean().default(false).optional(),
+ })
+ .strict(strictMessage);
+
+const fire1Refine = (obj) => {
+ if (obj.agent?.model?.toLowerCase() === "fire-1" && obj.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {
+ return false;
+ }
+ return true;
+}
+const fire1RefineOpts = {
+ message: "You may only specify the FIRE-1 model in agent or jsonOptions.agent, but not both.",
+};
+const waitForRefine = (obj) => {
+ if (obj.waitFor && obj.timeout) {
+ if (typeof obj.timeout !== 'number' || obj.timeout <= 0) {
+ return false;
+ }
+ return obj.waitFor <= obj.timeout / 2;
+ }
+ return true;
+};
+const waitForRefineOpts = {
+ message: "waitFor must not exceed half of timeout",
+ path: ["waitFor"],
+};
+const extractRefine = (obj) => {
+ const hasExtractFormat = obj.formats?.includes("extract");
+ const hasExtractOptions = obj.extract !== undefined;
+ const hasJsonFormat = obj.formats?.includes("json");
+ const hasJsonOptions = obj.jsonOptions !== undefined;
+ return (
+ ((hasExtractFormat && hasExtractOptions) ||
+ (!hasExtractFormat && !hasExtractOptions)) &&
+ ((hasJsonFormat && hasJsonOptions) || (!hasJsonFormat && !hasJsonOptions))
+ );
+};
+const extractRefineOpts = {
+ message:
+ "When 'extract' or 'json' format is specified, corresponding options must be provided, and vice versa",
+};
+const extractTransform = (obj) => {
+ // Handle timeout
+ if (
+ (obj.formats?.includes("extract") ||
+ obj.extract ||
+ obj.formats?.includes("json") ||
+ obj.jsonOptions) &&
+ obj.timeout === 30000
+ ) {
+ obj = { ...obj, timeout: 60000 };
+ }
+
+ if (obj.formats?.includes("changeTracking") && (obj.waitFor === undefined || obj.waitFor < 5000)) {
+ obj = { ...obj, waitFor: 5000 };
+ }
+
+ if (obj.formats?.includes("changeTracking") && obj.timeout === 30000) {
+ obj = { ...obj, timeout: 60000 };
+ }
+
+ if (obj.agent) {
+ obj = { ...obj, timeout: 300000 };
+ }
+
+ if ((obj.proxy === "stealth" || obj.proxy === "auto") && obj.timeout === 30000) {
+ obj = { ...obj, timeout: 120000 };
+ }
+
+ if (obj.formats?.includes("json")) {
+ obj.formats.push("extract");
+ }
+
+ // Convert JSON options to extract options if needed
+ if (obj.jsonOptions && !obj.extract) {
+ obj = {
+ ...obj,
+ extract: {
+ prompt: obj.jsonOptions.prompt,
+ systemPrompt: obj.jsonOptions.systemPrompt,
+ schema: obj.jsonOptions.schema,
+ agent: obj.jsonOptions.agent,
+ mode: "llm",
+ },
+ };
+ }
+
+ return obj;
+};
+
+export const scrapeOptions = baseScrapeOptions
+ .extend({
+ agent: z
+ .object({
+ model: z.string().default(agentExtractModelValue),
+ prompt: z.string().optional(),
+ sessionId: z.string().optional(),
+ waitBeforeClosingMs: z.number().optional(),
+ })
+ .optional(),
+ extract: extractOptionsWithAgent.optional(),
+ jsonOptions: extractOptionsWithAgent.optional(),
+ })
+ .strict(strictMessage)
+ .refine(
+ (obj) => {
+ if (!obj.actions) return true;
+ return (
+ calculateTotalWaitTime(obj.actions, obj.waitFor) <=
+ ACTIONS_MAX_WAIT_TIME * 1000
+ );
+ },
+ {
+ message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,
+ },
+ )
+ .refine(extractRefine, extractRefineOpts)
+ .refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
+ .transform(extractTransform);
+
+export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;
+
+export type ScrapeOptions = BaseScrapeOptions & {
+ extract?: z.infer<typeof extractOptionsWithAgent>,
+ jsonOptions?: z.infer<typeof extractOptionsWithAgent>,
+ agent?: {
+ model: string,
+ prompt: string,
+ sessionId?: string,
+ waitBeforeClosingMs?: number,
+ },
+};
+
+import Ajv from "ajv";
+import type { CostTracking } from "../../lib/extract/extraction-service";
+
+const ajv = new Ajv();
+
+export const extractV1Options = z
+ .object({
+ urls: url
+ .array()
+ .max(10, "Maximum of 10 URLs allowed per request while in beta.")
+ .optional(),
+ prompt: z.string().max(10000).optional(),
+ systemPrompt: z.string().max(10000).optional(),
+ schema: z
+ .any()
+ .optional()
+ .refine(
+ (val) => {
+ if (!val) return true; // Allow undefined schema
+ try {
+ const validate = ajv.compile(val);
+ return typeof validate === "function";
+ } catch (e) {
+ return false;
+ }
+ },
+ {
+ message: "Invalid JSON schema.",
+ },
+ ),
+ limit: z.number().int().positive().finite().safe().optional(),
+ ignoreSitemap: z.boolean().default(false),
+ includeSubdomains: z.boolean().default(true),
+ allowExternalLinks: z.boolean().default(false),
+ enableWebSearch: z.boolean().default(false),
+ scrapeOptions: baseScrapeOptions.default({ onlyMainContent: false }).optional(),
+ origin: z.string().optional().default("api"),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
+ urlTrace: z.boolean().default(false),
+ timeout: z.number().int().positive().finite().safe().default(60000),
+ __experimental_streamSteps: z.boolean().default(false),
+ __experimental_llmUsage: z.boolean().default(false),
+ __experimental_showSources: z.boolean().default(false),
+ showSources: z.boolean().default(false),
+ __experimental_cacheKey: z.string().optional(),
+ __experimental_cacheMode: z
+ .enum(["direct", "save", "load"])
+ .default("direct")
+ .optional(),
+ agent: agentOptionsExtract.optional(),
+ __experimental_showCostTracking: z.boolean().default(false),
+ ignoreInvalidURLs: z.boolean().default(false),
+ })
+ .strict(strictMessage)
+ .refine((obj) => obj.urls || obj.prompt, {
+ message: "Either 'urls' or 'prompt' must be provided.",
+ })
+ .transform((obj) => ({
+ ...obj,
+ allowExternalLinks: obj.allowExternalLinks || obj.enableWebSearch,
+ }))
+ .refine(
+ (x) => (x.scrapeOptions ? extractRefine(x.scrapeOptions) : true),
+ extractRefineOpts,
+ )
+ .refine(
+ (x) => (x.scrapeOptions ? fire1Refine(x.scrapeOptions) : true),
+ fire1RefineOpts,
+ )
+ .refine(
+ (x) => (x.scrapeOptions ? waitForRefine(x.scrapeOptions) : true),
+ waitForRefineOpts,
+ )
+ .transform((x) => ({
+ ...x,
+ scrapeOptions: x.scrapeOptions
+ ? extractTransform(x.scrapeOptions)
+ : x.scrapeOptions,
+ }));
+
+export type ExtractV1Options = z.infer<typeof extractV1Options>;
+export const extractRequestSchema = extractV1Options;
+export type ExtractRequest = z.infer<typeof extractRequestSchema>;
+export type ExtractRequestInput = z.input<typeof extractRequestSchema>;
+
+export const scrapeRequestSchema = baseScrapeOptions
+ .omit({ timeout: true })
+ .extend({
+ url,
+ agent: z
+ .object({
+ model: z.string().default(agentExtractModelValue),
+ prompt: z.string().optional(),
+ sessionId: z.string().optional(),
+ waitBeforeClosingMs: z.number().optional(),
+ })
+ .optional(),
+ extract: extractOptionsWithAgent.optional(),
+ jsonOptions: extractOptionsWithAgent.optional(),
+ origin: z.string().optional().default("api"),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
+ timeout: z.number().int().positive().finite().safe().default(30000),
+ zeroDataRetention: z.boolean().optional(),
+ })
+ .strict(strictMessage)
+ .refine(extractRefine, extractRefineOpts)
+ .refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
+ .transform(extractTransform);
+
+export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
+export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;
+
+export const webhookSchema = z.preprocess(
+ (x) => {
+ if (typeof x === "string") {
+ return { url: x };
+ } else {
+ return x;
+ }
+ },
+ z
+ .object({
+ url: z.string().url(),
+ headers: z.record(z.string(), z.string()).default({}),
+ metadata: z.record(z.string(), z.string()).default({}),
+ events: z
+ .array(z.enum(["completed", "failed", "page", "started"]))
+ .default(["completed", "failed", "page", "started"]),
+ })
+ .strict(strictMessage),
+);
+
+export const batchScrapeRequestSchema = baseScrapeOptions
+ .extend({
+ urls: url.array(),
+ origin: z.string().optional().default("api"),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
+ webhook: webhookSchema.optional(),
+ appendToId: z.string().uuid().optional(),
+ ignoreInvalidURLs: z.boolean().default(false),
+ maxConcurrency: z.number().positive().int().optional(),
+ zeroDataRetention: z.boolean().optional(),
+ })
+ .strict(strictMessage)
+ .refine(extractRefine, extractRefineOpts)
+ .refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
+ .transform(extractTransform);
+
+export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
+ .extend({
+ urls: z.string().array(),
+ origin: z.string().optional().default("api"),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
+ webhook: webhookSchema.optional(),
+ appendToId: z.string().uuid().optional(),
+ ignoreInvalidURLs: z.boolean().default(false),
+ maxConcurrency: z.number().positive().int().optional(),
+ zeroDataRetention: z.boolean().optional(),
+ })
+ .strict(strictMessage)
+ .refine(extractRefine, extractRefineOpts)
+ .refine(fire1Refine, fire1RefineOpts)
+ .refine(waitForRefine, waitForRefineOpts)
+ .transform(extractTransform);
+
+export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;
+export type BatchScrapeRequestInput = z.input<typeof batchScrapeRequestSchema>;
+
+const crawlerOptions = z
+ .object({
+ includePaths: z.string().array().default([]),
+ excludePaths: z.string().array().default([]),
+ maxDepth: z.number().default(10), // default?
+ maxDiscoveryDepth: z.number().optional(),
+ limit: z.number().default(10000), // default?
+ allowBackwardLinks: z.boolean().default(false), // DEPRECATED: use crawlEntireDomain
+ crawlEntireDomain: z.boolean().optional(),
+ allowExternalLinks: z.boolean().default(false),
+ allowSubdomains: z.boolean().default(false),
+ ignoreRobotsTxt: z.boolean().default(false),
+ ignoreSitemap: z.boolean().default(false),
+ deduplicateSimilarURLs: z.boolean().default(true),
+ ignoreQueryParameters: z.boolean().default(false),
+ regexOnFullURL: z.boolean().default(false),
+ delay: z.number().positive().optional(),
+ })
+ .strict(strictMessage);
+
+// export type CrawlerOptions = {
+// includePaths?: string[];
+// excludePaths?: string[];
+// maxDepth?: number;
+// limit?: number;
+// allowBackwardLinks?: boolean; // DEPRECATED: use crawlEntireDomain
+// crawlEntireDomain?: boolean;
+// allowExternalLinks?: boolean;
+// ignoreSitemap?: boolean;
+// };
+
+export type CrawlerOptions = z.infer<typeof crawlerOptions>;
+
+export const crawlRequestSchema = crawlerOptions
+ .extend({
+ url,
+ origin: z.string().optional().default("api"),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
+ scrapeOptions: baseScrapeOptions.default({}),
+ webhook: webhookSchema.optional(),
+ limit: z.number().default(10000),
+ maxConcurrency: z.number().positive().int().optional(),
+ zeroDataRetention: z.boolean().optional(),
+ })
+ .strict(strictMessage)
+ .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)
+ .refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)
+ .refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)
+ .refine(
+ (data) => {
+ try {
+ const urlDepth = getURLDepth(data.url);
+ return urlDepth <= data.maxDepth;
+ } catch (e) {
+ return false;
+ }
+ },
+ {
+ message: "URL depth exceeds the specified maxDepth",
+ path: ["url"]
+ }
+ )
+ .transform((x) => {
+ if (x.crawlEntireDomain !== undefined) {
+ x.allowBackwardLinks = x.crawlEntireDomain;
+ }
+ return {
+ ...x,
+ scrapeOptions: extractTransform(x.scrapeOptions),
+ };
+ });
+
+// export type CrawlRequest = {
+// url: string;
+// crawlerOptions?: CrawlerOptions;
+// scrapeOptions?: Exclude<ScrapeRequest, "url">;
+// };
+
+// export type ExtractorOptions = {
+// mode: "markdown" | "llm-extraction" | "llm-extraction-from-markdown" | "llm-extraction-from-raw-html";
+// extractionPrompt?: string;
+// extractionSchema?: Record<string, any>;
+// }
+
+export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
+export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
+
+export const mapRequestSchema = crawlerOptions
+ .extend({
+ url,
+ origin: z.string().optional().default("api"),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
+ includeSubdomains: z.boolean().default(true),
+ search: z.string().optional(),
+ ignoreSitemap: z.boolean().default(false),
+ sitemapOnly: z.boolean().default(false),
+ limit: z.number().min(1).max(30000).default(5000),
+ timeout: z.number().positive().finite().optional(),
+ useMock: z.string().optional(),
+ filterByPath: z.boolean().default(true),
+ useIndex: z.boolean().default(true),
+ })
+ .strict(strictMessage);
+
+// export type MapRequest = {
+// url: string;
+// crawlerOptions?: CrawlerOptions;
+// };
+
+export type MapRequest = z.infer<typeof mapRequestSchema>;
+export type MapRequestInput = z.input<typeof mapRequestSchema>;
+
+export type Document = {
+ title?: string;
+ description?: string;
+ url?: string;
+ markdown?: string;
+ html?: string;
+ rawHtml?: string;
+ links?: string[];
+ screenshot?: string;
+ extract?: any;
+ json?: any;
+ warning?: string;
+ actions?: {
+ screenshots?: string[];
+ scrapes?: ScrapeActionContent[];
+ javascriptReturns?: {
+ type: string;
+ value: unknown;
+ }[];
+ pdfs?: string[];
+ };
+ changeTracking?: {
+ previousScrapeAt: string | null;
+ changeStatus: "new" | "same" | "changed" | "removed";
+ visibility: "visible" | "hidden";
+ diff?: {
+ text: string;
+ json: {
+ files: Array<{
+ from: string | null;
+ to: string | null;
+ chunks: Array<{
+ content: string;
+ changes: Array<{
+ type: string;
+ normal?: boolean;
+ ln?: number;
+ ln1?: number;
+ ln2?: number;
+ content: string;
+ }>;
+ }>;
+ }>;
+ };
+ };
+ json?: any;
+ }
+ metadata: {
+ title?: string;
+ description?: string;
+ language?: string;
+ keywords?: string;
+ robots?: string;
+ ogTitle?: string;
+ ogDescription?: string;
+ ogUrl?: string;
+ ogImage?: string;
+ ogAudio?: string;
+ ogDeterminer?: string;
+ ogLocale?: string;
+ ogLocaleAlternate?: string[];
+ ogSiteName?: string;
+ ogVideo?: string;
+ favicon?: string;
+ dcTermsCreated?: string;
+ dcDateCreated?: string;
+ dcDate?: string;
+ dcTermsType?: string;
+ dcType?: string;
+ dcTermsAudience?: string;
+ dcTermsSubject?: string;
+ dcSubject?: string;
+ dcDescription?: string;
+ dcTermsKeywords?: string;
+ modifiedTime?: string;
+ publishedTime?: string;
+ articleTag?: string;
+ articleSection?: string;
+ url?: string;
+ sourceURL?: string;
+ statusCode: number;
+ scrapeId?: string;
+ error?: string;
+ numPages?: number;
+ contentType?: string;
+ proxyUsed: "basic" | "stealth";
+ cacheState?: "hit" | "miss";
+ cachedAt?: string;
+ creditsUsed?: number;
+ // [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
+ };
+ serpResults?: {
+ title: string;
+ description: string;
+ url: string;
+ };
+};
+
+export type ErrorResponse = {
+ success: false;
+ error: string;
+ details?: any;
+};
+
+export type ScrapeResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ warning?: string;
+ data: Document;
+ scrape_id?: string;
+ };
+
+export interface ScrapeResponseRequestTest {
+ statusCode: number;
+ body: ScrapeResponse;
+ error?: string;
+}
+
+export interface URLTrace {
+ url: string;
+ status: "mapped" | "scraped" | "error";
+ timing: {
+ discoveredAt: string;
+ scrapedAt?: string;
+ completedAt?: string;
+ };
+ error?: string;
+ warning?: string;
+ contentStats?: {
+ rawContentLength: number;
+ processedContentLength: number;
+ tokensUsed: number;
+ };
+ relevanceScore?: number;
+ usedInCompletion?: boolean;
+ extractedFields?: string[];
+}
+
+export interface ExtractResponse {
+ success: boolean;
+ error?: string;
+ data?: any;
+ scrape_id?: string;
+ id?: string;
+ warning?: string;
+ urlTrace?: URLTrace[];
+ sources?: {
+ [key: string]: string[];
+ };
+ tokensUsed?: number;
+}
+
+export interface ExtractResponseRequestTest {
+ statusCode: number;
+ body: ExtractResponse;
+ error?: string;
+}
+
+export type CrawlResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ id: string;
+ url: string;
+ };
+
+export type BatchScrapeResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ id: string;
+ url: string;
+ invalidURLs?: string[];
+ };
+
+export type MapResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ links: string[];
+ scrape_id?: string;
+ };
+
+export type CrawlStatusParams = {
+ jobId: string;
+};
+
+export type ConcurrencyCheckParams = {
+ teamId: string;
+};
+
+export type ConcurrencyCheckResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ concurrency: number;
+ maxConcurrency: number;
+ };
+
+export type CrawlStatusResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ status: "scraping" | "completed" | "failed" | "cancelled";
+ completed: number;
+ total: number;
+ creditsUsed: number;
+ expiresAt: string;
+ next?: string;
+ data: Document[];
+ };
+
+export type OngoingCrawlsResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ crawls: {
+ id: string;
+ teamId: string;
+ url: string;
+ created_at: string;
+ options: CrawlerOptions;
+ }[];
+ };
+
+export type CrawlErrorsResponse =
+ | ErrorResponse
+ | {
+ errors: {
+ id: string;
+ timestamp?: string;
+ url: string;
+ error: string;
+ }[];
+ robotsBlocked: string[];
+ };
+
+type AuthObject = {
+ team_id: string;
+};
+
+type Account = {
+ remainingCredits: number;
+};
+
+export type AuthCreditUsageChunk = {
+ api_key: string;
+ team_id: string;
+ sub_id: string | null;
+ sub_current_period_start: string | null;
+ sub_current_period_end: string | null;
+ sub_user_id: string | null;
+ price_id: string | null;
+ price_credits: number; // credit limit with assoicated price, or free_credits (500) if free plan
+ credits_used: number;
+ coupon_credits: number; // do not rely on this number to be up to date after calling a billTeam
+ adjusted_credits_used: number; // credits this period minus coupons used
+ remaining_credits: number;
+ total_credits_sum: number;
+ plan_priority: {
+ bucketLimit: number;
+ planModifier: number;
+ };
+ rate_limits: {
+ crawl: number;
+ scrape: number;
+ search: number;
+ map: number;
+ extract: number;
+ preview: number;
+ crawlStatus: number;
+ extractStatus: number;
+ extractAgentPreview?: number;
+ scrapeAgentPreview?: number;
+ };
+ concurrency: number;
+ flags: TeamFlags;
+
+ // appended on JS-side
+ is_extract?: boolean;
+};
+
+export type TeamFlags = {
+ ignoreRobots?: boolean;
+ unblockedDomains?: string[];
+ forceZDR?: boolean;
+ allowZDR?: boolean;
+ zdrCost?: number;
+ checkRobotsOnScrape?: boolean;
+} | null;
+
+export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;
+
+export interface RequestWithMaybeACUC<
+ ReqParams = {},
+ ReqBody = undefined,
+ ResBody = undefined,
+> extends Request<ReqParams, ReqBody, ResBody> {
+ acuc?: AuthCreditUsageChunk;
+}
+
+export interface RequestWithACUC<
+ ReqParams = {},
+ ReqBody = undefined,
+ ResBody = undefined,
+> extends Request<ReqParams, ReqBody, ResBody> {
+ acuc: AuthCreditUsageChunk;
+}
+
+export interface RequestWithAuth<
+ ReqParams = {},
+ ReqBody = undefined,
+ ResBody = undefined,
+> extends Request<ReqParams, ReqBody, ResBody> {
+ auth: AuthObject;
+ account?: Account;
+}
+
+export interface RequestWithMaybeAuth<
+ ReqParams = {},
+ ReqBody = undefined,
+ ResBody = undefined,
+> extends RequestWithMaybeACUC<ReqParams, ReqBody, ResBody> {
+ auth?: AuthObject;
+ account?: Account;
+}
+
+export interface RequestWithAuth<
+ ReqParams = {},
+ ReqBody = undefined,
+ ResBody = undefined,
+> extends RequestWithACUC<ReqParams, ReqBody, ResBody> {
+ auth: AuthObject;
+ account?: Account;
+}
+
+export interface ResponseWithSentry<ResBody = undefined>
+ extends Response<ResBody> {
+ sentry?: string;
+}
+
+export function toLegacyCrawlerOptions(x: CrawlerOptions) {
+ return {
+ includes: x.includePaths,
+ excludes: x.excludePaths,
+ maxCrawledLinks: x.limit,
+ maxDepth: x.maxDepth,
+ limit: x.limit,
+ generateImgAltText: false,
+ allowBackwardCrawling: x.crawlEntireDomain ?? x.allowBackwardLinks,
+ allowExternalContentLinks: x.allowExternalLinks,
+ allowSubdomains: x.allowSubdomains,
+ ignoreRobotsTxt: x.ignoreRobotsTxt,
+ ignoreSitemap: x.ignoreSitemap,
+ deduplicateSimilarURLs: x.deduplicateSimilarURLs,
+ ignoreQueryParameters: x.ignoreQueryParameters,
+ regexOnFullURL: x.regexOnFullURL,
+ maxDiscoveryDepth: x.maxDiscoveryDepth,
+ currentDiscoveryDepth: 0,
+ delay: x.delay,
+ };
+}
+
+export function toNewCrawlerOptions(x: any): CrawlerOptions {
+ return {
+ includePaths: x.includes,
+ excludePaths: x.excludes,
+ limit: x.limit,
+ maxDepth: x.maxDepth,
+ allowBackwardLinks: x.allowBackwardCrawling,
+ crawlEntireDomain: x.allowBackwardCrawling,
+ allowExternalLinks: x.allowExternalContentLinks,
+ allowSubdomains: x.allowSubdomains,
+ ignoreRobotsTxt: x.ignoreRobotsTxt,
+ ignoreSitemap: x.ignoreSitemap,
+ deduplicateSimilarURLs: x.deduplicateSimilarURLs,
+ ignoreQueryParameters: x.ignoreQueryParameters,
+ regexOnFullURL: x.regexOnFullURL,
+ maxDiscoveryDepth: x.maxDiscoveryDepth,
+ delay: x.delay,
+ }
+}
+
+export function fromLegacyCrawlerOptions(x: any, teamId: string): {
+ crawlOptions: CrawlerOptions;
+ internalOptions: InternalOptions;
+} {
+ return {
+ crawlOptions: crawlerOptions.parse({
+ includePaths: x.includes,
+ excludePaths: x.excludes,
+ limit: x.maxCrawledLinks ?? x.limit,
+ maxDepth: x.maxDepth,
+ allowBackwardLinks: x.allowBackwardCrawling,
+ crawlEntireDomain: x.allowBackwardCrawling,
+ allowExternalLinks: x.allowExternalContentLinks,
+ allowSubdomains: x.allowSubdomains,
+ ignoreRobotsTxt: x.ignoreRobotsTxt,
+ ignoreSitemap: x.ignoreSitemap,
+ deduplicateSimilarURLs: x.deduplicateSimilarURLs,
+ ignoreQueryParameters: x.ignoreQueryParameters,
+ regexOnFullURL: x.regexOnFullURL,
+ maxDiscoveryDepth: x.maxDiscoveryDepth,
+ delay: x.delay,
+ }),
+ internalOptions: {
+ v0CrawlOnlyUrls: x.returnOnlyUrls,
+ teamId,
+ },
+ };
+}
+
+export interface MapDocument {
+ url: string;
+ title?: string;
+ description?: string;
+}
+export function fromLegacyScrapeOptions(
+ pageOptions: PageOptions,
+ extractorOptions: ExtractorOptions | undefined,
+ timeout: number | undefined,
+ teamId: string,
+): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
+ return {
+ scrapeOptions: scrapeOptions.parse({
+ formats: [
+ (pageOptions.includeMarkdown ?? true) ? ("markdown" as const) : null,
+ (pageOptions.includeHtml ?? false) ? ("html" as const) : null,
+ (pageOptions.includeRawHtml ?? false) ? ("rawHtml" as const) : null,
+ (pageOptions.screenshot ?? false) ? ("screenshot" as const) : null,
+ (pageOptions.fullPageScreenshot ?? false)
+ ? ("screenshot@fullPage" as const)
+ : null,
+ extractorOptions !== undefined &&
+ extractorOptions.mode.includes("llm-extraction")
+ ? ("extract" as const)
+ : null,
+ "links",
+ ].filter((x) => x !== null),
+ waitFor: pageOptions.waitFor,
+ headers: pageOptions.headers,
+ includeTags:
+ typeof pageOptions.onlyIncludeTags === "string"
+ ? [pageOptions.onlyIncludeTags]
+ : pageOptions.onlyIncludeTags,
+ excludeTags:
+ typeof pageOptions.removeTags === "string"
+ ? [pageOptions.removeTags]
+ : pageOptions.removeTags,
+ onlyMainContent: pageOptions.onlyMainContent ?? false,
+ timeout: timeout,
+ parsePDF: pageOptions.parsePDF,
+ actions: pageOptions.actions,
+ location: pageOptions.geolocation,
+ skipTlsVerification: pageOptions.skipTlsVerification,
+ removeBase64Images: pageOptions.removeBase64Images,
+ extract:
+ extractorOptions !== undefined &&
+ extractorOptions.mode.includes("llm-extraction")
+ ? {
+ systemPrompt: extractorOptions.extractionPrompt,
+ prompt: extractorOptions.userPrompt,
+ schema: extractorOptions.extractionSchema,
+ }
+ : undefined,
+ mobile: pageOptions.mobile,
+ fastMode: pageOptions.useFastMode,
+ }),
+ internalOptions: {
+ atsv: pageOptions.atsv,
+ v0DisableJsDom: pageOptions.disableJsDom,
+ teamId,
+ },
+ // TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
+ };
+}
+
+export function fromLegacyCombo(
+ pageOptions: PageOptions,
+ extractorOptions: ExtractorOptions | undefined,
+ timeout: number | undefined,
+ crawlerOptions: any,
+ teamId: string,
+): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
+ const { scrapeOptions, internalOptions: i1 } = fromLegacyScrapeOptions(
+ pageOptions,
+ extractorOptions,
+ timeout,
+ teamId,
+ );
+ const { internalOptions: i2 } = fromLegacyCrawlerOptions(crawlerOptions, teamId);
+ return { scrapeOptions, internalOptions: Object.assign(i1, i2) };
+}
+
+export function toLegacyDocument(
+ document: Document,
+ internalOptions: InternalOptions,
+): V0Document | { url: string } {
+ if (internalOptions.v0CrawlOnlyUrls) {
+ return { url: document.metadata.sourceURL! };
+ }
+
+ return {
+ content: document.markdown!,
+ markdown: document.markdown!,
+ html: document.html,
+ rawHtml: document.rawHtml,
+ linksOnPage: document.links,
+ llm_extraction: document.extract,
+ metadata: {
+ ...document.metadata,
+ error: undefined,
+ statusCode: undefined,
+ pageError: document.metadata.error,
+ pageStatusCode: document.metadata.statusCode,
+ screenshot: document.screenshot,
+ },
+ actions: document.actions,
+ warning: document.warning,
+ };
+}
+
+export const searchRequestSchema = z
+ .object({
+ query: z.string(),
+ limit: z
+ .number()
+ .int()
+ .positive()
+ .finite()
+ .safe()
+ .max(100)
+ .optional()
+ .default(5),
+ tbs: z.string().optional(),
+ filter: z.string().optional(),
+ lang: z.string().optional().default("en"),
+ country: z.string().optional().default("us"),
+ location: z.string().optional(),
+ origin: z.string().optional().default("api"),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
+ timeout: z.number().int().positive().finite().safe().default(60000),
+ ignoreInvalidURLs: z.boolean().optional().default(false),
+ __searchPreviewToken: z.string().optional(),
+ scrapeOptions: baseScrapeOptions
+ .extend({
+ formats: z
+ .array(
+ z.enum([
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ "screenshot",
+ "screenshot@fullPage",
+ "extract",
+ "json",
+ ]),
+ )
+ .default([]),
+ })
+ .default({}),
+ })
+ .strict(
+ "Unrecognized key in body -- please review the v1 API documentation for request body changes",
+ )
+ .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)
+ .refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)
+ .refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)
+ .transform((x) => ({
+ ...x,
+ scrapeOptions: extractTransform(x.scrapeOptions),
+ }));
+
+export type SearchRequest = z.infer<typeof searchRequestSchema>;
+export type SearchRequestInput = z.input<typeof searchRequestSchema>;
+
+export type SearchResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ warning?: string;
+ data: Document[];
+ };
+
+export type TokenUsage = {
+ promptTokens: number;
+ completionTokens: number;
+ totalTokens: number;
+ step?: string;
+ model?: string;
+};
+
+export const generateLLMsTextRequestSchema = z.object({
+ url: url.describe("The URL to generate text from"),
+ maxUrls: z
+ .number()
+ .min(1)
+ .max(5000)
+ .default(10)
+ .describe("Maximum number of URLs to process"),
+ showFullText: z
+ .boolean()
+ .default(false)
+ .describe("Whether to show the full LLMs-full.txt in the response"),
+ cache: z
+ .boolean()
+ .default(true)
+ .describe("Whether to use cached content if available"),
+ __experimental_stream: z.boolean().optional(),
+});
+
+export type GenerateLLMsTextRequest = z.infer<
+ typeof generateLLMsTextRequestSchema
+>;
+
+export class TimeoutSignal extends Error {
+ constructor() {
+ super("Operation timed out");
+ }
+}
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 613b09eb5..bcfc11058 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -26,6 +26,7 @@ import { v4 as uuidv4 } from "uuid";
import { RateLimiterMode } from "./types";
import { attachWsProxy } from "./services/agentLivecastWS";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
+import { v2Router } from "./routes/v2";
const { createBullBoard } = require("@bull-board/api");
const { BullAdapter } = require("@bull-board/api/bullAdapter");
@@ -82,6 +83,7 @@ app.get("/test", async (req, res) => {
// register router
app.use(v0Router);
app.use("/v1", v1Router);
+app.use("/v2", v2Router);
app.use(adminRouter);
const DEFAULT_PORT = process.env.PORT ?? 3002;
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index fc8a3f8dc..a8de40fd4 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -6,7 +6,7 @@ import {
cleanBefore24hCompleteJobsController,
queuesController,
} from "../controllers/v0/admin/queue";
-import { wrap } from "./v1";
+import { wrap } from "./shared";
import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
import { cclogController } from "../controllers/v0/admin/cclog";
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
new file mode 100644
index 000000000..2ceb7317e
--- /dev/null
+++ b/apps/api/src/routes/shared.ts
@@ -0,0 +1,174 @@
+import { NextFunction, Request, Response } from "express";
+// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";
+import {
+ isAgentExtractModelValid,
+ RequestWithACUC,
+ RequestWithAuth,
+ RequestWithMaybeAuth,
+} from "../controllers/v1/types";
+import { RateLimiterMode } from "../types";
+import { authenticateUser } from "../controllers/auth";
+import { createIdempotencyKey } from "../services/idempotency/create";
+import { validateIdempotencyKey } from "../services/idempotency/validate";
+import { checkTeamCredits } from "../services/billing/credit_billing";
+import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
+import { logger } from "../lib/logger";
+import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
+
+export function checkCreditsMiddleware(
+ _minimum?: number,
+): (req: RequestWithAuth, res: Response, next: NextFunction) => void {
+ return (req, res, next) => {
+ let minimum = _minimum;
+ (async () => {
+ if (!minimum && req.body) {
+ minimum =
+ (req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1;
+ }
+ const { success, remainingCredits, chunk } = await checkTeamCredits(
+ req.acuc,
+ req.auth.team_id,
+ minimum ?? 1,
+ );
+ if (chunk) {
+ req.acuc = chunk;
+ }
+ req.account = { remainingCredits };
+ if (!success) {
+ if (
+ !minimum &&
+ req.body &&
+ (req.body as any).limit !== undefined &&
+ remainingCredits > 0
+ ) {
+ logger.warn("Adjusting limit to remaining credits", {
+ teamId: req.auth.team_id,
+ remainingCredits,
+ request: req.body,
+ });
+ (req.body as any).limit = remainingCredits;
+ return next();
+ }
+
+ const currencyName = req.acuc.is_extract ? "tokens" : "credits";
+ logger.error(
+ `Insufficient ${currencyName}: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`,
+ {
+ teamId: req.auth.team_id,
+ minimum,
+ remainingCredits,
+ request: req.body,
+ path: req.path,
+ },
+ );
+ if (
+ !res.headersSent &&
+ req.auth.team_id !== "8c528896-7882-4587-a4b6-768b721b0b53"
+ ) {
+ return res.status(402).json({
+ success: false,
+ error:
+ "Insufficient " +
+ currencyName +
+ " to perform this request. For more " +
+ currencyName +
+ ", you can upgrade your plan at " +
+ (currencyName === "credits"
+ ? "https://firecrawl.dev/pricing or try changing the request limit to a lower value"
+ : "https://www.firecrawl.dev/extract#pricing") +
+ ".",
+ });
+ }
+ }
+ next();
+ })().catch((err) => next(err));
+ };
+}
+
+export function authMiddleware(
+ rateLimiterMode: RateLimiterMode,
+): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {
+ return (req, res, next) => {
+ (async () => {
+ let currentRateLimiterMode = rateLimiterMode;
+ if (
+ currentRateLimiterMode === RateLimiterMode.Extract &&
+ isAgentExtractModelValid((req.body as any)?.agent?.model)
+ ) {
+ currentRateLimiterMode = RateLimiterMode.ExtractAgentPreview;
+ }
+
+ // if (currentRateLimiterMode === RateLimiterMode.Scrape && isAgentExtractModelValid((req.body as any)?.agent?.model)) {
+ // currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;
+ // }
+
+ const auth = await authenticateUser(req, res, currentRateLimiterMode);
+
+ if (!auth.success) {
+ if (!res.headersSent) {
+ return res
+ .status(auth.status)
+ .json({ success: false, error: auth.error });
+ } else {
+ return;
+ }
+ }
+
+ const { team_id, chunk } = auth;
+
+ req.auth = { team_id };
+ req.acuc = chunk ?? undefined;
+ if (chunk) {
+ req.account = { remainingCredits: chunk.remaining_credits };
+ }
+ next();
+ })().catch((err) => next(err));
+ };
+}
+
+export function idempotencyMiddleware(
+ req: Request,
+ res: Response,
+ next: NextFunction,
+) {
+ (async () => {
+ if (req.headers["x-idempotency-key"]) {
+ const isIdempotencyValid = await validateIdempotencyKey(req);
+ if (!isIdempotencyValid) {
+ if (!res.headersSent) {
+ return res
+ .status(409)
+ .json({ success: false, error: "Idempotency key already used" });
+ }
+ }
+ createIdempotencyKey(req);
+ }
+ next();
+ })().catch((err) => next(err));
+}
+export function blocklistMiddleware(
+ req: RequestWithACUC<any, any, any>,
+ res: Response,
+ next: NextFunction,
+) {
+ if (
+ typeof req.body.url === "string" &&
+ isUrlBlocked(req.body.url, req.acuc?.flags ?? null)
+ ) {
+ if (!res.headersSent) {
+ return res.status(403).json({
+ success: false,
+ error: BLOCKLISTED_URL_MESSAGE,
+ });
+ }
+ }
+ next();
+}
+
+export function wrap(
+ controller: (req: Request, res: Response) => Promise<any>,
+): (req: Request, res: Response, next: NextFunction) => any {
+ return (req, res, next) => {
+ controller(req, res).catch((err) => next(err));
+ };
+}
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index cfb98cbe7..583ba4a5f 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -1,33 +1,20 @@
-import express, { NextFunction, Request, Response } from "express";
+import express from "express";
import { crawlController } from "../controllers/v1/crawl";
// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";
import { scrapeController } from "../../src/controllers/v1/scrape";
import { crawlStatusController } from "../controllers/v1/crawl-status";
import { mapController } from "../controllers/v1/map";
-import {
- ErrorResponse,
- isAgentExtractModelValid,
- RequestWithACUC,
- RequestWithAuth,
- RequestWithMaybeAuth,
-} from "../controllers/v1/types";
+import { RequestWithAuth } from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
-import { authenticateUser } from "../controllers/auth";
-import { createIdempotencyKey } from "../services/idempotency/create";
-import { validateIdempotencyKey } from "../services/idempotency/validate";
-import { checkTeamCredits } from "../services/billing/credit_billing";
import expressWs from "express-ws";
import { crawlStatusWSController } from "../controllers/v1/crawl-status-ws";
-import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
import { crawlCancelController } from "../controllers/v1/crawl-cancel";
-import { logger } from "../lib/logger";
import { scrapeStatusController } from "../controllers/v1/scrape-status";
import { concurrencyCheckController } from "../controllers/v1/concurrency-check";
import { batchScrapeController } from "../controllers/v1/batch-scrape";
import { extractController } from "../controllers/v1/extract";
import { extractStatusController } from "../controllers/v1/extract-status";
import { creditUsageController } from "../controllers/v1/credit-usage";
-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
import { searchController } from "../controllers/v1/search";
import { crawlErrorsController } from "../controllers/v1/crawl-errors";
import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";
@@ -36,139 +23,13 @@ import { deepResearchController } from "../controllers/v1/deep-research";
import { deepResearchStatusController } from "../controllers/v1/deep-research-status";
import { tokenUsageController } from "../controllers/v1/token-usage";
import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";
-
-function checkCreditsMiddleware(
- _minimum?: number,
-): (req: RequestWithAuth, res: Response, next: NextFunction) => void {
- return (req, res, next) => {
- let minimum = _minimum;
- (async () => {
- if (!minimum && req.body) {
- minimum =
- (req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1;
- }
- const { success, remainingCredits, chunk } = await checkTeamCredits(
- req.acuc,
- req.auth.team_id,
- minimum ?? 1,
- );
- if (chunk) {
- req.acuc = chunk;
- }
- req.account = { remainingCredits };
- if (!success) {
- if (!minimum && req.body && (req.body as any).limit !== undefined && remainingCredits > 0) {
- logger.warn("Adjusting limit to remaining credits", {
- teamId: req.auth.team_id,
- remainingCredits,
- request: req.body,
- });
- (req.body as any).limit = remainingCredits;
- return next();
- }
-
- const currencyName = req.acuc.is_extract ? "tokens" : "credits"
- logger.error(
- `Insufficient ${currencyName}: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`,
- {
- teamId: req.auth.team_id,
- minimum,
- remainingCredits,
- request: req.body,
- path: req.path
- }
- );
- if (!res.headersSent && req.auth.team_id !== "8c528896-7882-4587-a4b6-768b721b0b53") {
- return res.status(402).json({
- success: false,
- error:
- "Insufficient " + currencyName + " to perform this request. For more " + currencyName + ", you can upgrade your plan at " + (currencyName === "credits" ? "https://firecrawl.dev/pricing or try changing the request limit to a lower value" : "https://www.firecrawl.dev/extract#pricing") + ".",
- });
- }
- }
- next();
- })().catch((err) => next(err));
- };
-}
-
-export function authMiddleware(
- rateLimiterMode: RateLimiterMode
-): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {
- return (req, res, next) => {
- (async () => {
- let currentRateLimiterMode = rateLimiterMode;
- if (currentRateLimiterMode === RateLimiterMode.Extract && isAgentExtractModelValid((req.body as any)?.agent?.model)) {
- currentRateLimiterMode = RateLimiterMode.ExtractAgentPreview;
- }
-
- // if (currentRateLimiterMode === RateLimiterMode.Scrape && isAgentExtractModelValid((req.body as any)?.agent?.model)) {
- // currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;
- // }
-
- const auth = await authenticateUser(req, res, currentRateLimiterMode);
-
- if (!auth.success) {
- if (!res.headersSent) {
- return res
- .status(auth.status)
- .json({ success: false, error: auth.error });
- } else {
- return;
- }
- }
-
- const { team_id, chunk } = auth;
-
- req.auth = { team_id };
- req.acuc = chunk ?? undefined;
- if (chunk) {
- req.account = { remainingCredits: chunk.remaining_credits };
- }
- next();
- })().catch((err) => next(err));
- };
-}
-
-function idempotencyMiddleware(
- req: Request,
- res: Response,
- next: NextFunction,
-) {
- (async () => {
- if (req.headers["x-idempotency-key"]) {
- const isIdempotencyValid = await validateIdempotencyKey(req);
- if (!isIdempotencyValid) {
- if (!res.headersSent) {
- return res
- .status(409)
- .json({ success: false, error: "Idempotency key already used" });
- }
- }
- createIdempotencyKey(req);
- }
- next();
- })().catch((err) => next(err));
-}
-
-function blocklistMiddleware(req: RequestWithACUC<any, any, any>, res: Response, next: NextFunction) {
- if (typeof req.body.url === "string" && isUrlBlocked(req.body.url, req.acuc?.flags ?? null)) {
- if (!res.headersSent) {
- return res.status(403).json({
- success: false,
- error: BLOCKLISTED_URL_MESSAGE,
- });
- }
- }
- next();
-}
-
-export function wrap(
- controller: (req: Request, res: Response) => Promise<any>,
-): (req: Request, res: Response, next: NextFunction) => any {
- return (req, res, next) => {
- controller(req, res).catch((err) => next(err));
- };
-}
+import {
+ authMiddleware,
+ checkCreditsMiddleware,
+ blocklistMiddleware,
+ idempotencyMiddleware,
+ wrap,
+} from "./shared";
expressWs(express());
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
new file mode 100644
index 000000000..4ff9f3f70
--- /dev/null
+++ b/apps/api/src/routes/v2.ts
@@ -0,0 +1,43 @@
+import express from "express";
+import { crawlController } from "../controllers/v1/crawl";
+// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";
+import { scrapeController } from "../../src/controllers/v1/scrape";
+import { crawlStatusController } from "../controllers/v1/crawl-status";
+import { mapController } from "../controllers/v1/map";
+import { RequestWithAuth } from "../controllers/v1/types";
+import { RateLimiterMode } from "../types";
+import expressWs from "express-ws";
+import { crawlStatusWSController } from "../controllers/v1/crawl-status-ws";
+import { crawlCancelController } from "../controllers/v1/crawl-cancel";
+import { scrapeStatusController } from "../controllers/v1/scrape-status";
+import { concurrencyCheckController } from "../controllers/v1/concurrency-check";
+import { batchScrapeController } from "../controllers/v1/batch-scrape";
+import { extractController } from "../controllers/v1/extract";
+import { extractStatusController } from "../controllers/v1/extract-status";
+import { creditUsageController } from "../controllers/v1/credit-usage";
+import { searchController } from "../controllers/v1/search";
+import { crawlErrorsController } from "../controllers/v1/crawl-errors";
+import { generateLLMsTextController } from "../controllers/v1/generate-llmstxt";
+import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmstxt-status";
+import { deepResearchController } from "../controllers/v1/deep-research";
+import { deepResearchStatusController } from "../controllers/v1/deep-research-status";
+import { tokenUsageController } from "../controllers/v1/token-usage";
+import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";
+import {
+ authMiddleware,
+ checkCreditsMiddleware,
+ blocklistMiddleware,
+ idempotencyMiddleware,
+ wrap,
+} from "./shared";
+
+expressWs(express());
+
+export const v2Router = express.Router();
+
+v2Router.post(
+ "/search",
+ authMiddleware(RateLimiterMode.Search),
+ checkCreditsMiddleware(),
+ wrap(searchController),
+);