Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: implement v2 batch scrape with improved error handling (#1851)

devin-ai-integration[bot]b4e0866bdc8b
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
new file mode 100644
index 000000000..a200f2e77
--- /dev/null
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -0,0 +1,203 @@
+import { Response } from "express";
+import { v4 as uuidv4 } from "uuid";
+import {
+ BatchScrapeRequest,
+ batchScrapeRequestSchema,
+ batchScrapeRequestSchemaNoURLValidation,
+ url as urlSchema,
+ RequestWithAuth,
+ ScrapeOptions,
+ BatchScrapeResponse,
+} from "./types";
+import {
+ addCrawlJobs,
+ finishCrawlKickoff,
+ getCrawl,
+ lockURLs,
+ saveCrawl,
+ StoredCrawl,
+} from "../../lib/crawl-redis";
+import { logCrawl } from "../../services/logging/crawl_log";
+import { getJobPriority } from "../../lib/job-priority";
+import { addScrapeJobs } from "../../services/queue-jobs";
+import { callWebhook } from "../../services/webhook";
+import { logger as _logger } from "../../lib/logger";
+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+
+export async function batchScrapeController(
+ req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
+ res: Response<BatchScrapeResponse>,
+) {
+ const preNormalizedBody = { ...req.body };
+
+ if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {
+ return res.status(400).json({
+ success: false,
+ error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",
+ });
+ }
+
+ const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
+
+ if (req.body?.ignoreInvalidURLs === true) {
+ req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);
+ } else {
+ req.body = batchScrapeRequestSchema.parse(req.body);
+ }
+
+ const id = req.body.appendToId ?? uuidv4();
+ const logger = _logger.child({
+ crawlId: id,
+ batchScrapeId: id,
+ module: "api/v2",
+ method: "batchScrapeController",
+ teamId: req.auth.team_id,
+ zeroDataRetention,
+ });
+
+ let urls: string[] = req.body.urls;
+ let unnormalizedURLs = preNormalizedBody.urls;
+ let invalidURLs: string[] | undefined = undefined;
+
+ if (req.body.ignoreInvalidURLs) {
+ invalidURLs = [];
+
+ let pendingURLs = urls;
+ urls = [];
+ unnormalizedURLs = [];
+ for (const u of pendingURLs) {
+ try {
+ const nu = urlSchema.parse(u);
+ if (!isUrlBlocked(nu, req.acuc?.flags ?? null)) {
+ urls.push(nu);
+ unnormalizedURLs.push(u);
+ } else {
+ invalidURLs.push(u);
+ }
+ } catch (_) {
+ invalidURLs.push(u);
+ }
+ }
+ } else {
+ if (req.body.urls?.some((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null))) {
+ if (!res.headersSent) {
+ return res.status(403).json({
+ success: false,
+ error: BLOCKLISTED_URL_MESSAGE,
+ });
+ }
+ }
+ }
+
+ logger.debug("Batch scrape " + id + " starting", {
+ urlsLength: urls.length,
+ appendToId: req.body.appendToId,
+ account: req.account,
+ });
+
+ if (!req.body.appendToId) {
+ await logCrawl(id, req.auth.team_id);
+ }
+
+ const sc: StoredCrawl = req.body.appendToId
+ ? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
+ : {
+ crawlerOptions: null,
+ scrapeOptions: req.body,
+ internalOptions: {
+ disableSmartWaitCache: true,
+ teamId: req.auth.team_id,
+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
+ zeroDataRetention,
+ }, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
+ team_id: req.auth.team_id,
+ createdAt: Date.now(),
+ maxConcurrency: req.body.maxConcurrency,
+ zeroDataRetention,
+ };
+
+ if (!req.body.appendToId) {
+ await saveCrawl(id, sc);
+ }
+
+ let jobPriority = 20;
+
+ // If it is over 1000, we need to get the job priority,
+ // otherwise we can use the default priority of 20
+ if (urls.length > 1000) {
+ // set base to 21
+ jobPriority = await getJobPriority({
+ team_id: req.auth.team_id,
+ basePriority: 21,
+ });
+ }
+ logger.debug("Using job priority " + jobPriority, { jobPriority });
+
+ const scrapeOptions: ScrapeOptions = { ...req.body };
+ delete (scrapeOptions as any).urls;
+ delete (scrapeOptions as any).appendToId;
+
+ const jobs = urls.map(x => ({
+ data: {
+ url: x,
+ mode: "single_urls" as const,
+ team_id: req.auth.team_id,
+ crawlerOptions: null,
+ scrapeOptions,
+ origin: "api",
+ integration: req.body.integration,
+ crawl_id: id,
+ sitemapped: true,
+ v1: false,
+ webhook: req.body.webhook,
+ internalOptions: sc.internalOptions,
+ zeroDataRetention,
+ },
+ opts: {
+ jobId: uuidv4(),
+ priority: 20,
+ },
+ }));
+
+ await finishCrawlKickoff(id);
+
+ logger.debug("Locking URLs...");
+ await lockURLs(
+ id,
+ sc,
+ jobs.map((x) => x.data.url),
+ logger,
+ );
+ logger.debug("Adding scrape jobs to Redis...");
+ await addCrawlJobs(
+ id,
+ jobs.map((x) => x.opts.jobId),
+ logger,
+ );
+ logger.debug("Adding scrape jobs to BullMQ...");
+ await addScrapeJobs(jobs);
+
+ if (req.body.webhook) {
+ logger.debug("Calling webhook with batch_scrape.started...", {
+ webhook: req.body.webhook,
+ });
+ await callWebhook({
+ teamId: req.auth.team_id,
+ crawlId: id,
+ data: null,
+ webhook: req.body.webhook,
+ v1: false,
+ eventType: "batch_scrape.started",
+ });
+ }
+
+ const protocol = process.env.ENV === "local" ? req.protocol : "https";
+
+ return res.status(200).json({
+ success: true,
+ id,
+ url: `${protocol}://${req.get("host")}/v2/batch/scrape/${id}`,
+ invalidURLs,
+ });
+}
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
new file mode 100644
index 000000000..94729506a
--- /dev/null
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -0,0 +1,449 @@
+import { Response } from "express";
+import {
+ CrawlStatusParams,
+ CrawlStatusResponse,
+ ErrorResponse,
+ RequestWithAuth,
+} from "./types";
+import {
+ getCrawl,
+ getCrawlExpiry,
+ getCrawlJobs,
+ getDoneJobsOrdered,
+ getDoneJobsOrderedLength,
+ isCrawlKickoffFinished,
+} from "../../lib/crawl-redis";
+import { getScrapeQueue } from "../../services/queue-service";
+import {
+ supabaseGetJobById,
+ supabaseGetJobsById,
+} from "../../lib/supabase-jobs";
+import { configDotenv } from "dotenv";
+import type { Job, JobState, Queue } from "bullmq";
+import { logger } from "../../lib/logger";
+import { supabase_rr_service, supabase_service } from "../../services/supabase";
+import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";
+import { getJobFromGCS } from "../../lib/gcs-jobs";
+configDotenv();
+
+export type PseudoJob<T> = {
+ id: string,
+ getState(): Promise<JobState | "unknown"> | JobState | "unknown",
+ returnvalue: T | null,
+ timestamp: number,
+ data: {
+ scrapeOptions: any,
+ teamId?: string,
+ },
+ failedReason?: string,
+}
+
+export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string }
+
+export async function getJob(id: string): Promise<PseudoJob<any> | null> {
+ const [bullJob, dbJob, gcsJob] = await Promise.all([
+ getScrapeQueue().getJob(id),
+ (process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
+ (process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
+ ]);
+
+ if (!bullJob && !dbJob) return null;
+
+ const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;
+ if (gcsJob === null && data) {
+ logger.warn("GCS Job not found", {
+ jobId: id,
+ });
+ }
+
+ const job: PseudoJob<any> = {
+ id,
+ getState: bullJob ? bullJob.getState : (() => dbJob!.success ? "completed" : "failed"),
+ returnvalue: Array.isArray(data)
+ ? data[0]
+ : data,
+ data: {
+ scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
+ },
+ timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),
+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
+ }
+
+ return job;
+}
+
+export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
+ const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],
+ process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
+ ]);
+
+ const bullJobMap = new Map<string, PseudoJob<any>>();
+ const dbJobMap = new Map<string, DBJob>();
+ const gcsJobMap = new Map<string, any>();
+
+ for (const job of bullJobs) {
+ bullJobMap.set(job.id, job);
+ }
+
+ for (const job of dbJobs) {
+ dbJobMap.set(job.job_id, job);
+ }
+
+ for (const job of gcsJobs) {
+ gcsJobMap.set(job.id, job.job);
+ }
+
+ const jobs: PseudoJob<any>[] = [];
+
+ for (const id of ids) {
+ const bullJob = bullJobMap.get(id);
+ const dbJob = dbJobMap.get(id);
+ const gcsJob = gcsJobMap.get(id);
+
+ if (!bullJob && !dbJob) continue;
+
+ const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;
+ if (gcsJob === null && data) {
+ logger.warn("GCS Job not found", {
+ jobId: id,
+ });
+ }
+
+ const job: PseudoJob<any> = {
+ id,
+ getState: bullJob ? (() => bullJob.getState()) : (() => dbJob!.success ? "completed" : "failed"),
+ returnvalue: Array.isArray(data)
+ ? data[0]
+ : data,
+ data: {
+ scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
+ },
+ timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),
+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
+ }
+
+ jobs.push(job);
+ }
+
+ return jobs;
+}
+
+export async function crawlStatusController(
+ req: RequestWithAuth<CrawlStatusParams, undefined, CrawlStatusResponse>,
+ res: Response<CrawlStatusResponse>,
+ isBatch = false,
+) {
+ const start =
+ typeof req.query.skip === "string" ? parseInt(req.query.skip, 10) : 0;
+ const end =
+ typeof req.query.limit === "string"
+ ? start + parseInt(req.query.limit, 10) - 1
+ : undefined;
+
+ const sc = await getCrawl(req.params.jobId);
+
+ let status: Exclude<CrawlStatusResponse, ErrorResponse>["status"];
+ let doneJobsLength: number;
+ let doneJobsOrder: string[];
+ let totalCount: number;
+ let creditsUsed: number;
+
+ if (sc) {
+ if (sc.team_id !== req.auth.team_id) {
+ return res.status(403).json({ success: false, error: "Forbidden" });
+ }
+
+ let jobIDs = await getCrawlJobs(req.params.jobId);
+ let jobStatuses = await Promise.all(
+ jobIDs.map(
+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
+ ),
+ );
+
+ if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {
+ for (let rangeStart = 0; ; rangeStart += 1000) {
+ const rangeEnd = rangeStart + 1000;
+
+ const currentJobs = await supabase_service
+ .from("firecrawl_jobs")
+ .select("success, job_id")
+ .eq("crawl_id", req.params.jobId)
+ .order("date_added", { ascending: true })
+ .range(rangeStart, rangeEnd);
+
+ const rangeLength = rangeEnd - rangeStart;
+
+ if (currentJobs.error) {
+ logger.error("Error getting current jobs", { error: currentJobs.error });
+ continue;
+ }
+
+ const data = currentJobs.data ?? [];
+
+ for (const job of data) {
+ let index = jobStatuses.findIndex((x) => x[0] === job.job_id);
+ if (index !== -1) {
+ jobStatuses[index] = [job.job_id, job.success ? "completed" : "failed"];
+ } else {
+ jobStatuses.push([job.job_id, job.success ? "completed" : "failed"]);
+ }
+ }
+
+ if (data.length < rangeLength) {
+ break;
+ }
+ }
+ }
+
+ const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id));
+ const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId));
+
+ const validJobStatuses: [string, JobState | "unknown"][] = [];
+ const validJobIDs: string[] = [];
+
+ for (const [id, status] of jobStatuses) {
+ if (throttledJobsSet.has(id)) {
+ validJobStatuses.push([id, "prioritized"]);
+ validJobIDs.push(id);
+ } else if (status === "unknown" && activeJobsSet.has(id)) {
+ validJobStatuses.push([id, "active"]);
+ validJobIDs.push(id);
+ } else if (
+ status !== "failed" &&
+ status !== "unknown"
+ ) {
+ validJobStatuses.push([id, status]);
+ validJobIDs.push(id);
+ }
+ }
+
+ status =
+ sc.cancelled
+ ? "cancelled"
+ : validJobStatuses.every((x) => x[1] === "completed") &&
+ (sc.crawlerOptions
+ ? await isCrawlKickoffFinished(req.params.jobId)
+ : true)
+ ? "completed"
+ : "scraping";
+
+ // Use validJobIDs instead of jobIDs for further processing
+ jobIDs = validJobIDs;
+
+ doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);
+ doneJobsOrder = await getDoneJobsOrdered(
+ req.params.jobId,
+ start,
+ end ?? -1,
+ );
+
+ totalCount = jobIDs.length;
+
+ if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {
+ const x = await supabase_rr_service
+ .from('firecrawl_jobs')
+ .select('*', { count: 'exact', head: true })
+ .eq("crawl_id", req.params.jobId)
+ .eq("success", true)
+
+ totalCount = x.count ?? 0;
+ }
+
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const creditsRpc = await supabase_rr_service
+ .rpc("credits_billed_by_crawl_id_1", {
+ i_crawl_id: req.params.jobId,
+ });
+
+ creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (
+ sc.scrapeOptions?.extract
+ ? 5
+ : 1
+ ));
+ } else {
+ creditsUsed = totalCount * (
+ sc.scrapeOptions?.extract
+ ? 5
+ : 1
+ )
+ }
+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {
+ // TODO: move to read replica
+ const { data: scrapeJobCounts, error: scrapeJobError } = await supabase_service
+ .rpc("count_jobs_of_crawl_team", { i_crawl_id: req.params.jobId, i_team_id: req.auth.team_id });
+
+ if (scrapeJobError || !scrapeJobCounts || scrapeJobCounts.length === 0) {
+ logger.error("Error getting scrape job count", { error: scrapeJobError });
+ throw scrapeJobError;
+ }
+
+ const scrapeJobCount: number = scrapeJobCounts[0].count ?? 0;
+
+ const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("job_id", req.params.jobId)
+ .limit(1)
+ .throwOnError();
+
+ if (crawlJobError) {
+ logger.error("Error getting crawl job", { error: crawlJobError });
+ throw crawlJobError;
+ }
+
+ if (!crawlJobs || crawlJobs.length === 0) {
+ if (scrapeJobCount === 0) {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ } else {
+ status = "completed"; // fake completed to cut the losses
+ }
+ } else {
+ status = crawlJobs[0].success ? "completed" : "failed";
+ }
+
+ const crawlJob = crawlJobs[0];
+
+ if (crawlJob && crawlJob.team_id !== req.auth.team_id) {
+ return res.status(403).json({ success: false, error: "Forbidden" });
+ }
+
+ const teamIdsExcludedFromExpiry = [
+ "8f819703-1b85-4f7f-a6eb-e03841ec6617",
+ "f96ad1a4-8102-4b35-9904-36fd517d3616",
+ ];
+
+ if (
+ crawlJob
+ && !teamIdsExcludedFromExpiry.includes(crawlJob.team_id)
+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000
+ ) {
+ return res.status(404).json({ success: false, error: "Job expired" });
+ }
+
+ doneJobsLength = scrapeJobCount!;
+ doneJobsOrder = [];
+
+ const step = 1000;
+ let i = 0;
+ while (true) {
+ const rangeStart = start + (i * step);
+ let rangeEnd = start + ((i + 1) * step);
+ if (end !== undefined) {
+ rangeEnd = Math.min(end, rangeEnd);
+ }
+
+ const currentJobs = await supabase_rr_service
+ .from("firecrawl_jobs")
+ .select("job_id")
+ .eq("crawl_id", req.params.jobId)
+ .eq("team_id", req.acuc.team_id)
+ .order("date_added", { ascending: true })
+ .range(rangeStart, rangeEnd)
+ .throwOnError();
+
+ const rangeLength = rangeEnd - rangeStart;
+
+ const data = currentJobs.data ?? [];
+
+ doneJobsOrder.push(...data.map(x => x.job_id));
+
+ if (data.length < rangeLength) {
+ break;
+ }
+
+ if (rangeEnd === end) {
+ break;
+ }
+
+ i++
+ }
+
+ totalCount = scrapeJobCount ?? 0;
+ creditsUsed = crawlJob?.credits_billed ?? totalCount;
+ } else {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
+
+ let doneJobs: PseudoJob<any>[] = [];
+
+ if (end === undefined) {
+ // determine 10 megabyte limit
+ let bytes = 0;
+ const bytesLimit = 10485760; // 10 MiB in bytes
+ const factor = 100; // chunking for faster retrieval
+
+ for (
+ let i = 0;
+ i < doneJobsOrder.length && bytes < bytesLimit;
+ i += factor
+ ) {
+ // get current chunk and retrieve jobs
+ const currentIDs = doneJobsOrder.slice(i, i + factor);
+ const jobs = await getJobs(currentIDs);
+
+ // iterate through jobs and add them one them one to the byte counter
+ // both loops will break once we cross the byte counter
+ for (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {
+ const job = jobs[ii];
+ const state = await job.getState();
+
+ if (state === "failed" || state === "active") {
+ // TODO: why is active here? race condition? shouldn't matter tho - MG
+ continue;
+ }
+
+ if (job.returnvalue === undefined || job.returnvalue === null) {
+ logger.warn(
+ "Job was considered done, but returnvalue is undefined!",
+ { jobId: job.id, state, returnvalue: job.returnvalue },
+ );
+ continue;
+ }
+ doneJobs.push(job);
+ bytes += JSON.stringify(job.returnvalue ?? null).length;
+ }
+ }
+
+ // if we ran over the bytes limit, remove the last document, except if it's the only document
+ if (bytes > bytesLimit && doneJobs.length !== 1) {
+ doneJobs.splice(doneJobs.length - 1, 1);
+ }
+ } else {
+ doneJobs = (
+ await Promise.all(
+ (await getJobs(doneJobsOrder)).map(async (x) =>
+ (await x.getState()) === "failed" ? null : x,
+ ),
+ )
+ ).filter((x) => x !== null) as PseudoJob<any>[];
+ }
+
+ const data = doneJobs.map((x) => x.returnvalue);
+
+ const protocol = process.env.ENV === "local" ? req.protocol : "https";
+ const nextURL = new URL(
+ `${protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}`,
+ );
+
+ nextURL.searchParams.set("skip", (start + data.length).toString());
+
+ if (typeof req.query.limit === "string") {
+ nextURL.searchParams.set("limit", req.query.limit);
+ }
+
+ res.status(200).json({
+ success: true,
+ status,
+ completed: doneJobsLength,
+ total: totalCount,
+ creditsUsed,
+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
+ next:
+ status !== "scraping" && start + data.length === doneJobsLength // if there's not gonna be any documents after this
+ ? undefined
+ : nextURL.href,
+ data: data,
+ });
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index ea919a21e..d207b8170 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -617,7 +617,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
webhook: webhookSchema.optional(),
appendToId: z.string().uuid().optional(),
- ignoreInvalidURLs: z.boolean().default(false),
+ ignoreInvalidURLs: z.boolean().default(true),
maxConcurrency: z.number().positive().int().optional(),
zeroDataRetention: z.boolean().optional(),
})
@@ -634,7 +634,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
webhook: webhookSchema.optional(),
appendToId: z.string().uuid().optional(),
- ignoreInvalidURLs: z.boolean().default(false),
+ ignoreInvalidURLs: z.boolean().default(true),
maxConcurrency: z.number().positive().int().optional(),
zeroDataRetention: z.boolean().optional(),
})
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 1122e82ac..13bc53787 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -2,6 +2,8 @@ import express from "express";
import { RateLimiterMode } from "../types";
import expressWs from "express-ws";
import { searchController } from "../controllers/v2/search";
+import { batchScrapeController } from "../controllers/v2/batch-scrape";
+import { crawlStatusController } from "../controllers/v2/crawl-status";
import {
authMiddleware,
checkCreditsMiddleware,
@@ -18,3 +20,16 @@ v2Router.post(
checkCreditsMiddleware(),
wrap(searchController),
);
+
+v2Router.post(
+ "/batch/scrape",
+ authMiddleware(RateLimiterMode.Scrape),
+ checkCreditsMiddleware(),
+ wrap(batchScrapeController),
+);
+
+v2Router.get(
+ "/batch/scrape/:jobId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap((req: any, res: any) => crawlStatusController(req, res, true)),
+);