Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: implement v2 batch scrape with improved error handling (#1851)
devin-ai-integration[bot]b4e0866bdc8b
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsnew file mode 100644index 000000000..a200f2e77--- /dev/null+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -0,0 +1,203 @@+import { Response } from "express";+import { v4 as uuidv4 } from "uuid";+import {+ BatchScrapeRequest,+ batchScrapeRequestSchema,+ batchScrapeRequestSchemaNoURLValidation,+ url as urlSchema,+ RequestWithAuth,+ ScrapeOptions,+ BatchScrapeResponse,+} from "./types";+import {+ addCrawlJobs,+ finishCrawlKickoff,+ getCrawl,+ lockURLs,+ saveCrawl,+ StoredCrawl,+} from "../../lib/crawl-redis";+import { logCrawl } from "../../services/logging/crawl_log";+import { getJobPriority } from "../../lib/job-priority";+import { addScrapeJobs } from "../../services/queue-jobs";+import { callWebhook } from "../../services/webhook";+import { logger as _logger } from "../../lib/logger";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";++export async function batchScrapeController(+ req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,+ res: Response<BatchScrapeResponse>,+) {+ const preNormalizedBody = { ...req.body };++ if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {+ return res.status(400).json({+ success: false,+ error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ });+ }++ const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;++ if (req.body?.ignoreInvalidURLs === true) {+ req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);+ } else {+ req.body = batchScrapeRequestSchema.parse(req.body);+ }++ const id = req.body.appendToId ?? uuidv4();+ const logger = _logger.child({+ crawlId: id,+ batchScrapeId: id,+ module: "api/v2",+ method: "batchScrapeController",+ teamId: req.auth.team_id,+ zeroDataRetention,+ });++ let urls: string[] = req.body.urls;+ let unnormalizedURLs = preNormalizedBody.urls;+ let invalidURLs: string[] | undefined = undefined;++ if (req.body.ignoreInvalidURLs) {+ invalidURLs = [];++ let pendingURLs = urls;+ urls = [];+ unnormalizedURLs = [];+ for (const u of pendingURLs) {+ try {+ const nu = urlSchema.parse(u);+ if (!isUrlBlocked(nu, req.acuc?.flags ?? null)) {+ urls.push(nu);+ unnormalizedURLs.push(u);+ } else {+ invalidURLs.push(u);+ }+ } catch (_) {+ invalidURLs.push(u);+ }+ }+ } else {+ if (req.body.urls?.some((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null))) {+ if (!res.headersSent) {+ return res.status(403).json({+ success: false,+ error: BLOCKLISTED_URL_MESSAGE,+ });+ }+ }+ }++ logger.debug("Batch scrape " + id + " starting", {+ urlsLength: urls.length,+ appendToId: req.body.appendToId,+ account: req.account,+ });++ if (!req.body.appendToId) {+ await logCrawl(id, req.auth.team_id);+ }++ const sc: StoredCrawl = req.body.appendToId+ ? ((await getCrawl(req.body.appendToId)) as StoredCrawl)+ : {+ crawlerOptions: null,+ scrapeOptions: req.body,+ internalOptions: {+ disableSmartWaitCache: true,+ teamId: req.auth.team_id,+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,+ zeroDataRetention,+ }, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter+ team_id: req.auth.team_id,+ createdAt: Date.now(),+ maxConcurrency: req.body.maxConcurrency,+ zeroDataRetention,+ };++ if (!req.body.appendToId) {+ await saveCrawl(id, sc);+ }++ let jobPriority = 20;++ // If it is over 1000, we need to get the job priority,+ // otherwise we can use the default priority of 20+ if (urls.length > 1000) {+ // set base to 21+ jobPriority = await getJobPriority({+ team_id: req.auth.team_id,+ basePriority: 21,+ });+ }+ logger.debug("Using job priority " + jobPriority, { jobPriority });++ const scrapeOptions: ScrapeOptions = { ...req.body };+ delete (scrapeOptions as any).urls;+ delete (scrapeOptions as any).appendToId;++ const jobs = urls.map(x => ({+ data: {+ url: x,+ mode: "single_urls" as const,+ team_id: req.auth.team_id,+ crawlerOptions: null,+ scrapeOptions,+ origin: "api",+ integration: req.body.integration,+ crawl_id: id,+ sitemapped: true,+ v1: false,+ webhook: req.body.webhook,+ internalOptions: sc.internalOptions,+ zeroDataRetention,+ },+ opts: {+ jobId: uuidv4(),+ priority: 20,+ },+ }));++ await finishCrawlKickoff(id);++ logger.debug("Locking URLs...");+ await lockURLs(+ id,+ sc,+ jobs.map((x) => x.data.url),+ logger,+ );+ logger.debug("Adding scrape jobs to Redis...");+ await addCrawlJobs(+ id,+ jobs.map((x) => x.opts.jobId),+ logger,+ );+ logger.debug("Adding scrape jobs to BullMQ...");+ await addScrapeJobs(jobs);++ if (req.body.webhook) {+ logger.debug("Calling webhook with batch_scrape.started...", {+ webhook: req.body.webhook,+ });+ await callWebhook({+ teamId: req.auth.team_id,+ crawlId: id,+ data: null,+ webhook: req.body.webhook,+ v1: false,+ eventType: "batch_scrape.started",+ });+ }++ const protocol = process.env.ENV === "local" ? req.protocol : "https";++ return res.status(200).json({+ success: true,+ id,+ url: `${protocol}://${req.get("host")}/v2/batch/scrape/${id}`,+ invalidURLs,+ });+}diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsnew file mode 100644index 000000000..94729506a--- /dev/null+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -0,0 +1,449 @@+import { Response } from "express";+import {+ CrawlStatusParams,+ CrawlStatusResponse,+ ErrorResponse,+ RequestWithAuth,+} from "./types";+import {+ getCrawl,+ getCrawlExpiry,+ getCrawlJobs,+ getDoneJobsOrdered,+ getDoneJobsOrderedLength,+ isCrawlKickoffFinished,+} from "../../lib/crawl-redis";+import { getScrapeQueue } from "../../services/queue-service";+import {+ supabaseGetJobById,+ supabaseGetJobsById,+} from "../../lib/supabase-jobs";+import { configDotenv } from "dotenv";+import type { Job, JobState, Queue } from "bullmq";+import { logger } from "../../lib/logger";+import { supabase_rr_service, supabase_service } from "../../services/supabase";+import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";+import { getJobFromGCS } from "../../lib/gcs-jobs";+configDotenv();++export type PseudoJob<T> = {+ id: string,+ getState(): Promise<JobState | "unknown"> | JobState | "unknown",+ returnvalue: T | null,+ timestamp: number,+ data: {+ scrapeOptions: any,+ teamId?: string,+ },+ failedReason?: string,+}++export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string }++export async function getJob(id: string): Promise<PseudoJob<any> | null> {+ const [bullJob, dbJob, gcsJob] = await Promise.all([+ getScrapeQueue().getJob(id),+ (process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,+ (process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,+ ]);++ if (!bullJob && !dbJob) return null;++ const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ if (gcsJob === null && data) {+ logger.warn("GCS Job not found", {+ jobId: id,+ });+ }++ const job: PseudoJob<any> = {+ id,+ getState: bullJob ? bullJob.getState : (() => dbJob!.success ? "completed" : "failed"),+ returnvalue: Array.isArray(data)+ ? data[0]+ : data,+ data: {+ scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,+ },+ timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ }++ return job;+}++export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {+ const [bullJobs, dbJobs, gcsJobs] = await Promise.all([+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],+ process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],+ ]);++ const bullJobMap = new Map<string, PseudoJob<any>>();+ const dbJobMap = new Map<string, DBJob>();+ const gcsJobMap = new Map<string, any>();++ for (const job of bullJobs) {+ bullJobMap.set(job.id, job);+ }++ for (const job of dbJobs) {+ dbJobMap.set(job.job_id, job);+ }++ for (const job of gcsJobs) {+ gcsJobMap.set(job.id, job.job);+ }++ const jobs: PseudoJob<any>[] = [];++ for (const id of ids) {+ const bullJob = bullJobMap.get(id);+ const dbJob = dbJobMap.get(id);+ const gcsJob = gcsJobMap.get(id);++ if (!bullJob && !dbJob) continue;++ const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue;+ if (gcsJob === null && data) {+ logger.warn("GCS Job not found", {+ jobId: id,+ });+ }++ const job: PseudoJob<any> = {+ id,+ getState: bullJob ? (() => bullJob.getState()) : (() => dbJob!.success ? "completed" : "failed"),+ returnvalue: Array.isArray(data)+ ? data[0]+ : data,+ data: {+ scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,+ },+ timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,+ }++ jobs.push(job);+ }++ return jobs;+}++export async function crawlStatusController(+ req: RequestWithAuth<CrawlStatusParams, undefined, CrawlStatusResponse>,+ res: Response<CrawlStatusResponse>,+ isBatch = false,+) {+ const start =+ typeof req.query.skip === "string" ? parseInt(req.query.skip, 10) : 0;+ const end =+ typeof req.query.limit === "string"+ ? start + parseInt(req.query.limit, 10) - 1+ : undefined;++ const sc = await getCrawl(req.params.jobId);++ let status: Exclude<CrawlStatusResponse, ErrorResponse>["status"];+ let doneJobsLength: number;+ let doneJobsOrder: string[];+ let totalCount: number;+ let creditsUsed: number;++ if (sc) {+ if (sc.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }++ let jobIDs = await getCrawlJobs(req.params.jobId);+ let jobStatuses = await Promise.all(+ jobIDs.map(+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ ),+ );++ if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {+ for (let rangeStart = 0; ; rangeStart += 1000) {+ const rangeEnd = rangeStart + 1000;++ const currentJobs = await supabase_service+ .from("firecrawl_jobs")+ .select("success, job_id")+ .eq("crawl_id", req.params.jobId)+ .order("date_added", { ascending: true })+ .range(rangeStart, rangeEnd);++ const rangeLength = rangeEnd - rangeStart;++ if (currentJobs.error) {+ logger.error("Error getting current jobs", { error: currentJobs.error });+ continue;+ }++ const data = currentJobs.data ?? [];++ for (const job of data) {+ let index = jobStatuses.findIndex((x) => x[0] === job.job_id);+ if (index !== -1) {+ jobStatuses[index] = [job.job_id, job.success ? "completed" : "failed"];+ } else {+ jobStatuses.push([job.job_id, job.success ? "completed" : "failed"]);+ }+ }++ if (data.length < rangeLength) {+ break;+ }+ }+ }++ const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id));+ const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId));++ const validJobStatuses: [string, JobState | "unknown"][] = [];+ const validJobIDs: string[] = [];++ for (const [id, status] of jobStatuses) {+ if (throttledJobsSet.has(id)) {+ validJobStatuses.push([id, "prioritized"]);+ validJobIDs.push(id);+ } else if (status === "unknown" && activeJobsSet.has(id)) {+ validJobStatuses.push([id, "active"]);+ validJobIDs.push(id);+ } else if (+ status !== "failed" &&+ status !== "unknown"+ ) {+ validJobStatuses.push([id, status]);+ validJobIDs.push(id);+ }+ }++ status =+ sc.cancelled+ ? "cancelled"+ : validJobStatuses.every((x) => x[1] === "completed") &&+ (sc.crawlerOptions+ ? await isCrawlKickoffFinished(req.params.jobId)+ : true)+ ? "completed"+ : "scraping";++ // Use validJobIDs instead of jobIDs for further processing+ jobIDs = validJobIDs;++ doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);+ doneJobsOrder = await getDoneJobsOrdered(+ req.params.jobId,+ start,+ end ?? -1,+ );++ totalCount = jobIDs.length;++ if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {+ const x = await supabase_rr_service+ .from('firecrawl_jobs')+ .select('*', { count: 'exact', head: true })+ .eq("crawl_id", req.params.jobId)+ .eq("success", true)++ totalCount = x.count ?? 0;+ }++ if (process.env.USE_DB_AUTHENTICATION === "true") {+ const creditsRpc = await supabase_rr_service+ .rpc("credits_billed_by_crawl_id_1", {+ i_crawl_id: req.params.jobId,+ });++ creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (+ sc.scrapeOptions?.extract+ ? 5+ : 1+ ));+ } else {+ creditsUsed = totalCount * (+ sc.scrapeOptions?.extract+ ? 5+ : 1+ )+ }+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {+ // TODO: move to read replica+ const { data: scrapeJobCounts, error: scrapeJobError } = await supabase_service+ .rpc("count_jobs_of_crawl_team", { i_crawl_id: req.params.jobId, i_team_id: req.auth.team_id });++ if (scrapeJobError || !scrapeJobCounts || scrapeJobCounts.length === 0) {+ logger.error("Error getting scrape job count", { error: scrapeJobError });+ throw scrapeJobError;+ }++ const scrapeJobCount: number = scrapeJobCounts[0].count ?? 0;++ const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("job_id", req.params.jobId)+ .limit(1)+ .throwOnError();++ if (crawlJobError) {+ logger.error("Error getting crawl job", { error: crawlJobError });+ throw crawlJobError;+ }++ if (!crawlJobs || crawlJobs.length === 0) {+ if (scrapeJobCount === 0) {+ return res.status(404).json({ success: false, error: "Job not found" });+ } else {+ status = "completed"; // fake completed to cut the losses+ }+ } else {+ status = crawlJobs[0].success ? "completed" : "failed";+ }++ const crawlJob = crawlJobs[0];++ if (crawlJob && crawlJob.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }++ const teamIdsExcludedFromExpiry = [+ "8f819703-1b85-4f7f-a6eb-e03841ec6617",+ "f96ad1a4-8102-4b35-9904-36fd517d3616",+ ];++ if (+ crawlJob+ && !teamIdsExcludedFromExpiry.includes(crawlJob.team_id)+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000+ ) {+ return res.status(404).json({ success: false, error: "Job expired" });+ }++ doneJobsLength = scrapeJobCount!;+ doneJobsOrder = [];++ const step = 1000;+ let i = 0;+ while (true) {+ const rangeStart = start + (i * step);+ let rangeEnd = start + ((i + 1) * step);+ if (end !== undefined) {+ rangeEnd = Math.min(end, rangeEnd);+ }++ const currentJobs = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("job_id")+ .eq("crawl_id", req.params.jobId)+ .eq("team_id", req.acuc.team_id)+ .order("date_added", { ascending: true })+ .range(rangeStart, rangeEnd)+ .throwOnError();++ const rangeLength = rangeEnd - rangeStart;++ const data = currentJobs.data ?? [];++ doneJobsOrder.push(...data.map(x => x.job_id));++ if (data.length < rangeLength) {+ break;+ }++ if (rangeEnd === end) {+ break;+ }++ i+++ }++ totalCount = scrapeJobCount ?? 0;+ creditsUsed = crawlJob?.credits_billed ?? totalCount;+ } else {+ return res.status(404).json({ success: false, error: "Job not found" });+ }++ let doneJobs: PseudoJob<any>[] = [];++ if (end === undefined) {+ // determine 10 megabyte limit+ let bytes = 0;+ const bytesLimit = 10485760; // 10 MiB in bytes+ const factor = 100; // chunking for faster retrieval++ for (+ let i = 0;+ i < doneJobsOrder.length && bytes < bytesLimit;+ i += factor+ ) {+ // get current chunk and retrieve jobs+ const currentIDs = doneJobsOrder.slice(i, i + factor);+ const jobs = await getJobs(currentIDs);++ // iterate through jobs and add them one them one to the byte counter+ // both loops will break once we cross the byte counter+ for (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {+ const job = jobs[ii];+ const state = await job.getState();++ if (state === "failed" || state === "active") {+ // TODO: why is active here? race condition? shouldn't matter tho - MG+ continue;+ }++ if (job.returnvalue === undefined || job.returnvalue === null) {+ logger.warn(+ "Job was considered done, but returnvalue is undefined!",+ { jobId: job.id, state, returnvalue: job.returnvalue },+ );+ continue;+ }+ doneJobs.push(job);+ bytes += JSON.stringify(job.returnvalue ?? null).length;+ }+ }++ // if we ran over the bytes limit, remove the last document, except if it's the only document+ if (bytes > bytesLimit && doneJobs.length !== 1) {+ doneJobs.splice(doneJobs.length - 1, 1);+ }+ } else {+ doneJobs = (+ await Promise.all(+ (await getJobs(doneJobsOrder)).map(async (x) =>+ (await x.getState()) === "failed" ? null : x,+ ),+ )+ ).filter((x) => x !== null) as PseudoJob<any>[];+ }++ const data = doneJobs.map((x) => x.returnvalue);++ const protocol = process.env.ENV === "local" ? req.protocol : "https";+ const nextURL = new URL(+ `${protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}`,+ );++ nextURL.searchParams.set("skip", (start + data.length).toString());++ if (typeof req.query.limit === "string") {+ nextURL.searchParams.set("limit", req.query.limit);+ }++ res.status(200).json({+ success: true,+ status,+ completed: doneJobsLength,+ total: totalCount,+ creditsUsed,+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),+ next:+ status !== "scraping" && start + data.length === doneJobsLength // if there's not gonna be any documents after this+ ? undefined+ : nextURL.href,+ data: data,+ });+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex ea919a21e..d207b8170 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -617,7 +617,7 @@ export const batchScrapeRequestSchema = baseScrapeOptionsintegration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),- ignoreInvalidURLs: z.boolean().default(false),+ ignoreInvalidURLs: z.boolean().default(true),maxConcurrency: z.number().positive().int().optional(),zeroDataRetention: z.boolean().optional(),})@@ -634,7 +634,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptionsintegration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),- ignoreInvalidURLs: z.boolean().default(false),+ ignoreInvalidURLs: z.boolean().default(true),maxConcurrency: z.number().positive().int().optional(),zeroDataRetention: z.boolean().optional(),})diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 1122e82ac..13bc53787 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -2,6 +2,8 @@ import express from "express";import { RateLimiterMode } from "../types";import expressWs from "express-ws";import { searchController } from "../controllers/v2/search";+import { batchScrapeController } from "../controllers/v2/batch-scrape";+import { crawlStatusController } from "../controllers/v2/crawl-status";import {authMiddleware,checkCreditsMiddleware,@@ -18,3 +20,16 @@ v2Router.post(checkCreditsMiddleware(),wrap(searchController),);++v2Router.post(+ "/batch/scrape",+ authMiddleware(RateLimiterMode.Scrape),+ checkCreditsMiddleware(),+ wrap(batchScrapeController),+);++v2Router.get(+ "/batch/scrape/:jobId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap((req: any, res: any) => crawlStatusController(req, res, true)),+);