Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
yeeted max depth
Gergő Móriczda2d054291d3
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsnew file mode 100644index 000000000..d1ae218f1--- /dev/null+++ b/apps/api/src/controllers/v2/crawl.ts@@ -0,0 +1,149 @@+import { Response } from "express";+import { v4 as uuidv4 } from "uuid";+import {+ CrawlRequest,+ crawlRequestSchema,+ CrawlResponse,+ RequestWithAuth,+ toLegacyCrawlerOptions,+} from "./types";+import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";+import { logCrawl } from "../../services/logging/crawl_log";+import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";+import { logger as _logger } from "../../lib/logger";++export async function crawlController(+ req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,+ res: Response<CrawlResponse>,+) {+ const preNormalizedBody = req.body;+ req.body = crawlRequestSchema.parse(req.body);++ if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {+ return res.status(400).json({+ success: false,+ error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ });+ }++ const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;++ const id = uuidv4();+ const logger = _logger.child({+ crawlId: id,+ module: "api/v1",+ method: "crawlController",+ teamId: req.auth.team_id,+ zeroDataRetention,+ });++ logger.debug("Crawl " + id + " starting", {+ request: req.body,+ originalRequest: preNormalizedBody,+ account: req.account,+ });++ await logCrawl(id, req.auth.team_id);++ let { remainingCredits } = req.account!;+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ if (!useDbAuthentication) {+ remainingCredits = Infinity;+ }++ const crawlerOptions = {+ ...req.body,+ url: undefined,+ scrapeOptions: undefined,+ };+ const scrapeOptions = req.body.scrapeOptions;++ // TODO: @rafa, is this right? copied from v0+ if (Array.isArray(crawlerOptions.includePaths)) {+ for (const x of crawlerOptions.includePaths) {+ try {+ new RegExp(x);+ } catch (e) {+ return res.status(400).json({ success: false, error: e.message });+ }+ }+ }++ if (Array.isArray(crawlerOptions.excludePaths)) {+ for (const x of crawlerOptions.excludePaths) {+ try {+ new RegExp(x);+ } catch (e) {+ return res.status(400).json({ success: false, error: e.message });+ }+ }+ }++ const originalLimit = crawlerOptions.limit;+ crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit);+ logger.debug("Determined limit: " + crawlerOptions.limit, {+ remainingCredits,+ bodyLimit: originalLimit,+ originalBodyLimit: preNormalizedBody.limit,+ });++ const sc: StoredCrawl = {+ originUrl: req.body.url,+ crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),+ scrapeOptions,+ internalOptions: {+ disableSmartWaitCache: true,+ teamId: req.auth.team_id,+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,+ zeroDataRetention,+ }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter+ team_id: req.auth.team_id,+ createdAt: Date.now(),+ maxConcurrency: req.body.maxConcurrency !== undefined ? Math.min(req.body.maxConcurrency, req.acuc.concurrency) : undefined,+ zeroDataRetention,+ };++ const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);++ try {+ sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);+ const robotsCrawlDelay = crawler.getRobotsCrawlDelay();+ if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {+ sc.crawlerOptions.delay = robotsCrawlDelay;+ }+ } catch (e) {+ logger.debug("Failed to get robots.txt (this is probably fine!)", {+ error: e,+ });+ }++ await saveCrawl(id, sc);++ await _addScrapeJobToBullMQ(+ {+ url: req.body.url,+ mode: "kickoff" as const,+ team_id: req.auth.team_id,+ crawlerOptions,+ scrapeOptions: sc.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: req.body.origin,+ integration: req.body.integration,+ crawl_id: id,+ webhook: req.body.webhook,+ v1: true,+ zeroDataRetention: zeroDataRetention || false,+ },+ {},+ crypto.randomUUID(),+ 10,+ );++ const protocol = process.env.ENV === "local" ? req.protocol : "https";++ return res.status(200).json({+ success: true,+ id,+ url: `${protocol}://${req.get("host")}/v1/crawl/${id}`,+ });+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex d207b8170..83f11779b 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -651,7 +651,6 @@ const crawlerOptions = z.object({includePaths: z.string().array().default([]),excludePaths: z.string().array().default([]),- maxDepth: z.number().default(10), // default?maxDiscoveryDepth: z.number().optional(),limit: z.number().default(10000), // default?allowBackwardLinks: z.boolean().default(false), // DEPRECATED: use crawlEntireDomain@@ -695,20 +694,6 @@ export const crawlRequestSchema = crawlerOptions.refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts).refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts).refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)- .refine(- (data) => {- try {- const urlDepth = getURLDepth(data.url);- return urlDepth <= data.maxDepth;- } catch (e) {- return false;- }- },- {- message: "URL depth exceeds the specified maxDepth",- path: ["url"]- }- ).transform((x) => {if (x.crawlEntireDomain !== undefined) {x.allowBackwardLinks = x.crawlEntireDomain;@@ -1106,7 +1091,7 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {includes: x.includePaths,excludes: x.excludePaths,maxCrawledLinks: x.limit,- maxDepth: x.maxDepth,+ maxDepth: 9999,limit: x.limit,generateImgAltText: false,allowBackwardCrawling: x.crawlEntireDomain ?? x.allowBackwardLinks,@@ -1128,7 +1113,6 @@ export function toNewCrawlerOptions(x: any): CrawlerOptions {includePaths: x.includes,excludePaths: x.excludes,limit: x.limit,- maxDepth: x.maxDepth,allowBackwardLinks: x.allowBackwardCrawling,crawlEntireDomain: x.allowBackwardCrawling,allowExternalLinks: x.allowExternalContentLinks,@@ -1152,7 +1136,6 @@ export function fromLegacyCrawlerOptions(x: any, teamId: string): {includePaths: x.includes,excludePaths: x.excludes,limit: x.maxCrawledLinks ?? x.limit,- maxDepth: x.maxDepth,allowBackwardLinks: x.allowBackwardCrawling,crawlEntireDomain: x.allowBackwardCrawling,allowExternalLinks: x.allowExternalContentLinks,