Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

yeeted max depth

Gergő Móriczda2d054291d3
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
new file mode 100644
index 000000000..d1ae218f1
--- /dev/null
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -0,0 +1,149 @@
+import { Response } from "express";
+import { v4 as uuidv4 } from "uuid";
+import {
+ CrawlRequest,
+ crawlRequestSchema,
+ CrawlResponse,
+ RequestWithAuth,
+ toLegacyCrawlerOptions,
+} from "./types";
+import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
+import { logCrawl } from "../../services/logging/crawl_log";
+import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
+import { logger as _logger } from "../../lib/logger";
+
+export async function crawlController(
+ req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
+ res: Response<CrawlResponse>,
+) {
+ const preNormalizedBody = req.body;
+ req.body = crawlRequestSchema.parse(req.body);
+
+ if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {
+ return res.status(400).json({
+ success: false,
+ error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",
+ });
+ }
+
+ const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
+
+ const id = uuidv4();
+ const logger = _logger.child({
+ crawlId: id,
+ module: "api/v1",
+ method: "crawlController",
+ teamId: req.auth.team_id,
+ zeroDataRetention,
+ });
+
+ logger.debug("Crawl " + id + " starting", {
+ request: req.body,
+ originalRequest: preNormalizedBody,
+ account: req.account,
+ });
+
+ await logCrawl(id, req.auth.team_id);
+
+ let { remainingCredits } = req.account!;
+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ if (!useDbAuthentication) {
+ remainingCredits = Infinity;
+ }
+
+ const crawlerOptions = {
+ ...req.body,
+ url: undefined,
+ scrapeOptions: undefined,
+ };
+ const scrapeOptions = req.body.scrapeOptions;
+
+ // TODO: @rafa, is this right? copied from v0
+ if (Array.isArray(crawlerOptions.includePaths)) {
+ for (const x of crawlerOptions.includePaths) {
+ try {
+ new RegExp(x);
+ } catch (e) {
+ return res.status(400).json({ success: false, error: e.message });
+ }
+ }
+ }
+
+ if (Array.isArray(crawlerOptions.excludePaths)) {
+ for (const x of crawlerOptions.excludePaths) {
+ try {
+ new RegExp(x);
+ } catch (e) {
+ return res.status(400).json({ success: false, error: e.message });
+ }
+ }
+ }
+
+ const originalLimit = crawlerOptions.limit;
+ crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit);
+ logger.debug("Determined limit: " + crawlerOptions.limit, {
+ remainingCredits,
+ bodyLimit: originalLimit,
+ originalBodyLimit: preNormalizedBody.limit,
+ });
+
+ const sc: StoredCrawl = {
+ originUrl: req.body.url,
+ crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),
+ scrapeOptions,
+ internalOptions: {
+ disableSmartWaitCache: true,
+ teamId: req.auth.team_id,
+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
+ zeroDataRetention,
+ }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
+ team_id: req.auth.team_id,
+ createdAt: Date.now(),
+ maxConcurrency: req.body.maxConcurrency !== undefined ? Math.min(req.body.maxConcurrency, req.acuc.concurrency) : undefined,
+ zeroDataRetention,
+ };
+
+ const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);
+
+ try {
+ sc.robots = await crawler.getRobotsTxt(scrapeOptions.skipTlsVerification);
+ const robotsCrawlDelay = crawler.getRobotsCrawlDelay();
+ if (robotsCrawlDelay !== null && !sc.crawlerOptions.delay) {
+ sc.crawlerOptions.delay = robotsCrawlDelay;
+ }
+ } catch (e) {
+ logger.debug("Failed to get robots.txt (this is probably fine!)", {
+ error: e,
+ });
+ }
+
+ await saveCrawl(id, sc);
+
+ await _addScrapeJobToBullMQ(
+ {
+ url: req.body.url,
+ mode: "kickoff" as const,
+ team_id: req.auth.team_id,
+ crawlerOptions,
+ scrapeOptions: sc.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: req.body.origin,
+ integration: req.body.integration,
+ crawl_id: id,
+ webhook: req.body.webhook,
+ v1: true,
+ zeroDataRetention: zeroDataRetention || false,
+ },
+ {},
+ crypto.randomUUID(),
+ 10,
+ );
+
+ const protocol = process.env.ENV === "local" ? req.protocol : "https";
+
+ return res.status(200).json({
+ success: true,
+ id,
+ url: `${protocol}://${req.get("host")}/v1/crawl/${id}`,
+ });
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index d207b8170..83f11779b 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -651,7 +651,6 @@ const crawlerOptions = z
.object({
includePaths: z.string().array().default([]),
excludePaths: z.string().array().default([]),
- maxDepth: z.number().default(10), // default?
maxDiscoveryDepth: z.number().optional(),
limit: z.number().default(10000), // default?
allowBackwardLinks: z.boolean().default(false), // DEPRECATED: use crawlEntireDomain
@@ -695,20 +694,6 @@ export const crawlRequestSchema = crawlerOptions
.refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)
.refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)
.refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)
- .refine(
- (data) => {
- try {
- const urlDepth = getURLDepth(data.url);
- return urlDepth <= data.maxDepth;
- } catch (e) {
- return false;
- }
- },
- {
- message: "URL depth exceeds the specified maxDepth",
- path: ["url"]
- }
- )
.transform((x) => {
if (x.crawlEntireDomain !== undefined) {
x.allowBackwardLinks = x.crawlEntireDomain;
@@ -1106,7 +1091,7 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {
includes: x.includePaths,
excludes: x.excludePaths,
maxCrawledLinks: x.limit,
- maxDepth: x.maxDepth,
+ maxDepth: 9999,
limit: x.limit,
generateImgAltText: false,
allowBackwardCrawling: x.crawlEntireDomain ?? x.allowBackwardLinks,
@@ -1128,7 +1113,6 @@ export function toNewCrawlerOptions(x: any): CrawlerOptions {
includePaths: x.includes,
excludePaths: x.excludes,
limit: x.limit,
- maxDepth: x.maxDepth,
allowBackwardLinks: x.allowBackwardCrawling,
crawlEntireDomain: x.allowBackwardCrawling,
allowExternalLinks: x.allowExternalContentLinks,
@@ -1152,7 +1136,6 @@ export function fromLegacyCrawlerOptions(x: any, teamId: string): {
includePaths: x.includes,
excludePaths: x.excludes,
limit: x.maxCrawledLinks ?? x.limit,
- maxDepth: x.maxDepth,
allowBackwardLinks: x.allowBackwardCrawling,
crawlEntireDomain: x.allowBackwardCrawling,
allowExternalLinks: x.allowExternalContentLinks,