Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: crawl params preview
Nicolasc0932208676a
diff --git a/apps/api/requests/v2/crawl.requests.http b/apps/api/requests/v2/crawl.requests.httpindex 8ddc0cb3f..47d34fdd2 100644--- a/apps/api/requests/v2/crawl.requests.http+++ b/apps/api/requests/v2/crawl.requests.http@@ -31,8 +31,8 @@ content-type: application/json# "success": true,# "data": {# "url": "https://firecrawl.dev",-# "includes": ["blog", "news"],-# "excludes": [],+# "includePaths": ["blog", "news"],+# "excludePaths": [],# "limit": 50,# "maxDepth": 2# }@@ -46,5 +46,5 @@ content-type: application/json{"url": "https://firecrawl.dev",- "includes": ["blog", "news"]+ "includePaths": ["blog", "news"]}diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsnew file mode 100644index 000000000..b951784e9--- /dev/null+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -0,0 +1,110 @@+import { Response } from "express";+import { z } from "zod";+import {+ RequestWithAuth,+ crawlRequestSchema,+} from "./types";+import { logger as _logger } from "../../lib/logger";+import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";+import { CostTracking } from "../../lib/extract/extraction-service";++// Define the request schema for params preview+// Only url and prompt are required/relevant for preview+const crawlParamsPreviewRequestSchema = z.object({+ url: z.string().url(),+ prompt: z.string().max(10000),+});++type CrawlParamsPreviewRequest = z.infer<typeof crawlParamsPreviewRequestSchema>;++type CrawlParamsPreviewResponse = {+ success: boolean;+ data?: {+ url: string;+ includePaths?: string[];+ excludePaths?: string[];+ maxDepth?: number;+ maxDiscoveryDepth?: number;+ crawlEntireDomain?: boolean;+ allowExternalLinks?: boolean;+ allowSubdomains?: boolean;+ ignoreSitemap?: boolean;+ ignoreQueryParameters?: boolean;+ deduplicateSimilarURLs?: boolean;+ delay?: number;+ limit?: number;+ };+ error?: string;+};++export async function crawlParamsPreviewController(+ req: RequestWithAuth<{}, CrawlParamsPreviewResponse, CrawlParamsPreviewRequest>,+ res: Response<CrawlParamsPreviewResponse>,+) {+ const logger = _logger.child({+ module: "api/v2",+ method: "crawlParamsPreviewController",+ teamId: req.auth.team_id,+ });++ try {+ // Parse and validate request body+ const parsedBody = crawlParamsPreviewRequestSchema.parse(req.body);++ logger.debug("Crawl params preview request", {+ url: parsedBody.url,+ prompt: parsedBody.prompt,+ });++ // Generate crawler options from prompt+ const costTracking = new CostTracking();+ const { extract } = await generateCrawlerOptionsFromPrompt(+ parsedBody.prompt,+ logger,+ costTracking+ );++ const generatedOptions = extract || {};++ logger.debug("Generated crawler options from prompt", {+ prompt: parsedBody.prompt,+ generatedOptions: generatedOptions,+ });++ // Prepare response data+ const responseData = {+ url: parsedBody.url,+ ...generatedOptions,+ };++ // Remove any undefined values for cleaner response+ Object.keys(responseData).forEach(key => {+ if (responseData[key] === undefined) {+ delete responseData[key];+ }+ });++ return res.status(200).json({+ success: true,+ data: responseData,+ });++ } catch (error) {+ if (error instanceof z.ZodError) {+ return res.status(400).json({+ success: false,+ error: "Invalid request parameters: " + error.errors.map(e => e.message).join(", "),+ });+ }++ logger.error("Failed to generate crawler params preview", {+ error: error.message,+ prompt: req.body.prompt,+ });++ return res.status(400).json({+ success: false,+ error: "Failed to process natural language prompt. Please try rephrasing.",+ });+ }+}\ No newline at end of filediff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 64b9b5c94..732fd4a30 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -5,6 +5,7 @@ import { searchController } from "../controllers/v2/search";import { scrapeController } from "../controllers/v2/scrape";import { batchScrapeController } from "../controllers/v2/batch-scrape";import { crawlController } from "../controllers/v2/crawl";+import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preview";import { crawlStatusController } from "../controllers/v2/crawl-status";import { mapV2Controller } from "../controllers/v2/map";import {@@ -56,6 +57,13 @@ v2Router.post(wrap(crawlController),);+v2Router.post(+ "/crawl/params-preview",+ authMiddleware(RateLimiterMode.Crawl),+ checkCreditsMiddleware(),+ wrap(crawlParamsPreviewController),+);+v2Router.get("/crawl/:jobId",authMiddleware(RateLimiterMode.CrawlStatus),