Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: crawl params preview

Nicolasc0932208676a
diff --git a/apps/api/requests/v2/crawl.requests.http b/apps/api/requests/v2/crawl.requests.http
index 8ddc0cb3f..47d34fdd2 100644
--- a/apps/api/requests/v2/crawl.requests.http
+++ b/apps/api/requests/v2/crawl.requests.http
@@ -31,8 +31,8 @@ content-type: application/json
# "success": true,
# "data": {
# "url": "https://firecrawl.dev",
-# "includes": ["blog", "news"],
-# "excludes": [],
+# "includePaths": ["blog", "news"],
+# "excludePaths": [],
# "limit": 50,
# "maxDepth": 2
# }
@@ -46,5 +46,5 @@ content-type: application/json
{
"url": "https://firecrawl.dev",
- "includes": ["blog", "news"]
+ "includePaths": ["blog", "news"]
}
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
new file mode 100644
index 000000000..b951784e9
--- /dev/null
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -0,0 +1,110 @@
+import { Response } from "express";
+import { z } from "zod";
+import {
+ RequestWithAuth,
+ crawlRequestSchema,
+} from "./types";
+import { logger as _logger } from "../../lib/logger";
+import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
+import { CostTracking } from "../../lib/extract/extraction-service";
+
+// Define the request schema for params preview
+// Only url and prompt are required/relevant for preview
+const crawlParamsPreviewRequestSchema = z.object({
+ url: z.string().url(),
+ prompt: z.string().max(10000),
+});
+
+type CrawlParamsPreviewRequest = z.infer<typeof crawlParamsPreviewRequestSchema>;
+
+type CrawlParamsPreviewResponse = {
+ success: boolean;
+ data?: {
+ url: string;
+ includePaths?: string[];
+ excludePaths?: string[];
+ maxDepth?: number;
+ maxDiscoveryDepth?: number;
+ crawlEntireDomain?: boolean;
+ allowExternalLinks?: boolean;
+ allowSubdomains?: boolean;
+ ignoreSitemap?: boolean;
+ ignoreQueryParameters?: boolean;
+ deduplicateSimilarURLs?: boolean;
+ delay?: number;
+ limit?: number;
+ };
+ error?: string;
+};
+
+export async function crawlParamsPreviewController(
+ req: RequestWithAuth<{}, CrawlParamsPreviewResponse, CrawlParamsPreviewRequest>,
+ res: Response<CrawlParamsPreviewResponse>,
+) {
+ const logger = _logger.child({
+ module: "api/v2",
+ method: "crawlParamsPreviewController",
+ teamId: req.auth.team_id,
+ });
+
+ try {
+ // Parse and validate request body
+ const parsedBody = crawlParamsPreviewRequestSchema.parse(req.body);
+
+ logger.debug("Crawl params preview request", {
+ url: parsedBody.url,
+ prompt: parsedBody.prompt,
+ });
+
+ // Generate crawler options from prompt
+ const costTracking = new CostTracking();
+ const { extract } = await generateCrawlerOptionsFromPrompt(
+ parsedBody.prompt,
+ logger,
+ costTracking
+ );
+
+ const generatedOptions = extract || {};
+
+ logger.debug("Generated crawler options from prompt", {
+ prompt: parsedBody.prompt,
+ generatedOptions: generatedOptions,
+ });
+
+ // Prepare response data
+ const responseData = {
+ url: parsedBody.url,
+ ...generatedOptions,
+ };
+
+ // Remove any undefined values for cleaner response
+ Object.keys(responseData).forEach(key => {
+ if (responseData[key] === undefined) {
+ delete responseData[key];
+ }
+ });
+
+ return res.status(200).json({
+ success: true,
+ data: responseData,
+ });
+
+ } catch (error) {
+ if (error instanceof z.ZodError) {
+ return res.status(400).json({
+ success: false,
+ error: "Invalid request parameters: " + error.errors.map(e => e.message).join(", "),
+ });
+ }
+
+ logger.error("Failed to generate crawler params preview", {
+ error: error.message,
+ prompt: req.body.prompt,
+ });
+
+ return res.status(400).json({
+ success: false,
+ error: "Failed to process natural language prompt. Please try rephrasing.",
+ });
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 64b9b5c94..732fd4a30 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -5,6 +5,7 @@ import { searchController } from "../controllers/v2/search";
import { scrapeController } from "../controllers/v2/scrape";
import { batchScrapeController } from "../controllers/v2/batch-scrape";
import { crawlController } from "../controllers/v2/crawl";
+import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preview";
import { crawlStatusController } from "../controllers/v2/crawl-status";
import { mapV2Controller } from "../controllers/v2/map";
import {
@@ -56,6 +57,13 @@ v2Router.post(
wrap(crawlController),
);
+v2Router.post(
+ "/crawl/params-preview",
+ authMiddleware(RateLimiterMode.Crawl),
+ checkCreditsMiddleware(),
+ wrap(crawlParamsPreviewController),
+);
+
v2Router.get(
"/crawl/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),