Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolasdc0a49e5a747
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index 93962e910..e41ba1b82 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -4,7 +4,7 @@ import { ErrorResponse, RequestWithAuth } from "./types";
import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostTracking } from "../../lib/cost-tracking";
-import { getMapResults } from "../../lib/map-utils";
+import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
// Define the request schema for params preview
// Only url and prompt are required/relevant for preview
@@ -61,35 +61,20 @@ export async function crawlParamsPreviewController(
prompt: parsedBody.prompt,
});
- // First, get the website structure by mapping URLs
- logger.debug("Getting website structure for crawl params preview");
- const mapResult = await getMapResults({
- url: parsedBody.url,
- limit: 120, // Limit to 120 URLs as requested
- includeSubdomains: true,
- crawlerOptions: {},
- teamId: req.auth.team_id,
- flags: req.acuc?.flags ?? null,
- allowExternalLinks: false,
- useIndex: true,
- maxFireEngineResults: 500, // Use smaller limit for preview
- });
-
- const websiteUrls = mapResult.mapResults.map(doc => doc.url);
-
- logger.debug("Found website URLs for crawl params preview", {
- urlCount: websiteUrls.length,
- sampleUrls: websiteUrls.slice(0, 5), // Log first 5 URLs for debugging
- });
-
- // Create enhanced prompt with website structure
- const enhancedPrompt = `${parsedBody.prompt}
-
---- WEBSITE STRUCTURE ---
-The website has the following URL structure (${websiteUrls.length} URLs found, here is a sample of the first 120 URLs):
-${websiteUrls.slice(0, 120).join('\n')}
-
-Based on this structure and the user's request, generate appropriate crawler options.`;
+ // Build enhanced prompt with website structure
+ const { prompt: enhancedPrompt, websiteUrls } =
+ await buildPromptWithWebsiteStructure({
+ basePrompt: parsedBody.prompt,
+ url: parsedBody.url,
+ teamId: req.auth.team_id,
+ flags: req.acuc?.flags ?? null,
+ logger,
+ limit: 120,
+ includeSubdomains: true,
+ allowExternalLinks: false,
+ useIndex: true,
+ maxFireEngineResults: 500,
+ });
// Generate crawler options from enhanced prompt
const costTracking = new CostTracking();
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 1982ad8b4..a297c55e5 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -13,6 +13,7 @@ import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostTracking } from "../../lib/cost-tracking";
import { checkPermissions } from "../../lib/permissions";
+import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -64,9 +65,22 @@ export async function crawlController(
let promptGeneratedOptions = {};
if (req.body.prompt) {
try {
+ // Enhance prompt with discovered site URLs (up to 120) to improve option generation
+ const { prompt: enhancedPrompt } = await buildPromptWithWebsiteStructure({
+ basePrompt: req.body.prompt,
+ url: req.body.url,
+ teamId: req.auth.team_id,
+ flags: req.acuc?.flags ?? null,
+ logger,
+ limit: 120,
+ includeSubdomains: true,
+ allowExternalLinks: false,
+ useIndex: true,
+ maxFireEngineResults: 500,
+ });
const costTracking = new CostTracking();
const { extract } = await generateCrawlerOptionsFromPrompt(
- req.body.prompt,
+ enhancedPrompt,
logger,
costTracking,
{ teamId: req.auth.team_id, crawlId: id },
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 058f4b20a..06fc262ec 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -6,6 +6,7 @@ import { fireEngineMap } from "../search/fireEngine";
import { redisEvictConnection } from "../services/redis";
import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../services/index";
import { performCosineSimilarityV2 } from "./map-cosine";
+import { Logger } from "winston";
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 500;
@@ -322,3 +323,60 @@ export async function getMapResults({
time_taken: totalTimeMs,
};
}
+
+export async function buildPromptWithWebsiteStructure({
+ basePrompt,
+ url,
+ teamId,
+ flags,
+ logger,
+ limit = 120,
+ includeSubdomains = true,
+ allowExternalLinks = false,
+ useIndex = true,
+ maxFireEngineResults = 500,
+}: {
+ basePrompt: string;
+ url: string;
+ teamId: string;
+ flags: TeamFlags | null;
+ logger: Logger;
+ limit?: number;
+ includeSubdomains?: boolean;
+ allowExternalLinks?: boolean;
+ useIndex?: boolean;
+ maxFireEngineResults?: number;
+}): Promise<{ prompt: string; websiteUrls: string[] }> {
+ try {
+ logger.debug("Getting website structure for prompt enhancement");
+ const mapResult = await getMapResults({
+ url,
+ limit,
+ includeSubdomains,
+ crawlerOptions: {},
+ teamId,
+ flags,
+ allowExternalLinks,
+ useIndex,
+ maxFireEngineResults,
+ });
+
+ const websiteUrls = mapResult.mapResults.map(doc => doc.url);
+ logger.debug("Found website URLs for prompt enhancement", {
+ urlCount: websiteUrls.length,
+ sampleUrls: websiteUrls.slice(0, 5),
+ });
+
+ const prompt = `${basePrompt}\n\n--- WEBSITE STRUCTURE ---\nThe website has the following URL structure (${websiteUrls.length} URLs found, here is a sample of the first ${Math.min(
+ 120,
+ websiteUrls.length,
+ )} URLs):\n${websiteUrls.slice(0, 120).join("\n")}\n\nBased on this structure and the user's request, generate appropriate crawler options.`;
+
+ return { prompt, websiteUrls };
+ } catch (e) {
+ logger.warn("Failed to get website structure for prompt enhancement", {
+ error: (e as any)?.message ?? e,
+ });
+ return { prompt: basePrompt, websiteUrls: [] };
+ }
+}