Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasdc0a49e5a747
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex 93962e910..e41ba1b82 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -4,7 +4,7 @@ import { ErrorResponse, RequestWithAuth } from "./types";import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";-import { getMapResults } from "../../lib/map-utils";+import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";// Define the request schema for params preview// Only url and prompt are required/relevant for preview@@ -61,35 +61,20 @@ export async function crawlParamsPreviewController(prompt: parsedBody.prompt,});- // First, get the website structure by mapping URLs- logger.debug("Getting website structure for crawl params preview");- const mapResult = await getMapResults({- url: parsedBody.url,- limit: 120, // Limit to 120 URLs as requested- includeSubdomains: true,- crawlerOptions: {},- teamId: req.auth.team_id,- flags: req.acuc?.flags ?? null,- allowExternalLinks: false,- useIndex: true,- maxFireEngineResults: 500, // Use smaller limit for preview- });-- const websiteUrls = mapResult.mapResults.map(doc => doc.url);-- logger.debug("Found website URLs for crawl params preview", {- urlCount: websiteUrls.length,- sampleUrls: websiteUrls.slice(0, 5), // Log first 5 URLs for debugging- });-- // Create enhanced prompt with website structure- const enhancedPrompt = `${parsedBody.prompt}----- WEBSITE STRUCTURE ----The website has the following URL structure (${websiteUrls.length} URLs found, here is a sample of the first 120 URLs):-${websiteUrls.slice(0, 120).join('\n')}--Based on this structure and the user's request, generate appropriate crawler options.`;+ // Build enhanced prompt with website structure+ const { prompt: enhancedPrompt, websiteUrls } =+ await buildPromptWithWebsiteStructure({+ basePrompt: parsedBody.prompt,+ url: parsedBody.url,+ teamId: req.auth.team_id,+ flags: req.acuc?.flags ?? null,+ logger,+ limit: 120,+ includeSubdomains: true,+ allowExternalLinks: false,+ useIndex: true,+ maxFireEngineResults: 500,+ });// Generate crawler options from enhanced promptconst costTracking = new CostTracking();diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 1982ad8b4..a297c55e5 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -13,6 +13,7 @@ import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";+import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -64,9 +65,22 @@ export async function crawlController(let promptGeneratedOptions = {};if (req.body.prompt) {try {+ // Enhance prompt with discovered site URLs (up to 120) to improve option generation+ const { prompt: enhancedPrompt } = await buildPromptWithWebsiteStructure({+ basePrompt: req.body.prompt,+ url: req.body.url,+ teamId: req.auth.team_id,+ flags: req.acuc?.flags ?? null,+ logger,+ limit: 120,+ includeSubdomains: true,+ allowExternalLinks: false,+ useIndex: true,+ maxFireEngineResults: 500,+ });const costTracking = new CostTracking();const { extract } = await generateCrawlerOptionsFromPrompt(- req.body.prompt,+ enhancedPrompt,logger,costTracking,{ teamId: req.auth.team_id, crawlId: id },diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 058f4b20a..06fc262ec 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -6,6 +6,7 @@ import { fireEngineMap } from "../search/fireEngine";import { redisEvictConnection } from "../services/redis";import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../services/index";import { performCosineSimilarityV2 } from "./map-cosine";+import { Logger } from "winston";// Max Links that "Smart /map" can returnconst MAX_FIRE_ENGINE_RESULTS = 500;@@ -322,3 +323,60 @@ export async function getMapResults({time_taken: totalTimeMs,};}++export async function buildPromptWithWebsiteStructure({+ basePrompt,+ url,+ teamId,+ flags,+ logger,+ limit = 120,+ includeSubdomains = true,+ allowExternalLinks = false,+ useIndex = true,+ maxFireEngineResults = 500,+}: {+ basePrompt: string;+ url: string;+ teamId: string;+ flags: TeamFlags | null;+ logger: Logger;+ limit?: number;+ includeSubdomains?: boolean;+ allowExternalLinks?: boolean;+ useIndex?: boolean;+ maxFireEngineResults?: number;+}): Promise<{ prompt: string; websiteUrls: string[] }> {+ try {+ logger.debug("Getting website structure for prompt enhancement");+ const mapResult = await getMapResults({+ url,+ limit,+ includeSubdomains,+ crawlerOptions: {},+ teamId,+ flags,+ allowExternalLinks,+ useIndex,+ maxFireEngineResults,+ });++ const websiteUrls = mapResult.mapResults.map(doc => doc.url);+ logger.debug("Found website URLs for prompt enhancement", {+ urlCount: websiteUrls.length,+ sampleUrls: websiteUrls.slice(0, 5),+ });++ const prompt = `${basePrompt}\n\n--- WEBSITE STRUCTURE ---\nThe website has the following URL structure (${websiteUrls.length} URLs found, here is a sample of the first ${Math.min(+ 120,+ websiteUrls.length,+ )} URLs):\n${websiteUrls.slice(0, 120).join("\n")}\n\nBased on this structure and the user's request, generate appropriate crawler options.`;++ return { prompt, websiteUrls };+ } catch (e) {+ logger.warn("Failed to get website structure for prompt enhancement", {+ error: (e as any)?.message ?? e,+ });+ return { prompt: basePrompt, websiteUrls: [] };+ }+}