Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update crawl.ts
Nicolas306e697be3d8
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 52215fe46..9e68a5099 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -86,10 +86,18 @@ export async function crawlController(}}- const finalCrawlerOptions = {- ...promptGeneratedOptions,- ...crawlerOptions,- };+ // Merge behavior:+ // - Start with parsed crawlerOptions (which contains schema defaults)+ // - Overlay promptGeneratedOptions ONLY for fields the user did not explicitly provide+ // in the original request (preNormalizedBody) or provided as null/undefined.+ // This prevents empty defaults like [] from overwriting meaningful prompt-generated values.+ const finalCrawlerOptions: any = { ...crawlerOptions };+ for (const [key, value] of Object.entries(promptGeneratedOptions)) {+ const userProvided = Object.prototype.hasOwnProperty.call(preNormalizedBody, key);+ if (!userProvided || preNormalizedBody[key] === undefined || preNormalizedBody[key] === null) {+ finalCrawlerOptions[key] = value;+ }+ }if (Array.isArray(finalCrawlerOptions.includePaths)) {for (const x of finalCrawlerOptions.includePaths) {