Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl): only extend URLs at pre-finish if changeTracking is on (#1837)
Gergő Móricz3ecbc2f03527
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex f900f708d..73f334e76 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -149,7 +149,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {let lastUrls: string[] = [];const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (useDbAuthentication) {+ if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {lastUrls = ((await supabase_service.rpc("diff_get_last_crawl_urls", {