Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix ignoreQueryParameters being ignored in URL deduplication - ENG-2804 (#1846)
devin-ai-integration[bot]4268470340a8
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex c3fe5a0cd..2bf7f12bd 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -212,7 +212,7 @@ export async function getCrawlJobCount(id: string): Promise<number> {export function normalizeURL(url: string, sc: StoredCrawl): string {const urlO = new URL(url);- if (!sc.crawlerOptions || sc.crawlerOptions.ignoreQueryParameters) {+ if (sc && sc.crawlerOptions && sc.crawlerOptions.ignoreQueryParameters) {urlO.search = "";}urlO.hash = "";@@ -283,6 +283,8 @@ export async function lockURL(sc: StoredCrawl,url: string,): Promise<boolean> {+ url = normalizeURL(url, sc);+if (typeof sc.crawlerOptions?.limit === "number") {if ((await redisEvictConnection.scard("crawl:" + id + ":visited_unique")) >=