Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Fix ignoreQueryParameters being ignored in URL deduplication - ENG-2804 (#1846)

devin-ai-integration[bot]4268470340a8
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index c3fe5a0cd..2bf7f12bd 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -212,7 +212,7 @@ export async function getCrawlJobCount(id: string): Promise<number> {
export function normalizeURL(url: string, sc: StoredCrawl): string {
const urlO = new URL(url);
- if (!sc.crawlerOptions || sc.crawlerOptions.ignoreQueryParameters) {
+ if (sc && sc.crawlerOptions && sc.crawlerOptions.ignoreQueryParameters) {
urlO.search = "";
}
urlO.hash = "";
@@ -283,6 +283,8 @@ export async function lockURL(
sc: StoredCrawl,
url: string,
): Promise<boolean> {
+ url = normalizeURL(url, sc);
+
if (typeof sc.crawlerOptions?.limit === "number") {
if (
(await redisEvictConnection.scard("crawl:" + id + ":visited_unique")) >=