Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl-redis/lockURL): only add to visited_unique if lock succeeds
Gergő Móricz37f58efe457d
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 0c9e0ff0d..602d13b32 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -233,13 +233,6 @@ export async function lockURL(url = normalizeURL(url, sc);logger = logger.child({ url });- await redisConnection.sadd("crawl:" + id + ":visited_unique", url);- await redisConnection.expire(- "crawl:" + id + ":visited_unique",- 24 * 60 * 60,- "NX",- );-let res: boolean;if (!sc.crawlerOptions?.deduplicateSimilarURLs) {res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0;@@ -255,6 +248,15 @@ export async function lockURL(await redisConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60, "NX");+ if (res) {+ await redisConnection.sadd("crawl:" + id + ":visited_unique", url);+ await redisConnection.expire(+ "crawl:" + id + ":visited_unique",+ 24 * 60 * 60,+ "NX",+ );+ }+logger.debug("Locking URL " + JSON.stringify(url) + "... result: " + res, {res,});