Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl-redis/lockURL): only add to visited_unique if lock succeeds

Gergő Móricz37f58efe457d
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 0c9e0ff0d..602d13b32 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -233,13 +233,6 @@ export async function lockURL(
url = normalizeURL(url, sc);
logger = logger.child({ url });
- await redisConnection.sadd("crawl:" + id + ":visited_unique", url);
- await redisConnection.expire(
- "crawl:" + id + ":visited_unique",
- 24 * 60 * 60,
- "NX",
- );
-
let res: boolean;
if (!sc.crawlerOptions?.deduplicateSimilarURLs) {
res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0;
@@ -255,6 +248,15 @@ export async function lockURL(
await redisConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60, "NX");
+ if (res) {
+ await redisConnection.sadd("crawl:" + id + ":visited_unique", url);
+ await redisConnection.expire(
+ "crawl:" + id + ":visited_unique",
+ 24 * 60 * 60,
+ "NX",
+ );
+ }
+
logger.debug("Locking URL " + JSON.stringify(url) + "... result: " + res, {
res,
});