Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl-redis): batch scrape lockURL
Móricz Gergőf6db9f142860
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 9f6a8cf4b..b5936ad6d 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -144,7 +144,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promiseawait redisConnection.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60, "NX");let res: boolean;- if (!sc.crawlerOptions.deduplicateSimilarURLs) {+ if (!sc.crawlerOptions?.deduplicateSimilarURLs) {res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0} else {const permutations = generateURLPermutations(url);