Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl-redis): normalize URL before locking
Gergő Móriczfe721fffbef3
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 9240018e4..6d578e5e4 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -1,5 +1,6 @@import { WebCrawler } from "../scraper/WebScraper/crawler";import { redisConnection } from "../services/queue-service";+import { Logger } from "./logger";export type StoredCrawl = {originUrl: string;@@ -88,6 +89,16 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promisereturn false;}}++ try {+ const urlO = new URL(url);+ urlO.search = "";+ urlO.hash = "";+ url = urlO.href;+ } catch (error) {+ Logger.warn("Failed to normalize URL " + JSON.stringify(url) + ": " + error);+ }+const res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0await redisConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60, "NX");return res;@@ -95,6 +106,19 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise/// NOTE: does not check limit. only use if limit is checked beforehand e.g. with sitemapexport async function lockURLs(id: string, urls: string[]): Promise<boolean> {+ urls = urls.map(url => {+ try {+ const urlO = new URL(url);+ urlO.search = "";+ urlO.hash = "";+ return urlO.href;+ } catch (error) {+ Logger.warn("Failed to normalize URL " + JSON.stringify(url) + ": " + error);+ }++ return url;+ });+const res = (await redisConnection.sadd("crawl:" + id + ":visited", ...urls)) !== 0await redisConnection.expire("crawl:" + id + ":visited", 24 * 60 * 60, "NX");return res;