Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: reverted redirect fix
Nicolas39ff49a8f348
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 617483122..140e01429 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -423,14 +423,18 @@ async function processJob(job: Job & { id: string }, token: string) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;if (doc.metadata.url !== undefined && doc.metadata.sourceURL !== undefined && normalizeURL(doc.metadata.url, sc) !== normalizeURL(doc.metadata.sourceURL, sc)) {- logger.debug("Was redirected, removing old URL and locking new URL...");- // Remove the old URL from visited sets- await redisConnection.srem("crawl:" + job.data.crawl_id + ":visited", normalizeURL(doc.metadata.sourceURL, sc));- if (sc.crawlerOptions?.deduplicateSimilarURLs) {- const permutations = generateURLPermutations(doc.metadata.sourceURL).map(x => x.href);- await redisConnection.srem("crawl:" + job.data.crawl_id + ":visited", ...permutations);- }- await redisConnection.srem("crawl:" + job.data.crawl_id + ":visited_unique", normalizeURL(doc.metadata.sourceURL, sc));+ // console.log("Original URL: ", doc.metadata.sourceURL);+ // console.log("New URL: ", doc.metadata.url);+ // console.log("Normalized original URL: ", normalizeURL(doc.metadata.sourceURL, sc));+ // console.log("Normalized new URL: ", normalizeURL(doc.metadata.url, sc));+ // logger.debug("Was redirected, removing old URL and locking new URL...");+ // // Remove the old URL from visited sets+ // await redisConnection.srem("crawl:" + job.data.crawl_id + ":visited", normalizeURL(doc.metadata.sourceURL, sc));+ // if (sc.crawlerOptions?.deduplicateSimilarURLs) {+ // const permutations = generateURLPermutations(doc.metadata.sourceURL).map(x => x.href);+ // await redisConnection.srem("crawl:" + job.data.crawl_id + ":visited", ...permutations);+ // }+ // await redisConnection.srem("crawl:" + job.data.crawl_id + ":visited_unique", normalizeURL(doc.metadata.sourceURL, sc));// Lock the new URLawait lockURL(job.data.crawl_id, sc, doc.metadata.url);