Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawler): handle cross-origin redirects differently than same-origin redirects (#1279)
Gergő Móricze8c698d613d0
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 147b69a6f..5e91b46e7 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -109,6 +109,9 @@ export class WebCrawler {// Check if the link exceeds the maximum depth allowedif (depth > maxDepth) {+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} DEPTH FAIL`);+ }return false;}@@ -119,6 +122,9 @@ export class WebCrawler {new RegExp(excludePattern).test(path),)) {+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} EXCLUDE FAIL`);+ }return false;}}@@ -130,6 +136,9 @@ export class WebCrawler {new RegExp(includePattern).test(path),)) {+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} INCLUDE FAIL`);+ }return false;}}@@ -140,6 +149,9 @@ export class WebCrawler {try {normalizedLink = new URL(link);} catch (_) {+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} URL PARSE FAIL`);+ }return false;}const initialHostname = normalizedInitialUrl.hostname.replace(@@ -158,6 +170,9 @@ export class WebCrawler {if (!normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)) {+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`);+ }return false;}}@@ -171,13 +186,22 @@ export class WebCrawler {method: "filterLinks",link,});+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} ROBOTS FAIL`);+ }return false;}if (this.isFile(link)) {+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} FILE FAIL`);+ }return false;}+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ this.logger.debug(`${link} OK`);+ }return true;}).slice(0, limit);diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex c9e5e5127..9405c3460 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -973,7 +973,13 @@ async function processJob(job: Job & { id: string }, token: string) {); // TODO: make this its own error type that is ignored by error tracking}- if (job.data.isCrawlSourceScrape) {+ // Only re-set originUrl if it's different from the current hostname+ // This is only done on this condition to handle cross-domain redirects+ // If this would be done for non-crossdomain redirects, but also for e.g.+ // redirecting / -> /introduction (like our docs site does), it would+ // break crawling the entire site without allowBackwardsCrawling - mogery+ const isHostnameDifferent = normalizeUrlOnlyHostname(doc.metadata.url) !== normalizeUrlOnlyHostname(doc.metadata.sourceURL);+ if (job.data.isCrawlSourceScrape && isHostnameDifferent) {// TODO: re-fetch sitemap for redirect target domainsc.originUrl = doc.metadata.url;await saveCrawl(job.data.crawl_id, sc);