Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl): surface robots.txt warning reliably
Ademílson Tonato1e640534f007
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex a7a65f86b..16e6d845c 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -475,7 +475,10 @@ export async function crawlStatusController(const robotsBlocked = await redisEvictConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked",);- if (robotsBlocked && robotsBlocked.length > 0) {+ const rbCount = robotsBlocked?.length ?? 0;+ // Emit as separate simple logs so no meta is lost in sinks+ const statusNow = outputBulkA.status ?? "scraping";+ if (rbCount > 0 && statusNow !== "scraping") {warning ="One or more pages were unable to be crawled because the robots.txt file prevented this. Please use the /scrape endpoint instead.";}@@ -486,7 +489,8 @@ export async function crawlStatusController(// Check if we should warn about base domain for crawl resultsconst resultCount = outputBulkA.completed ?? outputBulkA.total ?? outputBulkB.data.length;- if (!warning && resultCount <= 1) {+ const currentStatus = outputBulkA.status ?? "scraping";+ if (!warning && currentStatus !== "scraping" && resultCount <= 1) {// Get the original crawl URL and options from stored crawl dataconst crawl = await getCrawl(req.params.jobId);if (crawl && crawl.originUrl && !isBaseDomain(crawl.originUrl)) {diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 615ae4d8d..ff4ee837f 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -49,6 +49,20 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {});}+export async function recordRobotsBlocked(+ crawlId: string,+ url: string,+) {+ await redisEvictConnection.sadd(+ "crawl:" + crawlId + ":robots_blocked",+ url,+ );+ await redisEvictConnection.expire(+ "crawl:" + crawlId + ":robots_blocked",+ 24 * 60 * 60,+ );+}+export async function markCrawlActive(id: string) {await redisEvictConnection.sadd("active_crawls", id);}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 3e5004162..6fceebf47 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -16,6 +16,7 @@ import {addCrawlJobs,addCrawlJobDone,crawlToCrawler,+ recordRobotsBlocked,finishCrawlKickoff,generateURLPermutations,getCrawl,@@ -337,13 +338,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {// Store robots blocked URLs in Redis setfor (const [url, reason] of links.denialReasons) {if (reason === "URL blocked by robots.txt") {- await redisEvictConnection.sadd(- "crawl:" + job.data.crawl_id + ":robots_blocked",- url,- );- await redisEvictConnection.expire(- "crawl:" + job.data.crawl_id + ":robots_blocked",- 24 * 60 * 60,+ await recordRobotsBlocked(+ job.data.crawl_id,+ url);}}@@ -547,6 +544,23 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {logger.info(`🐂 Job done ${job.id}`);return data;} catch (error) {+ // Record top-level robots.txt rejections so crawl status can warn+ try {+ if (+ job.data.crawl_id &&+ job.data.crawlerOptions !== null &&+ error instanceof Error &&+ error.message === "URL blocked by robots.txt"+ ) {+ await recordRobotsBlocked(+ job.data.crawl_id,+ job.data.url,+ );+ }+ } catch (e) {+ logger.debug("Failed to record top-level robots block", { e });+ }+if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;