Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl): surface robots.txt warning reliably

Ademílson Tonato1e640534f007
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index a7a65f86b..16e6d845c 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -475,7 +475,10 @@ export async function crawlStatusController(
const robotsBlocked = await redisEvictConnection.smembers(
"crawl:" + req.params.jobId + ":robots_blocked",
);
- if (robotsBlocked && robotsBlocked.length > 0) {
+ const rbCount = robotsBlocked?.length ?? 0;
+ // Emit as separate simple logs so no meta is lost in sinks
+ const statusNow = outputBulkA.status ?? "scraping";
+ if (rbCount > 0 && statusNow !== "scraping") {
warning =
"One or more pages were unable to be crawled because the robots.txt file prevented this. Please use the /scrape endpoint instead.";
}
@@ -486,7 +489,8 @@ export async function crawlStatusController(
// Check if we should warn about base domain for crawl results
const resultCount = outputBulkA.completed ?? outputBulkA.total ?? outputBulkB.data.length;
- if (!warning && resultCount <= 1) {
+ const currentStatus = outputBulkA.status ?? "scraping";
+ if (!warning && currentStatus !== "scraping" && resultCount <= 1) {
// Get the original crawl URL and options from stored crawl data
const crawl = await getCrawl(req.params.jobId);
if (crawl && crawl.originUrl && !isBaseDomain(crawl.originUrl)) {
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 615ae4d8d..ff4ee837f 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -49,6 +49,20 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
});
}
+export async function recordRobotsBlocked(
+ crawlId: string,
+ url: string,
+) {
+ await redisEvictConnection.sadd(
+ "crawl:" + crawlId + ":robots_blocked",
+ url,
+ );
+ await redisEvictConnection.expire(
+ "crawl:" + crawlId + ":robots_blocked",
+ 24 * 60 * 60,
+ );
+}
+
export async function markCrawlActive(id: string) {
await redisEvictConnection.sadd("active_crawls", id);
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 3e5004162..6fceebf47 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -16,6 +16,7 @@ import {
addCrawlJobs,
addCrawlJobDone,
crawlToCrawler,
+ recordRobotsBlocked,
finishCrawlKickoff,
generateURLPermutations,
getCrawl,
@@ -337,13 +338,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
// Store robots blocked URLs in Redis set
for (const [url, reason] of links.denialReasons) {
if (reason === "URL blocked by robots.txt") {
- await redisEvictConnection.sadd(
- "crawl:" + job.data.crawl_id + ":robots_blocked",
- url,
- );
- await redisEvictConnection.expire(
- "crawl:" + job.data.crawl_id + ":robots_blocked",
- 24 * 60 * 60,
+ await recordRobotsBlocked(
+ job.data.crawl_id,
+ url
);
}
}
@@ -547,6 +544,23 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
logger.info(`🐂 Job done ${job.id}`);
return data;
} catch (error) {
+ // Record top-level robots.txt rejections so crawl status can warn
+ try {
+ if (
+ job.data.crawl_id &&
+ job.data.crawlerOptions !== null &&
+ error instanceof Error &&
+ error.message === "URL blocked by robots.txt"
+ ) {
+ await recordRobotsBlocked(
+ job.data.crawl_id,
+ job.data.url,
+ );
+ }
+ } catch (e) {
+ logger.debug("Failed to record top-level robots block", { e });
+ }
+
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;