Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add warnings for low results and robots.txt restrictions in map and crawl controllers

Ademílson Tonato75debd6f6993
diff --git a/apps/api/src/__tests__/snips/v1/map.test.ts b/apps/api/src/__tests__/snips/v1/map.test.ts
index f4f820848..b7fa45320 100644
--- a/apps/api/src/__tests__/snips/v1/map.test.ts
+++ b/apps/api/src/__tests__/snips/v1/map.test.ts
@@ -69,4 +69,31 @@ describe("Map tests", () => {
},
60000,
);
+
+ it.concurrent(
+ "shows warning when results ≤ 1 and URL is not base domain",
+ async () => {
+ // Use a mock that returns 0 or 1 results to test the warning
+ const response = await map(
+ {
+ url: "https://example.com/some/path",
+ useMock: "map-empty", // This should return 0 or 1 results
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+
+ // If we get ≤ 1 results and the URL is not a base domain, we should get a warning
+ if (response.body.links.length <= 1) {
+ expect(response.body.warning).toBeDefined();
+ expect(response.body.warning).toContain("Only");
+ expect(response.body.warning).toContain("result(s) found");
+ expect(response.body.warning).toContain("base domain");
+ expect(response.body.warning).toContain("example.com");
+ }
+ },
+ 60000,
+ );
});
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index d81e7a74e..f202ff55e 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -388,4 +388,26 @@ describe("Crawl tests", () => {
);
});
}
+
+ it.concurrent(
+ "shows warning when robots.txt blocks URLs",
+ async () => {
+ // Test with a site that has robots.txt blocking some paths
+ const results = await crawl(
+ {
+ url: "https://mairistumpf.com",
+ limit: 5,
+ ignoreRobotsTxt: false, // Respect robots.txt
+ },
+ identity,
+ );
+
+ // Check if warning is present when robots.txt blocks URLs
+ if (results.warning) {
+ expect(results.warning).toContain("robots.txt");
+ expect(results.warning).toContain("/scrape endpoint");
+ }
+ },
+ 10 * scrapeTimeout,
+ );
});
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.ts
index 9275fe08d..650c881c3 100644
--- a/apps/api/src/__tests__/snips/v2/map.test.ts
+++ b/apps/api/src/__tests__/snips/v2/map.test.ts
@@ -84,4 +84,31 @@ describe("Map tests", () => {
expect(response.body.success).toBe(true);
expect(response.body.links.length).toBe(10);
});
+
+ it.concurrent(
+ "shows warning when results ≤ 1 and URL is not base domain",
+ async () => {
+ // Use a mock that returns 0 or 1 results to test the warning
+ const response = await map(
+ {
+ url: "https://example.com/some/path",
+ useMock: "map-empty", // This should return 0 or 1 results
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+
+ // If we get ≤ 1 results and the URL is not a base domain, we should get a warning
+ if (response.body.links.length <= 1) {
+ expect(response.body.warning).toBeDefined();
+ expect(response.body.warning).toContain("Only");
+ expect(response.body.warning).toContain("result(s) found");
+ expect(response.body.warning).toContain("base domain");
+ expect(response.body.warning).toContain("example.com");
+ }
+ },
+ 60000,
+ );
});
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index ea3ba4ccc..9a83169b5 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -18,6 +18,7 @@ import {
} from "../../lib/validateUrl";
import { fireEngineMap } from "../../search/fireEngine";
import { billTeam } from "../../services/billing/credit_billing";
+import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { logJob } from "../../services/logging/log_job";
import { performCosineSimilarity } from "../../lib/map-cosine";
import { logger } from "../../lib/logger";
@@ -474,10 +475,20 @@ export async function mapController(
linksCount: result.links.length,
});
+ // Check if we should warn about base domain
+ let warning: string | undefined;
+ if (result.links.length <= 1 && !isBaseDomain(req.body.url)) {
+ const baseDomain = extractBaseDomain(req.body.url);
+ if (baseDomain) {
+ warning = `Only ${result.links.length} result(s) found. For broader coverage, try mapping the base domain: ${baseDomain}`;
+ }
+ }
+
const response = {
success: true as const,
links: result.links,
scrape_id: result.scrape_id,
+ ...(warning && { warning }),
};
return res.status(200).json(response);
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 1f7d8a4da..ccd612d1f 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1101,6 +1101,7 @@ export type MapResponse =
success: true;
links: string[];
scrape_id?: string;
+ warning?: string;
};
export type CrawlStatusParams = {
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 03255e85e..6d7172da4 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -23,6 +23,7 @@ import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";
import { ScrapeJobSingleUrls } from "../../types";
+import { redisEvictConnection } from "../../services/redis";
configDotenv();
export type PseudoJob<T> = {
@@ -467,6 +468,21 @@ export async function crawlStatusController(
};
}
+ // Check for robots.txt blocked URLs and add warning if found
+ let warning: string | undefined;
+ try {
+ const robotsBlocked = await redisEvictConnection.smembers(
+ "crawl:" + req.params.jobId + ":robots_blocked",
+ );
+ if (robotsBlocked && robotsBlocked.length > 0) {
+ warning =
+ "One or more pages were unable to be crawled because the robots.txt file prevented this. Please use the /scrape endpoint instead.";
+ }
+ } catch (error) {
+ // If we can't check robots blocked URLs, continue without warning
+ logger.debug("Failed to check robots blocked URLs", { error });
+ }
+
return res.status(200).json({
success: true,
status: outputBulkA.status ?? "scraping",
@@ -476,5 +492,6 @@ export async function crawlStatusController(
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
next: outputBulkB.next,
data: outputBulkB.data,
+ ...(warning && { warning }),
});
}
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index b747a643d..ca6dbd3c0 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -14,6 +14,7 @@ import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
import { getMapResults, MapResult } from "../../lib/map-utils";
import { v4 as uuidv4 } from "uuid";
+import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
configDotenv();
@@ -159,9 +160,19 @@ export async function mapController(
linksCount: result.mapResults.length,
});
+ // Check if we should warn about base domain
+ let warning: string | undefined;
+ if (result.mapResults.length <= 1 && !isBaseDomain(req.body.url)) {
+ const baseDomain = extractBaseDomain(req.body.url);
+ if (baseDomain) {
+ warning = `Only ${result.mapResults.length} result(s) found. For broader coverage, try mapping the base domain: ${baseDomain}`;
+ }
+ }
+
const response = {
success: true as const,
links: result.mapResults,
+ ...(warning && { warning }),
};
return res.status(200).json(response);
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index bb9e20f87..28a681195 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -976,6 +976,7 @@ export type MapResponse =
| {
success: true;
links?: MapDocument[];
+ warning?: string;
};
export type CrawlStatusParams = {
@@ -1005,6 +1006,7 @@ export type CrawlStatusResponse =
expiresAt: string;
next?: string;
data: Document[];
+ warning?: string;
};
export type OngoingCrawlsResponse =
diff --git a/apps/api/src/lib/__tests__/url-utils.test.ts b/apps/api/src/lib/__tests__/url-utils.test.ts
new file mode 100644
index 000000000..1c9ead0b6
--- /dev/null
+++ b/apps/api/src/lib/__tests__/url-utils.test.ts
@@ -0,0 +1,57 @@
+import { isBaseDomain, extractBaseDomain } from "../url-utils";
+
+describe("URL Utils", () => {
+ describe("isBaseDomain", () => {
+ it("should return true for base domains", () => {
+ expect(isBaseDomain("https://example.com")).toBe(true);
+ expect(isBaseDomain("https://www.example.com")).toBe(true);
+ expect(isBaseDomain("http://example.com")).toBe(true);
+ expect(isBaseDomain("https://example.com/")).toBe(true);
+ });
+
+ it("should return false for subdomains", () => {
+ expect(isBaseDomain("https://blog.example.com")).toBe(false);
+ expect(isBaseDomain("https://api.example.com")).toBe(false);
+ expect(isBaseDomain("https://www.blog.example.com")).toBe(false);
+ });
+
+ it("should return false for URLs with paths", () => {
+ expect(isBaseDomain("https://example.com/path")).toBe(false);
+ expect(isBaseDomain("https://example.com/path/to/page")).toBe(false);
+ expect(isBaseDomain("https://example.com/path?query=1")).toBe(false);
+ });
+
+ it("should return false for invalid URLs", () => {
+ expect(isBaseDomain("not-a-url")).toBe(false);
+ expect(isBaseDomain("")).toBe(false);
+ });
+ });
+
+ describe("extractBaseDomain", () => {
+ it("should extract base domain correctly", () => {
+ expect(extractBaseDomain("https://example.com")).toBe("example.com");
+ expect(extractBaseDomain("https://www.example.com")).toBe("example.com");
+ expect(extractBaseDomain("https://blog.example.com")).toBe("example.com");
+ expect(extractBaseDomain("https://api.example.com")).toBe("example.com");
+ expect(extractBaseDomain("https://example.com/path")).toBe("example.com");
+ });
+
+ it("should handle complex domains", () => {
+ expect(extractBaseDomain("https://subdomain.example.co.uk")).toBe(
+ "example.co.uk",
+ );
+ expect(extractBaseDomain("https://www.example.co.uk")).toBe(
+ "example.co.uk",
+ );
+ expect(extractBaseDomain("https://subdomain.example.com")).toBe(
+ "example.com",
+ );
+ expect(extractBaseDomain("https://www.example.com")).toBe("example.com");
+ });
+
+ it("should return null for invalid URLs", () => {
+ expect(extractBaseDomain("not-a-url")).toBe(null);
+ expect(extractBaseDomain("")).toBe(null);
+ });
+ });
+});
diff --git a/apps/api/src/lib/url-utils.ts b/apps/api/src/lib/url-utils.ts
new file mode 100644
index 000000000..d46ae24db
--- /dev/null
+++ b/apps/api/src/lib/url-utils.ts
@@ -0,0 +1,90 @@
+/**
+ * Utility functions for URL manipulation and analysis
+ */
+
+/**
+ * Determines if a URL is a base domain (e.g., example.com vs blog.example.com or example.com/path)
+ * A base domain is considered to be the root domain without subdomains or paths
+ * @param url - The URL to check
+ * @returns true if the URL is a base domain, false otherwise
+ */
+export function isBaseDomain(url: string): boolean {
+ try {
+ const urlObj = new URL(url);
+ const hostname = urlObj.hostname;
+
+ // Remove www. prefix for consistency
+ const cleanHostname = hostname.startsWith("www.")
+ ? hostname.slice(4)
+ : hostname;
+
+ // Check if there are subdomains (more than 2 parts when split by dots)
+ const parts = cleanHostname.split(".");
+ if (parts.length > 2) {
+ return false; // Has subdomains
+ }
+
+ // Check if there's a path beyond the root
+ const pathname = urlObj.pathname;
+ if (pathname && pathname !== "/" && pathname.trim() !== "") {
+ return false; // Has a path
+ }
+
+ return true;
+ } catch (error) {
+ // If URL parsing fails, consider it not a base domain
+ return false;
+ }
+}
+
+/**
+ * Extracts the base domain from a URL
+ * @param url - The URL to extract base domain from
+ * @returns The base domain (e.g., "example.com") or null if extraction fails
+ */
+export function extractBaseDomain(url: string): string | null {
+ try {
+ const urlObj = new URL(url);
+ const hostname = urlObj.hostname;
+
+ // Remove www. prefix for consistency
+ const cleanHostname = hostname.startsWith("www.")
+ ? hostname.slice(4)
+ : hostname;
+
+ // Extract the base domain
+ const parts = cleanHostname.split(".");
+ if (parts.length >= 2) {
+ // Handle special cases for multi-part TLDs like .co.uk, .com.au, etc.
+ const lastTwoParts = parts.slice(-2).join(".");
+
+ // Common multi-part TLDs that should be treated as single TLD
+ const multiPartTlds = [
+ "co.uk",
+ "com.au",
+ "org.uk",
+ "net.uk",
+ "ac.uk",
+ "gov.uk",
+ "co.nz",
+ "com.br",
+ "co.jp",
+ "co.kr",
+ "co.in",
+ "co.za",
+ ];
+
+ if (multiPartTlds.includes(lastTwoParts) && parts.length >= 3) {
+ // For domains like subdomain.example.co.uk, return example.co.uk
+ return parts.slice(-3).join(".");
+ } else {
+ // For regular domains like example.com, return the last 2 parts
+ return lastTwoParts;
+ }
+ }
+
+ return cleanHostname;
+ } catch (error) {
+ return null;
+ }
+}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 674929d76..3e5004162 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -25,6 +25,7 @@ import {
saveCrawl,
StoredCrawl,
} from "../../lib/crawl-redis";
+import { redisEvictConnection } from "../redis";
import {
_addScrapeJobToBullMQ,
addScrapeJob,
@@ -39,7 +40,6 @@ import { createWebhookSender, WebhookEvent } from "../webhook";
import { CustomError } from "../../lib/custom-error";
import { startWebScraperPipeline } from "../../main/runWebScraper";
import { CostTracking } from "../../lib/cost-tracking";
-import { redisEvictConnection } from "../redis";
import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
@@ -334,6 +334,20 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
linksLength: links.links.length,
});
+ // Store robots blocked URLs in Redis set
+ for (const [url, reason] of links.denialReasons) {
+ if (reason === "URL blocked by robots.txt") {
+ await redisEvictConnection.sadd(
+ "crawl:" + job.data.crawl_id + ":robots_blocked",
+ url,
+ );
+ await redisEvictConnection.expire(
+ "crawl:" + job.data.crawl_id + ":robots_blocked",
+ 24 * 60 * 60,
+ );
+ }
+ }
+
for (const link of links.links) {
if (await lockURL(job.data.crawl_id, sc, link)) {
// This seems to work really welel