Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add warnings for low results and robots.txt restrictions in map and crawl controllers
AdemÃlson Tonato75debd6f6993
diff --git a/apps/api/src/__tests__/snips/v1/map.test.ts b/apps/api/src/__tests__/snips/v1/map.test.tsindex f4f820848..b7fa45320 100644--- a/apps/api/src/__tests__/snips/v1/map.test.ts+++ b/apps/api/src/__tests__/snips/v1/map.test.ts@@ -69,4 +69,31 @@ describe("Map tests", () => {},60000,);++ it.concurrent(+ "shows warning when results ≤ 1 and URL is not base domain",+ async () => {+ // Use a mock that returns 0 or 1 results to test the warning+ const response = await map(+ {+ url: "https://example.com/some/path",+ useMock: "map-empty", // This should return 0 or 1 results+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);++ // If we get ≤ 1 results and the URL is not a base domain, we should get a warning+ if (response.body.links.length <= 1) {+ expect(response.body.warning).toBeDefined();+ expect(response.body.warning).toContain("Only");+ expect(response.body.warning).toContain("result(s) found");+ expect(response.body.warning).toContain("base domain");+ expect(response.body.warning).toContain("example.com");+ }+ },+ 60000,+ );});diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex d81e7a74e..f202ff55e 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -388,4 +388,26 @@ describe("Crawl tests", () => {);});}++ it.concurrent(+ "shows warning when robots.txt blocks URLs",+ async () => {+ // Test with a site that has robots.txt blocking some paths+ const results = await crawl(+ {+ url: "https://mairistumpf.com",+ limit: 5,+ ignoreRobotsTxt: false, // Respect robots.txt+ },+ identity,+ );++ // Check if warning is present when robots.txt blocks URLs+ if (results.warning) {+ expect(results.warning).toContain("robots.txt");+ expect(results.warning).toContain("/scrape endpoint");+ }+ },+ 10 * scrapeTimeout,+ );});diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.tsindex 9275fe08d..650c881c3 100644--- a/apps/api/src/__tests__/snips/v2/map.test.ts+++ b/apps/api/src/__tests__/snips/v2/map.test.ts@@ -84,4 +84,31 @@ describe("Map tests", () => {expect(response.body.success).toBe(true);expect(response.body.links.length).toBe(10);});++ it.concurrent(+ "shows warning when results ≤ 1 and URL is not base domain",+ async () => {+ // Use a mock that returns 0 or 1 results to test the warning+ const response = await map(+ {+ url: "https://example.com/some/path",+ useMock: "map-empty", // This should return 0 or 1 results+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);++ // If we get ≤ 1 results and the URL is not a base domain, we should get a warning+ if (response.body.links.length <= 1) {+ expect(response.body.warning).toBeDefined();+ expect(response.body.warning).toContain("Only");+ expect(response.body.warning).toContain("result(s) found");+ expect(response.body.warning).toContain("base domain");+ expect(response.body.warning).toContain("example.com");+ }+ },+ 60000,+ );});diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex ea3ba4ccc..9a83169b5 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -18,6 +18,7 @@ import {} from "../../lib/validateUrl";import { fireEngineMap } from "../../search/fireEngine";import { billTeam } from "../../services/billing/credit_billing";+import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";import { logJob } from "../../services/logging/log_job";import { performCosineSimilarity } from "../../lib/map-cosine";import { logger } from "../../lib/logger";@@ -474,10 +475,20 @@ export async function mapController(linksCount: result.links.length,});+ // Check if we should warn about base domain+ let warning: string | undefined;+ if (result.links.length <= 1 && !isBaseDomain(req.body.url)) {+ const baseDomain = extractBaseDomain(req.body.url);+ if (baseDomain) {+ warning = `Only ${result.links.length} result(s) found. For broader coverage, try mapping the base domain: ${baseDomain}`;+ }+ }+const response = {success: true as const,links: result.links,scrape_id: result.scrape_id,+ ...(warning && { warning }),};return res.status(200).json(response);diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 1f7d8a4da..ccd612d1f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1101,6 +1101,7 @@ export type MapResponse =success: true;links: string[];scrape_id?: string;+ warning?: string;};export type CrawlStatusParams = {diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 03255e85e..6d7172da4 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -23,6 +23,7 @@ import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getJobFromGCS } from "../../lib/gcs-jobs";import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";import { ScrapeJobSingleUrls } from "../../types";+import { redisEvictConnection } from "../../services/redis";configDotenv();export type PseudoJob<T> = {@@ -467,6 +468,21 @@ export async function crawlStatusController(};}+ // Check for robots.txt blocked URLs and add warning if found+ let warning: string | undefined;+ try {+ const robotsBlocked = await redisEvictConnection.smembers(+ "crawl:" + req.params.jobId + ":robots_blocked",+ );+ if (robotsBlocked && robotsBlocked.length > 0) {+ warning =+ "One or more pages were unable to be crawled because the robots.txt file prevented this. Please use the /scrape endpoint instead.";+ }+ } catch (error) {+ // If we can't check robots blocked URLs, continue without warning+ logger.debug("Failed to check robots blocked URLs", { error });+ }+return res.status(200).json({success: true,status: outputBulkA.status ?? "scraping",@@ -476,5 +492,6 @@ export async function crawlStatusController(expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),next: outputBulkB.next,data: outputBulkB.data,+ ...(warning && { warning }),});}diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex b747a643d..ca6dbd3c0 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -14,6 +14,7 @@ import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";import { getMapResults, MapResult } from "../../lib/map-utils";import { v4 as uuidv4 } from "uuid";+import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";configDotenv();@@ -159,9 +160,19 @@ export async function mapController(linksCount: result.mapResults.length,});+ // Check if we should warn about base domain+ let warning: string | undefined;+ if (result.mapResults.length <= 1 && !isBaseDomain(req.body.url)) {+ const baseDomain = extractBaseDomain(req.body.url);+ if (baseDomain) {+ warning = `Only ${result.mapResults.length} result(s) found. For broader coverage, try mapping the base domain: ${baseDomain}`;+ }+ }+const response = {success: true as const,links: result.mapResults,+ ...(warning && { warning }),};return res.status(200).json(response);diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex bb9e20f87..28a681195 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -976,6 +976,7 @@ export type MapResponse =| {success: true;links?: MapDocument[];+ warning?: string;};export type CrawlStatusParams = {@@ -1005,6 +1006,7 @@ export type CrawlStatusResponse =expiresAt: string;next?: string;data: Document[];+ warning?: string;};export type OngoingCrawlsResponse =diff --git a/apps/api/src/lib/__tests__/url-utils.test.ts b/apps/api/src/lib/__tests__/url-utils.test.tsnew file mode 100644index 000000000..1c9ead0b6--- /dev/null+++ b/apps/api/src/lib/__tests__/url-utils.test.ts@@ -0,0 +1,57 @@+import { isBaseDomain, extractBaseDomain } from "../url-utils";++describe("URL Utils", () => {+ describe("isBaseDomain", () => {+ it("should return true for base domains", () => {+ expect(isBaseDomain("https://example.com")).toBe(true);+ expect(isBaseDomain("https://www.example.com")).toBe(true);+ expect(isBaseDomain("http://example.com")).toBe(true);+ expect(isBaseDomain("https://example.com/")).toBe(true);+ });++ it("should return false for subdomains", () => {+ expect(isBaseDomain("https://blog.example.com")).toBe(false);+ expect(isBaseDomain("https://api.example.com")).toBe(false);+ expect(isBaseDomain("https://www.blog.example.com")).toBe(false);+ });++ it("should return false for URLs with paths", () => {+ expect(isBaseDomain("https://example.com/path")).toBe(false);+ expect(isBaseDomain("https://example.com/path/to/page")).toBe(false);+ expect(isBaseDomain("https://example.com/path?query=1")).toBe(false);+ });++ it("should return false for invalid URLs", () => {+ expect(isBaseDomain("not-a-url")).toBe(false);+ expect(isBaseDomain("")).toBe(false);+ });+ });++ describe("extractBaseDomain", () => {+ it("should extract base domain correctly", () => {+ expect(extractBaseDomain("https://example.com")).toBe("example.com");+ expect(extractBaseDomain("https://www.example.com")).toBe("example.com");+ expect(extractBaseDomain("https://blog.example.com")).toBe("example.com");+ expect(extractBaseDomain("https://api.example.com")).toBe("example.com");+ expect(extractBaseDomain("https://example.com/path")).toBe("example.com");+ });++ it("should handle complex domains", () => {+ expect(extractBaseDomain("https://subdomain.example.co.uk")).toBe(+ "example.co.uk",+ );+ expect(extractBaseDomain("https://www.example.co.uk")).toBe(+ "example.co.uk",+ );+ expect(extractBaseDomain("https://subdomain.example.com")).toBe(+ "example.com",+ );+ expect(extractBaseDomain("https://www.example.com")).toBe("example.com");+ });++ it("should return null for invalid URLs", () => {+ expect(extractBaseDomain("not-a-url")).toBe(null);+ expect(extractBaseDomain("")).toBe(null);+ });+ });+});diff --git a/apps/api/src/lib/url-utils.ts b/apps/api/src/lib/url-utils.tsnew file mode 100644index 000000000..d46ae24db--- /dev/null+++ b/apps/api/src/lib/url-utils.ts@@ -0,0 +1,90 @@+/**+ * Utility functions for URL manipulation and analysis+ */++/**+ * Determines if a URL is a base domain (e.g., example.com vs blog.example.com or example.com/path)+ * A base domain is considered to be the root domain without subdomains or paths+ * @param url - The URL to check+ * @returns true if the URL is a base domain, false otherwise+ */+export function isBaseDomain(url: string): boolean {+ try {+ const urlObj = new URL(url);+ const hostname = urlObj.hostname;++ // Remove www. prefix for consistency+ const cleanHostname = hostname.startsWith("www.")+ ? hostname.slice(4)+ : hostname;++ // Check if there are subdomains (more than 2 parts when split by dots)+ const parts = cleanHostname.split(".");+ if (parts.length > 2) {+ return false; // Has subdomains+ }++ // Check if there's a path beyond the root+ const pathname = urlObj.pathname;+ if (pathname && pathname !== "/" && pathname.trim() !== "") {+ return false; // Has a path+ }++ return true;+ } catch (error) {+ // If URL parsing fails, consider it not a base domain+ return false;+ }+}++/**+ * Extracts the base domain from a URL+ * @param url - The URL to extract base domain from+ * @returns The base domain (e.g., "example.com") or null if extraction fails+ */+export function extractBaseDomain(url: string): string | null {+ try {+ const urlObj = new URL(url);+ const hostname = urlObj.hostname;++ // Remove www. prefix for consistency+ const cleanHostname = hostname.startsWith("www.")+ ? hostname.slice(4)+ : hostname;++ // Extract the base domain+ const parts = cleanHostname.split(".");+ if (parts.length >= 2) {+ // Handle special cases for multi-part TLDs like .co.uk, .com.au, etc.+ const lastTwoParts = parts.slice(-2).join(".");++ // Common multi-part TLDs that should be treated as single TLD+ const multiPartTlds = [+ "co.uk",+ "com.au",+ "org.uk",+ "net.uk",+ "ac.uk",+ "gov.uk",+ "co.nz",+ "com.br",+ "co.jp",+ "co.kr",+ "co.in",+ "co.za",+ ];++ if (multiPartTlds.includes(lastTwoParts) && parts.length >= 3) {+ // For domains like subdomain.example.co.uk, return example.co.uk+ return parts.slice(-3).join(".");+ } else {+ // For regular domains like example.com, return the last 2 parts+ return lastTwoParts;+ }+ }++ return cleanHostname;+ } catch (error) {+ return null;+ }+}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 674929d76..3e5004162 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -25,6 +25,7 @@ import {saveCrawl,StoredCrawl,} from "../../lib/crawl-redis";+import { redisEvictConnection } from "../redis";import {_addScrapeJobToBullMQ,addScrapeJob,@@ -39,7 +40,6 @@ import { createWebhookSender, WebhookEvent } from "../webhook";import { CustomError } from "../../lib/custom-error";import { startWebScraperPipeline } from "../../main/runWebScraper";import { CostTracking } from "../../lib/cost-tracking";-import { redisEvictConnection } from "../redis";import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";@@ -334,6 +334,20 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {linksLength: links.links.length,});+ // Store robots blocked URLs in Redis set+ for (const [url, reason] of links.denialReasons) {+ if (reason === "URL blocked by robots.txt") {+ await redisEvictConnection.sadd(+ "crawl:" + job.data.crawl_id + ":robots_blocked",+ url,+ );+ await redisEvictConnection.expire(+ "crawl:" + job.data.crawl_id + ":robots_blocked",+ 24 * 60 * 60,+ );+ }+ }+for (const link of links.links) {if (await lockURL(job.data.crawl_id, sc, link)) {// This seems to work really welel