Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): engines for robots and scrape + reduced sitemap limit (#2545)
delong307ace0549654
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex ab2194f24..9e5e1f6d2 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -5,6 +5,9 @@ import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";import { scrapeURL } from "../scraper/scrapeURL";import { Engine } from "../scraper/scrapeURL/engines";import { CostTracking } from "./cost-tracking";+import { useIndex } from "../services";++const ROBOTS_MAX_AGE = 1 * 24 * 60 * 60 * 1000;const useFireEngine =config.FIRE_ENGINE_BETA_URL !== "" &&@@ -36,10 +39,14 @@ export async function fetchRobotsTxt(const shouldPrioritizeFireEngine = location && useFireEngine;const forceEngine: Engine[] = [+ ...(useIndex ? ["index" as const] : []),...(shouldPrioritizeFireEngine? ["fire-engine;tlsclient" as const,"fire-engine;tlsclient;stealth" as const,+ // final fallback to chrome-cdp to fill the index+ "fire-engine;chrome-cdp" as const,+ "fire-engine;chrome-cdp;stealth" as const,]: []),"fetch",@@ -47,6 +54,9 @@ export async function fetchRobotsTxt(? ["fire-engine;tlsclient" as const,"fire-engine;tlsclient;stealth" as const,+ // final fallback to chrome-cdp to fill the index+ "fire-engine;chrome-cdp" as const,+ "fire-engine;chrome-cdp;stealth" as const,]: []),];@@ -57,7 +67,8 @@ export async function fetchRobotsTxt(robotsTxtUrl,scrapeOptions.parse({formats: ["rawHtml"],- timeout: 5000,+ timeout: 8000,+ maxAge: ROBOTS_MAX_AGE,...(location ? { location } : {}),}),{diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 410fe79d6..a1e2e2bf6 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -17,8 +17,8 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { ScrapeOptions } from "../../controllers/v2/types";import { filterLinks, filterUrl } from "@mendable/firecrawl-rs";-export const SITEMAP_LIMIT = 100;-const SITEMAP_MAX_AGE = 48 * 60 * 60 * 1000;+export const SITEMAP_LIMIT = 25;+const SITEMAP_MAX_AGE = 7 * 24 * 60 * 60 * 1000;interface FilterResult {allowed: boolean;@@ -44,6 +44,7 @@ interface FilterLinksResult {links: string[];denialReasons: Map<string, string>;}+export class WebCrawler {private jobId: string;private initialUrl: string;diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex 7319d9ce9..1af5c4d48 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -58,6 +58,9 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {? ["fire-engine;tlsclient" as const,"fire-engine;tlsclient;stealth" as const,+ // final fallback to chrome-cdp to fill the index+ "fire-engine;chrome-cdp" as const,+ "fire-engine;chrome-cdp;stealth" as const,]: []),"fetch",@@ -65,6 +68,9 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {? ["fire-engine;tlsclient" as const,"fire-engine;tlsclient;stealth" as const,+ // final fallback to chrome-cdp to fill the index+ "fire-engine;chrome-cdp" as const,+ "fire-engine;chrome-cdp;stealth" as const,]: []),];diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 98db64ec4..0ae5b054d 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -392,6 +392,11 @@ async function scrapeURLLoopIter("Scrape via " +engine +" deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.",+ {+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },+ statusCode: engineResult.statusCode,+ length: engineResult.html?.trim().length ?? 0,+ },);throw new AddFeatureError(["stealthProxy"]);}