Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): engines for robots and scrape + reduced sitemap limit (#2545)

delong307ace0549654
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index ab2194f24..9e5e1f6d2 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -5,6 +5,9 @@ import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";
import { scrapeURL } from "../scraper/scrapeURL";
import { Engine } from "../scraper/scrapeURL/engines";
import { CostTracking } from "./cost-tracking";
+import { useIndex } from "../services";
+
+const ROBOTS_MAX_AGE = 1 * 24 * 60 * 60 * 1000;
const useFireEngine =
config.FIRE_ENGINE_BETA_URL !== "" &&
@@ -36,10 +39,14 @@ export async function fetchRobotsTxt(
const shouldPrioritizeFireEngine = location && useFireEngine;
const forceEngine: Engine[] = [
+ ...(useIndex ? ["index" as const] : []),
...(shouldPrioritizeFireEngine
? [
"fire-engine;tlsclient" as const,
"fire-engine;tlsclient;stealth" as const,
+ // final fallback to chrome-cdp to fill the index
+ "fire-engine;chrome-cdp" as const,
+ "fire-engine;chrome-cdp;stealth" as const,
]
: []),
"fetch",
@@ -47,6 +54,9 @@ export async function fetchRobotsTxt(
? [
"fire-engine;tlsclient" as const,
"fire-engine;tlsclient;stealth" as const,
+ // final fallback to chrome-cdp to fill the index
+ "fire-engine;chrome-cdp" as const,
+ "fire-engine;chrome-cdp;stealth" as const,
]
: []),
];
@@ -57,7 +67,8 @@ export async function fetchRobotsTxt(
robotsTxtUrl,
scrapeOptions.parse({
formats: ["rawHtml"],
- timeout: 5000,
+ timeout: 8000,
+ maxAge: ROBOTS_MAX_AGE,
...(location ? { location } : {}),
}),
{
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 410fe79d6..a1e2e2bf6 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -17,8 +17,8 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
import { ScrapeOptions } from "../../controllers/v2/types";
import { filterLinks, filterUrl } from "@mendable/firecrawl-rs";
-export const SITEMAP_LIMIT = 100;
-const SITEMAP_MAX_AGE = 48 * 60 * 60 * 1000;
+export const SITEMAP_LIMIT = 25;
+const SITEMAP_MAX_AGE = 7 * 24 * 60 * 60 * 1000;
interface FilterResult {
allowed: boolean;
@@ -44,6 +44,7 @@ interface FilterLinksResult {
links: string[];
denialReasons: Map<string, string>;
}
+
export class WebCrawler {
private jobId: string;
private initialUrl: string;
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
index 7319d9ce9..1af5c4d48 100644
--- a/apps/api/src/scraper/crawler/sitemap.ts
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -58,6 +58,9 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {
? [
"fire-engine;tlsclient" as const,
"fire-engine;tlsclient;stealth" as const,
+ // final fallback to chrome-cdp to fill the index
+ "fire-engine;chrome-cdp" as const,
+ "fire-engine;chrome-cdp;stealth" as const,
]
: []),
"fetch",
@@ -65,6 +68,9 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {
? [
"fire-engine;tlsclient" as const,
"fire-engine;tlsclient;stealth" as const,
+ // final fallback to chrome-cdp to fill the index
+ "fire-engine;chrome-cdp" as const,
+ "fire-engine;chrome-cdp;stealth" as const,
]
: []),
];
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 98db64ec4..0ae5b054d 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -392,6 +392,11 @@ async function scrapeURLLoopIter(
"Scrape via " +
engine +
" deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.",
+ {
+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
+ statusCode: engineResult.statusCode,
+ length: engineResult.html?.trim().length ?? 0,
+ },
);
throw new AddFeatureError(["stealthProxy"]);
}