Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix robots.txt parser panic with content type validation (#1843)
devin-ai-integration[bot]0f477beafd02
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsindex c1f661137..45d70240e 100644--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ b/apps/api/sharedLibs/crawler/src/lib.rs@@ -509,6 +509,7 @@ mod tests {assert_eq!(result.instructions[0].urls[0], "https://example.com/sitemap1.xml");assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");}+}/// Frees a string allocated in Rust-land.diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 153e8425b..468850860 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -31,6 +31,16 @@ export async function fetchRobotsTxt(...extraArgs,});+ const contentType = (Object.entries(response.headers).find(+ (x) => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? "";++ if (contentType.includes("text/html") ||+ contentType.includes("application/json") ||+ contentType.includes("application/xml")) {+ return "";+ }+return response.data;}