Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Fix robots.txt parser panic with content type validation (#1843)

devin-ai-integration[bot]0f477beafd02
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rs
index c1f661137..45d70240e 100644
--- a/apps/api/sharedLibs/crawler/src/lib.rs
+++ b/apps/api/sharedLibs/crawler/src/lib.rs
@@ -509,6 +509,7 @@ mod tests {
assert_eq!(result.instructions[0].urls[0], "https://example.com/sitemap1.xml");
assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");
}
+
}
/// Frees a string allocated in Rust-land.
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 153e8425b..468850860 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -31,6 +31,16 @@ export async function fetchRobotsTxt(
...extraArgs,
});
+ const contentType = (Object.entries(response.headers).find(
+ (x) => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? "";
+
+ if (contentType.includes("text/html") ||
+ contentType.includes("application/json") ||
+ contentType.includes("application/xml")) {
+ return "";
+ }
+
return response.data;
}