Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): sitemap / robots 404 handling + skip markdown check for robots

delong35625f480baec
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 1a67051d2..6a53018d3 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -84,6 +84,11 @@ export async function fetchRobotsTxt(
) {
content = response.document.rawHtml!;
} else {
+ if (response.success && response.document.metadata.statusCode === 404) {
+ logger.warn("Robots.txt not found", { robotsTxtUrl }); // should probably index 404 robots.txt
+ return { content: "", url: robotsTxtUrl };
+ }
+
logger.error(`Request failed for robots.txt fetch`, {
method: "fetchRobotsTxt",
robotsTxtUrl,
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 348358121..7c28a867e 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -131,6 +131,14 @@ export async function getLinksFromSitemap(
) {
content = response.document.rawHtml!;
} else {
+ if (
+ response.success &&
+ response.document.metadata.statusCode === 404
+ ) {
+ logger.warn("Sitemap not found", { sitemapUrl }); // should probably index 404 sitemaps
+ return 0;
+ }
+
logger.error(`Request failed for sitemap fetch`, {
method: "getLinksFromSitemap",
mode,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 72b012a47..3967437c1 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -384,7 +384,10 @@ async function scrapeURLLoopIter(
);
let checkMarkdown: string;
- if (meta.internalOptions.teamId === "sitemap") {
+ if (
+ meta.internalOptions.teamId === "sitemap" ||
+ meta.internalOptions.teamId === "robots-txt"
+ ) {
checkMarkdown = engineResult.html?.trim() ?? "";
} else {
checkMarkdown = await parseMarkdown(