Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): sitemap / robots 404 handling + skip markdown check for robots
delong35625f480baec
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 1a67051d2..6a53018d3 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -84,6 +84,11 @@ export async function fetchRobotsTxt() {content = response.document.rawHtml!;} else {+ if (response.success && response.document.metadata.statusCode === 404) {+ logger.warn("Robots.txt not found", { robotsTxtUrl }); // should probably index 404 robots.txt+ return { content: "", url: robotsTxtUrl };+ }+logger.error(`Request failed for robots.txt fetch`, {method: "fetchRobotsTxt",robotsTxtUrl,diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 348358121..7c28a867e 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -131,6 +131,14 @@ export async function getLinksFromSitemap() {content = response.document.rawHtml!;} else {+ if (+ response.success &&+ response.document.metadata.statusCode === 404+ ) {+ logger.warn("Sitemap not found", { sitemapUrl }); // should probably index 404 sitemaps+ return 0;+ }+logger.error(`Request failed for sitemap fetch`, {method: "getLinksFromSitemap",mode,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 72b012a47..3967437c1 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -384,7 +384,10 @@ async function scrapeURLLoopIter();let checkMarkdown: string;- if (meta.internalOptions.teamId === "sitemap") {+ if (+ meta.internalOptions.teamId === "sitemap" ||+ meta.internalOptions.teamId === "robots-txt"+ ) {checkMarkdown = engineResult.html?.trim() ?? "";} else {checkMarkdown = await parseMarkdown(