Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Skip Markdown Check for Big HTML Documents (#2622)

Abimael Martell46d9ced9bbc4
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index f81cc9c24..3d4c9a38f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -333,6 +333,8 @@ type EngineScrapeResultWithContext = {
result: EngineScrapeResult;
};
+const MAX_HTML_SIZE_FOR_MARKDOWN_CHECK = 300 * 1024; // 300KB
+
async function scrapeURLLoopIter(
meta: Meta,
engine: Engine,
@@ -359,6 +361,9 @@ async function scrapeURLLoopIter(
hasMarkdown || hasChangeTracking || hasJson || hasSummary;
let checkMarkdown: string;
+ const htmlSize = engineResult.html?.length ?? 0;
+ const shouldSkipMarkdownCheck = htmlSize > MAX_HTML_SIZE_FOR_MARKDOWN_CHECK;
+
if (
meta.internalOptions.teamId === "sitemap" ||
meta.internalOptions.teamId === "robots-txt"
@@ -366,6 +371,16 @@ async function scrapeURLLoopIter(
checkMarkdown = engineResult.html?.trim() ?? "";
} else if (!needsMarkdown) {
checkMarkdown = engineResult.html?.trim() ?? "";
+ } else if (shouldSkipMarkdownCheck) {
+ // Skip markdown conversion for large HTML to avoid slowdowns
+ meta.logger.debug(
+ "Skipping markdown conversion for quality check due to large HTML size",
+ {
+ htmlSize,
+ threshold: MAX_HTML_SIZE_FOR_MARKDOWN_CHECK,
+ },
+ );
+ checkMarkdown = engineResult.html?.trim() ?? "";
} else {
const requestId = meta.id || meta.internalOptions.crawlId;
checkMarkdown = await parseMarkdown(