Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Skip Markdown Check for Big HTML Documents (#2622)
Abimael Martell46d9ced9bbc4
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex f81cc9c24..3d4c9a38f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -333,6 +333,8 @@ type EngineScrapeResultWithContext = {result: EngineScrapeResult;};+const MAX_HTML_SIZE_FOR_MARKDOWN_CHECK = 300 * 1024; // 300KB+async function scrapeURLLoopIter(meta: Meta,engine: Engine,@@ -359,6 +361,9 @@ async function scrapeURLLoopIter(hasMarkdown || hasChangeTracking || hasJson || hasSummary;let checkMarkdown: string;+ const htmlSize = engineResult.html?.length ?? 0;+ const shouldSkipMarkdownCheck = htmlSize > MAX_HTML_SIZE_FOR_MARKDOWN_CHECK;+if (meta.internalOptions.teamId === "sitemap" ||meta.internalOptions.teamId === "robots-txt"@@ -366,6 +371,16 @@ async function scrapeURLLoopIter(checkMarkdown = engineResult.html?.trim() ?? "";} else if (!needsMarkdown) {checkMarkdown = engineResult.html?.trim() ?? "";+ } else if (shouldSkipMarkdownCheck) {+ // Skip markdown conversion for large HTML to avoid slowdowns+ meta.logger.debug(+ "Skipping markdown conversion for quality check due to large HTML size",+ {+ htmlSize,+ threshold: MAX_HTML_SIZE_FOR_MARKDOWN_CHECK,+ },+ );+ checkMarkdown = engineResult.html?.trim() ?? "";} else {const requestId = meta.id || meta.internalOptions.crawlId;checkMarkdown = await parseMarkdown(