Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fixed developer.notion special case
rafaelsideguidec1f98d037139
diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.tsindex e2f8d8cc1..e5841978d 100644--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts+++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts@@ -5,7 +5,7 @@ export async function handleCustomScraping(url: string): Promise<{ scraper: string; url: string; waitAfterLoad?: number, pageOptions?: { scrollXPaths?: string[] } } | null> {// Check for Readme Docs special case- if (text.includes('<meta name="readme-deploy"')) {+ if (text.includes('<meta name="readme-deploy"') && !url.includes('developers.notion.com')) {Logger.debug(`Special use case detected for ${url}, using Fire Engine with wait time 1000ms`);