Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fixed developer.notion special case

rafaelsideguidec1f98d037139
diff --git a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
index e2f8d8cc1..e5841978d 100644
--- a/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
+++ b/apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
@@ -5,7 +5,7 @@ export async function handleCustomScraping(
url: string
): Promise<{ scraper: string; url: string; waitAfterLoad?: number, pageOptions?: { scrollXPaths?: string[] } } | null> {
// Check for Readme Docs special case
- if (text.includes('<meta name="readme-deploy"')) {
+ if (text.includes('<meta name="readme-deploy"') && !url.includes('developers.notion.com')) {
Logger.debug(
`Special use case detected for ${url}, using Fire Engine with wait time 1000ms`
);