Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update search.ts
Nicolas7ce780ac8113
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 37b469c94..ab1975c68 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -112,7 +112,6 @@ export async function searchController(});// Filter blocked URLs early to avoid unnecessary billing- searchResults = searchResults.filter((r) => !isUrlBlocked(r.url));if (searchResults.length > limit) {searchResults = searchResults.slice(0, limit);}@@ -152,7 +151,10 @@ export async function searchController(});}- // Scrape each result, handling timeouts individually+ // Filter out blocked URLs before scraping+ searchResults = searchResults.filter((r) => !isUrlBlocked(r.url));++ // Scrape each non-blocked result, handling timeouts individuallyconst scrapePromises = searchResults.map((result) =>scrapeSearchResult(result, {teamId: req.auth.team_id,