Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update search.ts
Nicolascbe07164397e
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 1648f3d22..7d7bf9d3a 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -18,6 +18,7 @@ import { getScrapeQueue } from "../../services/queue-service";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";async function scrapeSearchResult(searchResult: { url: string; title: string; description: string },@@ -37,6 +38,9 @@ async function scrapeSearchResult(});try {+ if (isUrlBlocked(searchResult.url)) {+ throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE);+ }await addScrapeJob({url: searchResult.url,@@ -75,9 +79,6 @@ async function scrapeSearchResult(description: searchResult.description,url: searchResult.url,metadata: {- title: searchResult.title,- description: searchResult.description,- sourceURL: searchResult.url,statusCode: 0,error: error.message,},@@ -145,10 +146,7 @@ export async function searchController(});}- // Filter out blocked URLs before scraping- searchResults = searchResults.filter((r) => !isUrlBlocked(r.url));-- // Scrape each non-blocked result, handling timeouts individually+ // Scrape each non-blocked result, handling timeouts individuallyconst scrapePromises = searchResults.map((result) =>scrapeSearchResult(result, {teamId: req.auth.team_id,