Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): ignoreBlockedURLs (FIR-1954) (#1580)
Gergő Móricz9bb97388e29f
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 4bfa61e49..c9c9bf3c4 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -196,6 +196,10 @@ export async function searchController(location: req.body.location,});+ if (req.body.ignoreInvalidURLs) {+ searchResults = searchResults.filter((result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null));+ }+logger.info("Searching completed", {num_results: searchResults.length,});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 029111250..f4a3134d8 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1163,6 +1163,7 @@ export const searchRequestSchema = zlocation: z.string().optional(),origin: z.string().optional().default("api"),timeout: z.number().int().positive().finite().safe().default(60000),+ ignoreInvalidURLs: z.boolean().optional().default(false),scrapeOptions: baseScrapeOptions.extend({formats: z