Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): ignoreBlockedURLs (FIR-1954) (#1580)

Gergő Móricz9bb97388e29f
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 4bfa61e49..c9c9bf3c4 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -196,6 +196,10 @@ export async function searchController(
location: req.body.location,
});
+ if (req.body.ignoreInvalidURLs) {
+ searchResults = searchResults.filter((result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null));
+ }
+
logger.info("Searching completed", {
num_results: searchResults.length,
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 029111250..f4a3134d8 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1163,6 +1163,7 @@ export const searchRequestSchema = z
location: z.string().optional(),
origin: z.string().optional().default("api"),
timeout: z.number().int().positive().finite().safe().default(60000),
+ ignoreInvalidURLs: z.boolean().optional().default(false),
scrapeOptions: baseScrapeOptions
.extend({
formats: z