Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/v1/extract): ignoreInvalidURLs (#1585)
Gergő Móricz749d89a55138
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 59f6f8f2e..4ffaddfe6 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -62,7 +62,9 @@ export async function extractController(const originalRequest = { ...req.body };req.body = extractRequestSchema.parse(req.body);- if (req.body.urls?.some((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null))) {+ const invalidURLs: string[] = req.body.urls?.filter((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null)) ?? [];++ if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {if (!res.headersSent) {return res.status(403).json({success: false,@@ -144,5 +146,8 @@ export async function extractController(success: true,id: extractId,urlTrace: [],+ ...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs ? {+ invalidURLs,+ } : {}),});}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex f4a3134d8..020cd5637 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -478,6 +478,7 @@ export const extractV1Options = z.optional(),agent: agentOptionsExtract.optional(),__experimental_showCostTracking: z.boolean().default(false),+ ignoreInvalidURLs: z.boolean().default(false),}).strict(strictMessage).refine((obj) => obj.urls || obj.prompt, {