Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/v1/extract): ignoreInvalidURLs (#1585)

Gergő Móricz749d89a55138
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 59f6f8f2e..4ffaddfe6 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -62,7 +62,9 @@ export async function extractController(
const originalRequest = { ...req.body };
req.body = extractRequestSchema.parse(req.body);
- if (req.body.urls?.some((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null))) {
+ const invalidURLs: string[] = req.body.urls?.filter((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null)) ?? [];
+
+ if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {
if (!res.headersSent) {
return res.status(403).json({
success: false,
@@ -144,5 +146,8 @@ export async function extractController(
success: true,
id: extractId,
urlTrace: [],
+ ...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs ? {
+ invalidURLs,
+ } : {}),
});
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index f4a3134d8..020cd5637 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -478,6 +478,7 @@ export const extractV1Options = z
.optional(),
agent: agentOptionsExtract.optional(),
__experimental_showCostTracking: z.boolean().default(false),
+ ignoreInvalidURLs: z.boolean().default(false),
})
.strict(strictMessage)
.refine((obj) => obj.urls || obj.prompt, {