Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas34b40f6a23ab
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 1ea289959..f4bc45b5a 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -33,6 +33,7 @@ export async function scrapeController(basePriority: 10,});+await addScrapeJob({url: req.body.url,@@ -96,7 +97,7 @@ export async function scrapeController(// Don't bill if we're early returningreturn;}- if (req.body.extract && req.body.formats.includes("extract")) {+ if (req.body.extract && req.body.formats.includes("extract") ) {creditsToBeBilled = 5;}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ee141625b..5573fbaa0 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -125,6 +125,7 @@ export const scrapeOptions = z"screenshot","screenshot@fullPage","extract",+ "json"]).array().optional()@@ -139,7 +140,10 @@ export const scrapeOptions = zonlyMainContent: z.boolean().default(true),timeout: z.number().int().positive().finite().safe().optional(),waitFor: z.number().int().nonnegative().finite().safe().default(0),+ // Deprecate this to jsonOptionsextract: extractOptions.optional(),+ // New+ jsonOptions: extractOptions.optional(),mobile: z.boolean().default(false),parsePDF: z.boolean().default(true),actions: actionsSchema.optional(),@@ -242,20 +246,43 @@ export const scrapeRequestSchema = scrapeOptions(obj) => {const hasExtractFormat = obj.formats?.includes("extract");const hasExtractOptions = obj.extract !== undefined;+ const hasJsonFormat = obj.formats?.includes("json");+ const hasJsonOptions = obj.jsonOptions !== undefined;return ((hasExtractFormat && hasExtractOptions) ||- (!hasExtractFormat && !hasExtractOptions)+ (!hasExtractFormat && !hasExtractOptions) ||+ (hasJsonFormat && hasJsonOptions) ||+ (!hasJsonFormat && !hasJsonOptions));},{message:- "When 'extract' format is specified, 'extract' options must be provided, and vice versa",+ "When 'extract' or 'json' format is specified, corresponding options must be provided, and vice versa",},).transform((obj) => {- if ((obj.formats?.includes("extract") || obj.extract) && !obj.timeout) {- return { ...obj, timeout: 60000 };+ // Handle timeout+ if ((obj.formats?.includes("extract") || obj.extract || obj.formats?.includes("json") || obj.jsonOptions) && !obj.timeout) {+ obj = { ...obj, timeout: 60000 };+ }++ if(obj.formats?.includes("json")) {+ obj.formats.push("extract");+ }++ // Convert JSON options to extract options if needed+ if (obj.jsonOptions && !obj.extract) {+ obj = {+ ...obj,+ extract: {+ prompt: obj.jsonOptions.prompt,+ systemPrompt: obj.jsonOptions.systemPrompt,+ schema: obj.jsonOptions.schema,+ mode: "llm"+ }+ };}+return obj;});@@ -410,6 +437,7 @@ export type Document = {links?: string[];screenshot?: string;extract?: any;+ json?: any;warning?: string;actions?: {screenshots?: string[];diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 5fdc3b75c..be97036f0 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -233,7 +233,12 @@ export async function performLLMExtract(document.markdown,document.warning,);- document.extract = extract;++ if (meta.options.formats.includes("json")) {+ document.json = extract;+ } else {+ document.extract = extract;+ }document.warning = warning;}