Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fixed nested data inside extract
rafaelmmiller943bbae88d67
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex e5f2595c8..f59b1ff13 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -211,7 +211,9 @@ export async function extractController(prompt: req.body.prompt,schema: req.body.schema,},- docs.map(x => buildDocument(x)).join('\n')+ docs.map(x => buildDocument(x)).join('\n'),+ undefined,+ true // isExtractEndpoint);// TODO: change this laterdiff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 3866683aa..64073eee6 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -58,7 +58,7 @@ function normalizeSchema(x: any): any {}}-export async function generateOpenAICompletions(logger: Logger, options: ExtractOptions, markdown?: string, previousWarning?: string): Promise<{ extract: any, numTokens: number, warning: string | undefined }> {+export async function generateOpenAICompletions(logger: Logger, options: ExtractOptions, markdown?: string, previousWarning?: string, isExtractEndpoint?: boolean): Promise<{ extract: any, numTokens: number, warning: string | undefined }> {let extract: any;let warning: string | undefined;@@ -158,7 +158,12 @@ export async function generateOpenAICompletions(logger: Logger, options: Extractif (extract === null && jsonCompletion.choices[0].message.content !== null) {try {- extract = JSON.parse(jsonCompletion.choices[0].message.content);+ if (!isExtractEndpoint) {+ extract = JSON.parse(jsonCompletion.choices[0].message.content);+ } else {+ const extractData = JSON.parse(jsonCompletion.choices[0].message.content);+ extract = extractData.data.extract;+ }} catch (e) {logger.error("Failed to parse returned JSON, no schema specified.", { error: e });throw new LLMRefusalError("Failed to parse returned JSON. Please specify a schema in the extract object.");