Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge branch 'main' into rafa/fix-default-on-schema-llm-extract
Rafael Miller8192d756e943
diff --cc apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex c35e20c13,6380edb8a..7b518300f--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@@ -19,198 -19,216 +19,330 @@@ export class LLMRefusalError extends Er}function normalizeSchema(x: any): any {- if (typeof x !== "object" || x === null) return x;-- if (x["$defs"] !== null && typeof x["$defs"] === "object") {- x["$defs"] = Object.fromEntries(Object.entries(x["$defs"]).map(([name, schema]) => [name, normalizeSchema(schema)]));- }+ if (typeof x !== "object" || x === null) return x;++ if (x["$defs"] !== null && typeof x["$defs"] === "object") {+ x["$defs"] = Object.fromEntries(+ Object.entries(x["$defs"]).map(([name, schema]) => [+ name,+ normalizeSchema(schema),+ ]),+ );+ }- if (x && x.anyOf) {- x.anyOf = x.anyOf.map(x => normalizeSchema(x));- }+ if (x && x.anyOf) {+ x.anyOf = x.anyOf.map((x) => normalizeSchema(x));+ }- if (x && x.oneOf) {- x.oneOf = x.oneOf.map(x => normalizeSchema(x));- }+ if (x && x.oneOf) {+ x.oneOf = x.oneOf.map((x) => normalizeSchema(x));+ }- if (x && x.allOf) {- x.allOf = x.allOf.map(x => normalizeSchema(x));- }+ if (x && x.allOf) {+ x.allOf = x.allOf.map((x) => normalizeSchema(x));+ }- if (x && x.not) {- x.not = normalizeSchema(x.not);- }+ if (x && x.not) {+ x.not = normalizeSchema(x.not);+ }- if (x && x.type === "object") {- return {- ...x,- properties: Object.fromEntries(Object.entries(x.properties).map(([k, v]) => [k, normalizeSchema(v)])),- required: Object.keys(x.properties),- additionalProperties: false,- }- } else if (x && x.type === "array") {- return {- ...x,- items: normalizeSchema(x.items),- }- } else {- return x;- }+ if (x && x.type === "object") {+ return {+ ...x,+ properties: Object.fromEntries(+ Object.entries(x.properties).map(([k, v]) => [k, normalizeSchema(v)]),+ ),+ required: Object.keys(x.properties),+ additionalProperties: false,+ };+ } else if (x && x.type === "array") {+ return {+ ...x,+ items: normalizeSchema(x.items),+ };+ } else {+ return x;+ }}- export async function generateOpenAICompletions(logger: Logger, options: ExtractOptions, markdown?: string, previousWarning?: string, isExtractEndpoint?: boolean): Promise<{ extract: any, numTokens: number, warning: string | undefined }> {- let extract: any;- let warning: string | undefined;+ export async function generateOpenAICompletions(+ logger: Logger,+ options: ExtractOptions,+ markdown?: string,+ previousWarning?: string,+ isExtractEndpoint?: boolean,+ ): Promise<{ extract: any; numTokens: number; warning: string | undefined }> {+ let extract: any;+ let warning: string | undefined;++ const openai = new OpenAI();+ const model: TiktokenModel =+ (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";++ if (markdown === undefined) {+ throw new Error("document.markdown is undefined -- this is unexpected");+ }++ // count number of tokens+ let numTokens = 0;+ const encoder = encoding_for_model(model as TiktokenModel);+ try {+ // Encode the message into tokens+ const tokens = encoder.encode(markdown);++ // Return the number of tokens+ numTokens = tokens.length;+ } catch (error) {+ logger.warn("Calculating num tokens of string failed", { error, markdown });++ markdown = markdown.slice(0, maxTokens * modifier);++ let w =+ "Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (" ++ maxTokens ++ ") we support.";+ warning = previousWarning === undefined ? w : w + " " + previousWarning;+ } finally {+ // Free the encoder resources after use+ encoder.free();+ }- const openai = new OpenAI();- const model: TiktokenModel = (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";+ if (numTokens > maxTokens) {+ // trim the document to the maximum number of tokens, tokens != characters+ markdown = markdown.slice(0, maxTokens * modifier);++ const w =+ "The extraction content would have used more tokens (" ++ numTokens ++ ") than the maximum we allow (" ++ maxTokens ++ "). -- the input has been automatically trimmed.";+ warning = previousWarning === undefined ? w : w + " " + previousWarning;+ }- if (markdown === undefined) {- throw new Error("document.markdown is undefined -- this is unexpected");- }+ let schema = options.schema;+ if (schema && schema.type === "array") {+ schema = {+ type: "object",+ properties: {+ items: options.schema,+ },+ required: ["items"],+ additionalProperties: false,+ };+ } else if (schema && typeof schema === "object" && !schema.type) {+ schema = {+ type: "object",+ properties: Object.fromEntries(+ Object.entries(schema).map(([key, value]) => [key, { type: value }]),+ ),+ required: Object.keys(schema),+ additionalProperties: false,+ };+ }- // count number of tokens- let numTokens = 0;- const encoder = encoding_for_model(model as TiktokenModel);+ schema = normalizeSchema(schema);++ const jsonCompletion = await openai.beta.chat.completions.parse({+ model,+ temperature: 0,+ messages: [+ {+ role: "system",+ content: options.systemPrompt,+ },+ {+ role: "user",+ content: [{ type: "text", text: markdown }],+ },+ {+ role: "user",+ content:+ options.prompt !== undefined+ ? `Transform the above content into structured JSON output based on the following user request: ${options.prompt}`+ : "Transform the above content into structured JSON output.",+ },+ ],+ response_format: options.schema+ ? {+ type: "json_schema",+ json_schema: {+ name: "websiteContent",+ schema: schema,+ strict: true,+ },+ }+ : { type: "json_object" },+ });++ if (jsonCompletion.choices[0].message.refusal !== null) {+ throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal);+ }++ extract = jsonCompletion.choices[0].message.parsed;++ if (extract === null && jsonCompletion.choices[0].message.content !== null) {try {+ // Encode the message into tokens+ const tokens = encoder.encode(markdown);++ // Return the number of tokens+ numTokens = tokens.length;+ } catch (error) {+ logger.warn("Calculating num tokens of string failed", { error, markdown });++ markdown = markdown.slice(0, maxTokens * modifier);++ let w = "Failed to derive number of LLM tokens the extraction might use -- the input has been automatically trimmed to the maximum number of tokens (" + maxTokens + ") we support.";+ warning = previousWarning === undefined ? w : w + " " + previousWarning;+ } finally {+ // Free the encoder resources after use+ encoder.free();+ }++ if (numTokens > maxTokens) {+ // trim the document to the maximum number of tokens, tokens != characters+ markdown = markdown.slice(0, maxTokens * modifier);++ const w = "The extraction content would have used more tokens (" + numTokens + ") than the maximum we allow (" + maxTokens + "). -- the input has been automatically trimmed.";+ warning = previousWarning === undefined ? w : w + " " + previousWarning;+ }++ let schema = options.schema;+ if (schema) {+ schema = removeDefaultProperty(schema);+ }++ if (schema && schema.type === "array") {+ schema = {+ type: "object",+ properties: {+ items: options.schema,+ },+ required: ["items"],+ additionalProperties: false,+ };+ } else if (schema && typeof schema === 'object' && !schema.type) {+ schema = {+ type: "object",+ properties: Object.fromEntries(+ Object.entries(schema).map(([key, value]) => {+ return [key, removeDefaultProperty(value)];+ })+ ),+ required: Object.keys(schema),+ additionalProperties: false+ };+ }++ schema = normalizeSchema(schema);++ const jsonCompletion = await openai.beta.chat.completions.parse({+ model,+ temperature: 0,+ messages: [+ {+ role: "system",+ content: options.systemPrompt,+ },+ {+ role: "user",+ content: [{ type: "text", text: markdown }],+ },+ {+ role: "user",+ content: options.prompt !== undefined+ ? `Transform the above content into structured JSON output based on the following user request: ${options.prompt}`+ : "Transform the above content into structured JSON output.",+ },+ ],+ response_format: options.schema ? {+ type: "json_schema",+ json_schema: {+ name: "websiteContent",+ schema: schema,+ strict: true,+ }+ } : { type: "json_object" },+ });++ if (jsonCompletion.choices[0].message.refusal !== null) {+ throw new LLMRefusalError(jsonCompletion.choices[0].message.refusal);+ }++ extract = jsonCompletion.choices[0].message.parsed;++ if (extract === null && jsonCompletion.choices[0].message.content !== null) {+ try {+ if (!isExtractEndpoint) {+ extract = JSON.parse(jsonCompletion.choices[0].message.content);+ } else {+ const extractData = JSON.parse(jsonCompletion.choices[0].message.content);+ extract = options.schema ? extractData.data.extract : extractData;+ }+ } catch (e) {+ logger.error("Failed to parse returned JSON, no schema specified.", { error: e });+ throw new LLMRefusalError("Failed to parse returned JSON. Please specify a schema in the extract object.");+ }+ }++ // If the users actually wants the items object, they can specify it as 'required' in the schema+ // otherwise, we just return the items array+ if (options.schema && options.schema.type === "array" && !schema?.required?.includes("items")) {+ extract = extract?.items;+ }+ return { extract, warning, numTokens };+}++export async function performLLMExtract(meta: Meta, document: Document): Promise<Document> {+ if (meta.options.formats.includes("extract")) {+ const { extract, warning } = await generateOpenAICompletions(+ meta.logger.child({ method: "performLLMExtract/generateOpenAICompletions" }),+ meta.options.extract!,+ document.markdown,+ document.warning,+ if (!isExtractEndpoint) {+ extract = JSON.parse(jsonCompletion.choices[0].message.content);+ } else {+ const extractData = JSON.parse(+ jsonCompletion.choices[0].message.content,);- document.extract = extract;- document.warning = warning;+ extract = options.schema ? extractData.data.extract : extractData;+ }+ } catch (e) {+ logger.error("Failed to parse returned JSON, no schema specified.", {+ error: e,+ });+ throw new LLMRefusalError(+ "Failed to parse returned JSON. Please specify a schema in the extract object.",+ );}+ }- return document;+ // If the users actually wants the items object, they can specify it as 'required' in the schema+ // otherwise, we just return the items array+ if (+ options.schema &&+ options.schema.type === "array" &&+ !schema?.required?.includes("items")+ ) {+ extract = extract?.items;+ }+ return { extract, warning, numTokens };}-export async function performLLMExtract(- meta: Meta,- document: Document,-): Promise<Document> {- if (meta.options.formats.includes("extract")) {- const { extract, warning } = await generateOpenAICompletions(- meta.logger.child({- method: "performLLMExtract/generateOpenAICompletions",- }),- meta.options.extract!,- document.markdown,- document.warning,- );- document.extract = extract;- document.warning = warning;+export function removeDefaultProperty(schema: any): any {+ if (typeof schema !== 'object' || schema === null) return schema;++ const { default: _, ...rest } = schema;++ for (const key in rest) {+ if (Array.isArray(rest[key])) {+ rest[key] = rest[key].map((item: any) => removeDefaultProperty(item));+ } else if (typeof rest[key] === 'object' && rest[key] !== null) {+ rest[key] = removeDefaultProperty(rest[key]);+ }}- return document;-}+ return rest;+}