Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): update model usage to gpt-4o-mini (#2520)
tom97bc27e93a8d
diff --git a/apps/api/src/lib/extract/completions/batchExtract.ts b/apps/api/src/lib/extract/completions/batchExtract.tsindex 864bc6cf6..1b4aaaf02 100644--- a/apps/api/src/lib/extract/completions/batchExtract.ts+++ b/apps/api/src/lib/extract/completions/batchExtract.ts@@ -84,8 +84,8 @@ export async function batchExtractPromise(},markdown: buildDocument(doc),isExtractEndpoint: true,- model: getModel("gemini-2.5-pro", "vertex"),- retryModel: getModel("gemini-2.5-pro", "google"),+ model: getModel("gpt-4o-mini", "openai"),+ retryModel: getModel("gpt-4o", "openai"),costTrackingOptions: {costTracking: options.costTracking,metadata: {diff --git a/apps/api/src/lib/extract/completions/singleAnswer.ts b/apps/api/src/lib/extract/completions/singleAnswer.tsindex 019474102..833f41234 100644--- a/apps/api/src/lib/extract/completions/singleAnswer.ts+++ b/apps/api/src/lib/extract/completions/singleAnswer.ts@@ -57,8 +57,8 @@ export async function singleAnswerCompletion({},markdown: `${singleAnswerDocs.map((x, i) => `[START_PAGE (ID: ${i})]` + buildDocument(x)).join("\n")} [END_PAGE]\n`,isExtractEndpoint: true,- model: getModel("gemini-2.5-pro", "vertex"),- retryModel: getModel("gemini-2.5-pro", "google"),+ model: getModel("gpt-4o-mini", "openai"),+ retryModel: getModel("gpt-4o", "openai"),costTrackingOptions: {costTracking,metadata: {diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.tsindex 4d235685b..c8faa2808 100644--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts@@ -100,7 +100,7 @@ interface TrimResult {function trimToTokenLimit_F0(text: string,maxTokens: number,- modelId: string = "gpt-4o",+ modelId: string = "gpt-4o-mini",previousWarning?: string,): TrimResult {try {@@ -495,7 +495,7 @@ export async function generateSchemaFromPrompt_F0(prompt: string,metadata: { teamId: string; functionId?: string; extractId?: string },): Promise<any> {- const model = getModel("gpt-4o");+ const model = getModel("gpt-4o-mini");const temperatures = [0, 0.1, 0.3]; // Different temperatures to trylet lastError: Error | null = null;diff --git a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts b/apps/api/src/lib/extract/fire-0/url-processor-f0.tsindex 8f59f516a..6c7098c10 100644--- a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts+++ b/apps/api/src/lib/extract/fire-0/url-processor-f0.ts@@ -20,7 +20,7 @@ export async function generateBasicCompletion_FO(metadata: { teamId: string; extractId?: string },) {const { text } = await generateText({- model: getModel("gpt-4o"),+ model: getModel("gpt-4o-mini"),prompt: prompt,temperature: 0,providerOptions: {diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 549a58221..43c784456 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -209,7 +209,13 @@ const resolveRefs = (if (refPath[0] === "#" && refPath[1] === "$defs") {const defName = refPath[refPath.length - 1];if (defs[defName]) {- return resolveRefs({ ...defs[defName] }, defs, logger, visited, depth + 1);+ return resolveRefs(+ { ...defs[defName] },+ defs,+ logger,+ visited,+ depth + 1,+ );}}return obj; // Return original if ref can't be resolved@@ -290,24 +296,32 @@ export async function extractData({hasRefPathInString: schemaString.includes("#/$defs/"),},);-+try {const resolvedSchema = resolveRefs(schema, defs, logger);const resolvedString = JSON.stringify(resolvedSchema);- const hasRemainingRefs = resolvedString.includes('"$ref"') || resolvedString.includes("#/$defs/");+ const hasRemainingRefs =+ resolvedString.includes('"$ref"') ||+ resolvedString.includes("#/$defs/");if (!hasRemainingRefs) {schema = resolvedSchema;- if (schema && typeof schema === "object" && schema.$defs) delete schema.$defs;+ if (schema && typeof schema === "object" && schema.$defs)+ delete schema.$defs;logger.info("Successfully resolved schema refs", {schema,});} else {- logger.info("Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema");+ logger.info(+ "Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema",+ );}} catch (error) {- logger.warn("Failed to resolve schema refs, preserving original schema", { error });+ logger.warn(+ "Failed to resolve schema refs, preserving original schema",+ { error },+ );}} else {logger.info("No recursive references detected, resolving refs", {@@ -438,8 +452,8 @@ export async function extractData({const newExtractOptions = {...extractOptions,markdown: markdown,- model: getModel("gemini-2.5-pro", "vertex"),- retryModel: getModel("gemini-2.5-pro", "google"),+ model: getModel("gpt-4o-mini", "openai"),+ retryModel: getModel("gpt-4o", "openai"),costTrackingOptions: {costTracking: extractOptions.costTrackingOptions.costTracking,metadata: {diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex a7fb738f7..c87414a6b 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -51,9 +51,9 @@ function selectModelForSchema(schema?: any): {const isRecursive = detectRecursiveSchema(schema);if (isRecursive) {- logger.info(`Model: gpt-4o | hasRef: true`);+ logger.info(`Model: gpt-4o-mini | hasRef: true`);return {- modelName: "gpt-4o",+ modelName: "gpt-4o-mini",reason: "recursive_schema_detected",};}@@ -162,7 +162,7 @@ interface TrimResult {export function trimToTokenLimit(text: string,maxTokens: number,- modelId: string = "gpt-4o",+ modelId: string = "gpt-4o-mini",previousWarning?: string,): TrimResult {try {@@ -303,7 +303,7 @@ export async function generateCompletions({model = getModel("gpt-4o-mini", "openai"),mode = "object",providerOptions,- retryModel = getModel("claude-3-5-sonnet-20240620", "anthropic"),+ retryModel = getModel("gpt-4o", "openai"),costTrackingOptions,metadata,}: GenerateCompletionsOptions): Promise<{@@ -1274,8 +1274,8 @@ export async function generateSchemaFromPrompt(scrapeId?: string;},): Promise<{ extract: any }> {- const model = getModel("gpt-4o", "openai");- const retryModel = getModel("gpt-4o-mini", "openai");+ const model = getModel("gpt-4o-mini", "openai");+ const retryModel = getModel("gpt-4o", "openai");const temperatures = [0, 0.1, 0.3]; // Different temperatures to trylet lastError: Error | null = null;@@ -1352,8 +1352,8 @@ export async function generateCrawlerOptionsFromPrompt(costTracking: CostTracking,metadata: { teamId: string; crawlId?: string },): Promise<{ extract: any }> {- const model = getModel("gpt-4o", "openai");- const retryModel = getModel("gpt-4o-mini", "openai");+ const model = getModel("gpt-4o-mini", "openai");+ const retryModel = getModel("gpt-4o", "openai");const temperatures = [0, 0.1, 0.3];let lastError: Error | null = null;