Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2/scrape): port json to use gemini-2.5-flash
Gergő Móricz98d25535455e
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 02155b78c..ba6b22bd4 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -120,6 +120,8 @@ export function coerceFieldsToFormats(document: Document,): Document {const formats = new Set(meta.options.formats);+ const hasJson = meta.options.formats.find(x => typeof x === "object" && x.type === "json");+ const hasChangeTracking = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking");if (!formats.has("markdown") && document.markdown !== undefined) {delete document.markdown;@@ -174,14 +176,14 @@ export function coerceFieldsToFormats();}- if (!formats.has("extract") && (document.extract !== undefined || document.json !== undefined)) {+ if (!hasJson && (document.extract !== undefined || document.json !== undefined)) {meta.logger.warn(- "Removed extract from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",+ "Removed json from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",);delete document.extract;- } else if (formats.has("extract") && document.extract === undefined && document.json === undefined) {+ } else if (hasJson && document.extract === undefined && document.json === undefined) {meta.logger.warn(- "Request had format extract, but there was no extract field in the result.",+ "Request had format json, but there was no json field in the result.",);}@@ -196,7 +198,7 @@ export function coerceFieldsToFormats();}- if (!formats.has("changeTracking") && document.changeTracking !== undefined) {+ if (!hasChangeTracking && document.changeTracking !== undefined) {meta.logger.warn("Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",);@@ -208,7 +210,7 @@ export function coerceFieldsToFormats(}if (document.changeTracking &&- (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("git-diff")) &&+ (!hasChangeTracking?.modes?.includes("git-diff")) &&document.changeTracking.diff !== undefined) {meta.logger.warn("Removed diff from changeTracking because git-diff mode wasn't specified in changeTrackingOptions.modes.",@@ -217,7 +219,7 @@ export function coerceFieldsToFormats(}if (document.changeTracking &&- (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("json")) &&+ (!hasChangeTracking?.modes?.includes("json")) &&document.changeTracking.json !== undefined) {meta.logger.warn("Removed structured from changeTracking because structured mode wasn't specified in changeTrackingOptions.modes.",diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 45d083918..935be4346 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -197,6 +197,7 @@ export function calculateCost(input_cost: 0.55,output_cost: 2.19,},+ "google/gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },};let modelCost = modelCosts[model] || { input_cost: 0, output_cost: 0 };//gemini-2.5-pro-exp-03-25 pricing@@ -680,8 +681,10 @@ export async function performLLMExtract(// model: getModel("qwen-qwq-32b", "groq"),// model: getModel("gemini-2.0-flash", "google"),// model: getModel("gemini-2.5-pro-preview-03-25", "vertex"),- model: getModel("gpt-4o-mini", "openai"),- retryModel: getModel("gpt-4o", "openai"),+ // model: getModel("gpt-4o-mini", "openai"),+ // retryModel: getModel("gpt-4o", "openai"),+ model: getModel("gemini-2.5-flash-lite", "vertex"),+ retryModel: getModel("gpt-4o-mini", "openai"),costTrackingOptions: {costTracking: meta.costTracking,metadata: {