Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v2/scrape): port json to use gemini-2.5-flash

Gergő Móricz98d25535455e
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 02155b78c..ba6b22bd4 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -120,6 +120,8 @@ export function coerceFieldsToFormats(
document: Document,
): Document {
const formats = new Set(meta.options.formats);
+ const hasJson = meta.options.formats.find(x => typeof x === "object" && x.type === "json");
+ const hasChangeTracking = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking");
if (!formats.has("markdown") && document.markdown !== undefined) {
delete document.markdown;
@@ -174,14 +176,14 @@ export function coerceFieldsToFormats(
);
}
- if (!formats.has("extract") && (document.extract !== undefined || document.json !== undefined)) {
+ if (!hasJson && (document.extract !== undefined || document.json !== undefined)) {
meta.logger.warn(
- "Removed extract from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
+ "Removed json from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
);
delete document.extract;
- } else if (formats.has("extract") && document.extract === undefined && document.json === undefined) {
+ } else if (hasJson && document.extract === undefined && document.json === undefined) {
meta.logger.warn(
- "Request had format extract, but there was no extract field in the result.",
+ "Request had format json, but there was no json field in the result.",
);
}
@@ -196,7 +198,7 @@ export function coerceFieldsToFormats(
);
}
- if (!formats.has("changeTracking") && document.changeTracking !== undefined) {
+ if (!hasChangeTracking && document.changeTracking !== undefined) {
meta.logger.warn(
"Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
);
@@ -208,7 +210,7 @@ export function coerceFieldsToFormats(
}
if (document.changeTracking &&
- (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("git-diff")) &&
+ (!hasChangeTracking?.modes?.includes("git-diff")) &&
document.changeTracking.diff !== undefined) {
meta.logger.warn(
"Removed diff from changeTracking because git-diff mode wasn't specified in changeTrackingOptions.modes.",
@@ -217,7 +219,7 @@ export function coerceFieldsToFormats(
}
if (document.changeTracking &&
- (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("json")) &&
+ (!hasChangeTracking?.modes?.includes("json")) &&
document.changeTracking.json !== undefined) {
meta.logger.warn(
"Removed structured from changeTracking because structured mode wasn't specified in changeTrackingOptions.modes.",
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 45d083918..935be4346 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -197,6 +197,7 @@ export function calculateCost(
input_cost: 0.55,
output_cost: 2.19,
},
+ "google/gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },
};
let modelCost = modelCosts[model] || { input_cost: 0, output_cost: 0 };
//gemini-2.5-pro-exp-03-25 pricing
@@ -680,8 +681,10 @@ export async function performLLMExtract(
// model: getModel("qwen-qwq-32b", "groq"),
// model: getModel("gemini-2.0-flash", "google"),
// model: getModel("gemini-2.5-pro-preview-03-25", "vertex"),
- model: getModel("gpt-4o-mini", "openai"),
- retryModel: getModel("gpt-4o", "openai"),
+ // model: getModel("gpt-4o-mini", "openai"),
+ // retryModel: getModel("gpt-4o", "openai"),
+ model: getModel("gemini-2.5-flash-lite", "vertex"),
+ retryModel: getModel("gpt-4o-mini", "openai"),
costTrackingOptions: {
costTracking: meta.costTracking,
metadata: {