Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(feat/deep-research) Improvements to final analysis (#1443)
Nicolasf52d6aab9194
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 357914831..1dc26a05b 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -51,7 +51,7 @@"typescript": "^5.8.3"},"dependencies": {- "@ai-sdk/openai": "^1.1.13",+ "@ai-sdk/openai": "^1.3.10","@anthropic-ai/sdk": "^0.24.3","@apidevtools/json-schema-ref-parser": "^11.7.3","@brillout/import": "^0.2.2",@@ -68,7 +68,7 @@"@supabase/supabase-js": "^2.44.2","@types/express-ws": "^3.0.4","@types/ws": "^8.5.12",- "ai": "^4.1.45",+ "ai": "^4.3.4","ajv": "^8.16.0","async": "^3.2.5","async-mutex": "^0.5.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 508fb7cd0..8644e7bd9 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -9,8 +9,8 @@ importers:.:dependencies:'@ai-sdk/openai':- specifier: ^1.1.13- version: 1.1.13([email protected])+ specifier: ^1.3.10+ version: 1.3.10([email protected])'@anthropic-ai/sdk':specifier: ^0.24.3version: 0.24.3([email protected])@@ -60,8 +60,8 @@ importers:specifier: ^8.5.12version: 8.5.12ai:- specifier: ^4.1.45- version: 4.1.45([email protected])([email protected])+ specifier: ^4.3.4+ version: 4.3.4([email protected])([email protected])ajv:specifier: ^8.16.0version: 8.16.0@@ -324,8 +324,8 @@ importers:packages:- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-IdChK1pJTW3NQis02PG/hHTG0gZSyQIMOLPt7f7ES56C0xH2yaKOU1Tp2aib7pZzWGwDlzTOW2h5TtAB8+V6CQ==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-XO0wF2lmAMWCYjkM5bLpWTKoXet61fBiIimTi+blqEGiLUjAvivt/1zZL1Lzhrv9+p19IC1rn9EWZI1dCelV8w==}engines: {node: '>=18'}peerDependencies:zod: ^3.0.0@@ -339,30 +339,35 @@ packages:zod:optional: true+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-sUlZ7Gnq84DCGWMQRIK8XVbkzIBnvPR1diV4v6JwPgpn5armnLI/j+rqn62MpLrU5ZCQZlDKl/Lw6ed3ulYqaA==}+ engines: {node: '>=18'}+ peerDependencies:+ zod: ^3.23.8+'@ai-sdk/[email protected]':resolution: {integrity: sha512-f9jSYwKMdXvm44Dmab1vUBnfCDSFfI5rOtvV1W9oKB7WYHR5dGvCC6x68Mk3NUfrdmNoMVHGoh6JT9HCVMlMow==}engines: {node: '>=18'}- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-NAuEflFvjw1uh1AOmpyi7rBF4xasWsiWUb86JQ8ScjDGxoGDYEdBnaHOxUpooLna0dGNbSPkvDMnVRhoLKoxPQ==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-ITdgNilJZwLKR7X5TnUr1BsQW6UTX5yFp0h66Nfx8XjBYkWD9W3yugr50GOz3CnE9m/U/Cd5OyEbTMI0rgi6ZQ==}+ engines: {node: '>=18'}++ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-S2FzCSi4uTF0JuSN6zYMXyiAWVAzi/Hho8ISYgHpGZiICYLNCP2si4DuXQOsnWef3IXzQPLVoE11C63lILZIkw==}engines: {node: '>=18'}peerDependencies:react: ^18 || ^19 || ^19.0.0-rc- zod: ^3.0.0+ zod: ^3.23.8peerDependenciesMeta:- react:- optional: truezod:optional: true- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-NsV/3CMmjc4m53snzRdtZM6teTQUXIKi8u0Kf7GBruSzaMSuZ4DWaAAlUshhR3p2FpZgtsogW+vYG1/rXsGu+Q==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-OVRxa4SDj0wVsMZ8tGr/whT89oqNtNoXBKmqWC2BRv5ZG6azL2LYZ5ZK35u3lb4l1IE7cWGsLlmq0py0ttsL7A==}engines: {node: '>=18'}peerDependencies:- zod: ^3.0.0- peerDependenciesMeta:- zod:- optional: true+ zod: ^3.23.8'@ampproject/[email protected]':resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==}@@ -1809,17 +1814,15 @@ packages:resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==}engines: {node: '>= 8.0.0'}- resolution: {integrity: sha512-nQkxQ2zCD+O/h8zJ+PxmBv9coyMaG1uP9kGJvhNaGAA25hbZRQWL0NbTsSJ/QMOUraXKLa+6fBm3VF1NkJK9Kg==}+ resolution: {integrity: sha512-uMjzrowIqfU8CCCxhx8QGl7ETydHBROeNL0VoEwetkmDCY6Q8ZTacj6jNNqGJOiCk595aUrGR9VHPY9Ylvy1fg==}engines: {node: '>=18'}peerDependencies:react: ^18 || ^19 || ^19.0.0-rc- zod: ^3.0.0+ zod: ^3.23.8peerDependenciesMeta:react:optional: true- zod:- optional: trueresolution: {integrity: sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==}@@ -4749,10 +4752,10 @@ packages:snapshots:- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 1.0.8- '@ai-sdk/provider-utils': 2.1.9([email protected])+ '@ai-sdk/provider': 1.1.2+ '@ai-sdk/provider-utils': 2.2.6([email protected])zod: 3.24.2'@ai-sdk/[email protected]([email protected])':@@ -4764,27 +4767,37 @@ snapshots:optionalDependencies:zod: 3.24.2+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 1.1.2+ nanoid: 3.3.8+ secure-json-parse: 2.7.0+ zod: 3.24.2+'@ai-sdk/[email protected]':dependencies:json-schema: 0.4.0+ '@ai-sdk/[email protected]':dependencies:- '@ai-sdk/provider-utils': 2.1.9([email protected])- '@ai-sdk/ui-utils': 1.1.15([email protected])+ json-schema: 0.4.0++ dependencies:+ '@ai-sdk/provider-utils': 2.2.6([email protected])+ '@ai-sdk/ui-utils': 1.2.7([email protected])+ react: 18.3.1swr: 2.3.2([email protected])throttleit: 2.1.0optionalDependencies:- react: 18.3.1zod: 3.24.2- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 1.0.8- '@ai-sdk/provider-utils': 2.1.9([email protected])- zod-to-json-schema: 3.24.2([email protected])- optionalDependencies:+ '@ai-sdk/provider': 1.1.2+ '@ai-sdk/provider-utils': 2.2.6([email protected])zod: 3.24.2+ zod-to-json-schema: 3.24.2([email protected])'@ampproject/[email protected]':dependencies:@@ -6978,17 +6991,17 @@ snapshots:dependencies:humanize-ms: 1.2.1dependencies:- '@ai-sdk/provider': 1.0.8- '@ai-sdk/provider-utils': 2.1.9([email protected])- '@ai-sdk/react': 1.1.17([email protected])([email protected])- '@ai-sdk/ui-utils': 1.1.15([email protected])+ '@ai-sdk/provider': 1.1.2+ '@ai-sdk/provider-utils': 2.2.6([email protected])+ '@ai-sdk/react': 1.2.8([email protected])([email protected])+ '@ai-sdk/ui-utils': 1.2.7([email protected])'@opentelemetry/api': 1.9.0jsondiffpatch: 0.6.0+ zod: 3.24.2optionalDependencies:react: 18.3.1- zod: 3.24.2dependencies:diff --git a/apps/api/src/lib/deep-research/research-manager.ts b/apps/api/src/lib/deep-research/research-manager.tsindex 0e9afa910..900dcd218 100644--- a/apps/api/src/lib/deep-research/research-manager.ts+++ b/apps/api/src/lib/deep-research/research-manager.ts@@ -7,6 +7,8 @@ import {} from "./deep-research-redis";import { generateCompletions, trimToTokenLimit } from "../../scraper/scrapeURL/transformers/llmExtract";import { ExtractOptions } from "../../controllers/v1/types";+import { openai } from "@ai-sdk/openai/dist";+import { getModel } from "../generic-ai";interface AnalysisResult {gaps: string[];nextSteps: string[];@@ -277,7 +279,7 @@ export class ResearchLLMService {}),systemPrompt: formats.includes('json')? "You are an expert research analyst who creates comprehensive, structured analysis following the provided JSON schema exactly."- : "You are an expert research analyst who creates comprehensive, well-structured reports. Your reports are detailed, properly formatted in Markdown, and include clear sections with citations. Today's date is " ++ : "You are an expert research analyst who creates comprehensive, well-structured reports. Don't begin the report by saying 'Here is the report', nor 'Below is the report', nor something similar. ALWAYS start with a great title that reflects the research topic and findings. Your reports are detailed, properly formatted in Markdown, and include clear sections with citations. Today's date is " +new Date().toISOString().split("T")[0],prompt: trimToTokenLimit(analysisPrompt@@ -296,11 +298,17 @@ export class ResearchLLMService {- Make it comprehensive and thorough (aim for 4+ pages worth of content)- Include all relevant findings and insights from the research- Cite sources- - Use bullet points and lists where appropriate for readability`,+ - Cite sources throughout the report+ - Use bullet points and lists where appropriate for readability+ - Don't begin the report by saying "Here is the report", nor "Below is the report", nor something similar.+ - ALWAYS Start with a great title that reflects the research topic and findings - concise and to the point. That's the first thing you should output.++ Begin!`,100000,).text,},markdown: "",+ model: getModel('o3-mini'),});return extract;diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex d5e6591de..8006b8800 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -122,7 +122,7 @@ export async function processBillingBatch() {group.is_extract);- logger.info(`β Successfully billed team ${group.team_id} for ${group.total_credits} credits`);+ logger.info(`β Successfully billed team ${group.team_id} for ${group.total_credits} ${group.is_extract ? 'tokens' : 'credits'}`);} catch (error) {logger.error(`β Failed to bill team ${group.team_id}`, { error, group });Sentry.captureException(error, {