Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Improve logging
Gergő Móricz2193bee13356
diff --git a/apps/api/src/lib/extract/completions/singleAnswer.ts b/apps/api/src/lib/extract/completions/singleAnswer.tsindex 5e76954ee..4b36a58f3 100644--- a/apps/api/src/lib/extract/completions/singleAnswer.ts+++ b/apps/api/src/lib/extract/completions/singleAnswer.ts@@ -23,7 +23,7 @@ export async function singleAnswerCompletion({prompt: string;systemPrompt: string;useAgent: boolean;- extractId?: string;+ extractId: string;}): Promise<{extract: any;tokenUsage: TokenUsage;@@ -35,7 +35,11 @@ export async function singleAnswerCompletion({}> {const docsPrompt = `Today is: ` + new Date().toISOString() + `.\n` + prompt;const generationOptions: GenerateCompletionsOptions = {- logger: logger.child({ module: "extract", method: "generateCompletions" }),+ logger: logger.child({+ module: "extract",+ method: "generateCompletions",+ extractId,+ }),options: {mode: "llm",systemPrompt:diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 7380f3806..955832ff7 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -211,7 +211,7 @@ export async function extractData({if (!schema && extractOptions.options.prompt) {logger.info("Generating schema from prompt");- const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt);+ const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt, logger);otherCallCount++;otherCost += genRes.cost;schema = genRes.extract;@@ -249,7 +249,7 @@ export async function extractData({} catch (error) {logger.error("failed during extractSmartScrape.ts:generateCompletions",- error,+ { error },);// console.log("failed during extractSmartScrape.ts:generateCompletions", error);}@@ -260,16 +260,12 @@ export async function extractData({// console.log("smartscrape_reasoning", extract?.smartscrape_reasoning);// console.log("smartscrape_prompt", extract?.smartscrape_prompt);try {- console.log("=========================================");- console.log(- "useAgent:",+ logger.info("Smart schema resolved", {useAgent,- "shouldUseSmartscrape:",- extract?.shouldUseSmartscrape,- );- console.log("url:", urls);- console.log("prompt:", extract?.smartscrape_prompt);- console.log("=========================================");+ shouldUseSmartscrape: extract?.shouldUseSmartscrape,+ url: urls,+ prompt: extract?.smartscrape_prompt,+ })if (useAgent && extract?.shouldUseSmartscrape) {let smartscrapeResults: SmartScrapeResult[];diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 502e268ee..ebdb7a6b9 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -761,6 +761,7 @@ export function removeDefaultProperty(schema: any): any {export async function generateSchemaFromPrompt(prompt: string,+ logger: Logger,): Promise<{ extract: any; cost: number }> {const model = getModel("gpt-4o", "openai");const retryModel = getModel("gpt-4o-mini", "openai");