Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Improve logging

Gergő Móricz2193bee13356
diff --git a/apps/api/src/lib/extract/completions/singleAnswer.ts b/apps/api/src/lib/extract/completions/singleAnswer.ts
index 5e76954ee..4b36a58f3 100644
--- a/apps/api/src/lib/extract/completions/singleAnswer.ts
+++ b/apps/api/src/lib/extract/completions/singleAnswer.ts
@@ -23,7 +23,7 @@ export async function singleAnswerCompletion({
prompt: string;
systemPrompt: string;
useAgent: boolean;
- extractId?: string;
+ extractId: string;
}): Promise<{
extract: any;
tokenUsage: TokenUsage;
@@ -35,7 +35,11 @@ export async function singleAnswerCompletion({
}> {
const docsPrompt = `Today is: ` + new Date().toISOString() + `.\n` + prompt;
const generationOptions: GenerateCompletionsOptions = {
- logger: logger.child({ module: "extract", method: "generateCompletions" }),
+ logger: logger.child({
+ module: "extract",
+ method: "generateCompletions",
+ extractId,
+ }),
options: {
mode: "llm",
systemPrompt:
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 7380f3806..955832ff7 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -211,7 +211,7 @@ export async function extractData({
if (!schema && extractOptions.options.prompt) {
logger.info("Generating schema from prompt");
- const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt);
+ const genRes = await generateSchemaFromPrompt(extractOptions.options.prompt, logger);
otherCallCount++;
otherCost += genRes.cost;
schema = genRes.extract;
@@ -249,7 +249,7 @@ export async function extractData({
} catch (error) {
logger.error(
"failed during extractSmartScrape.ts:generateCompletions",
- error,
+ { error },
);
// console.log("failed during extractSmartScrape.ts:generateCompletions", error);
}
@@ -260,16 +260,12 @@ export async function extractData({
// console.log("smartscrape_reasoning", extract?.smartscrape_reasoning);
// console.log("smartscrape_prompt", extract?.smartscrape_prompt);
try {
- console.log("=========================================");
- console.log(
- "useAgent:",
+ logger.info("Smart schema resolved", {
useAgent,
- "shouldUseSmartscrape:",
- extract?.shouldUseSmartscrape,
- );
- console.log("url:", urls);
- console.log("prompt:", extract?.smartscrape_prompt);
- console.log("=========================================");
+ shouldUseSmartscrape: extract?.shouldUseSmartscrape,
+ url: urls,
+ prompt: extract?.smartscrape_prompt,
+ })
if (useAgent && extract?.shouldUseSmartscrape) {
let smartscrapeResults: SmartScrapeResult[];
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 502e268ee..ebdb7a6b9 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -761,6 +761,7 @@ export function removeDefaultProperty(schema: any): any {
export async function generateSchemaFromPrompt(
prompt: string,
+ logger: Logger,
): Promise<{ extract: any; cost: number }> {
const model = getModel("gpt-4o", "openai");
const retryModel = getModel("gpt-4o-mini", "openai");