Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(extract): add logging
Móricz Gergőd3518e85a86a
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 727d5da45..a5257f0b5 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -5,6 +5,7 @@ import { getScrapeQueue } from "../../services/queue-service";import { waitForJob } from "../../services/queue-jobs";import { addScrapeJob } from "../../services/queue-jobs";import { getJobPriority } from "../job-priority";+import type { Logger } from "winston";interface ScrapeDocumentOptions {url: string;@@ -18,6 +19,7 @@ interface ScrapeDocumentOptions {export async function scrapeDocument(options: ScrapeDocumentOptions,urlTraces: URLTrace[],+ logger: Logger,): Promise<Document | null> {const trace = urlTraces.find((t) => t.url === options.url);if (trace) {@@ -68,16 +70,25 @@ export async function scrapeDocument(try {try {- return await attemptScrape(options.timeout);+ logger.debug("Attempting scrape...");+ const x = await attemptScrape(options.timeout);+ logger.debug("Scrape finished!");+ return x;} catch (timeoutError) {+ logger.warn("Scrape failed.", { error: timeoutError });+if (options.isSingleUrl) {// For single URLs, try again with double timeout- return await attemptScrape(options.timeout * 2);+ logger.debug("Attempting scrape...");+ const x = await attemptScrape(options.timeout * 2);+ logger.debug("Scrape finished!");+ return x;}+throw timeoutError;}} catch (error) {- logger.error(`Error in scrapeDocument: ${error}`);+ logger.error(`error in scrapeDocument`, { error });if (trace) {trace.status = "error";trace.error = error.message;diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 72312aed8..ab27e75c2 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -193,7 +193,7 @@ export async function performExtraction(let totalUrlsScraped = 0;const logger = _logger.child({- module: "extraction-service",+ module: "extract",method: "performExtraction",extractId,});@@ -215,6 +215,9 @@ export async function performExtraction(let startMap = Date.now();let aggMapLinks: string[] = [];+ logger.debug("Processing URLs...", {+ urlCount: request.urls.length,+ });// Process URLsconst urlPromises = request.urls.map((url) =>processUrl(@@ -243,11 +246,15 @@ export async function performExtraction(],});},+ logger.child({ module: "extract", method: "processUrl", url }),),);const processedUrls = await Promise.all(urlPromises);const links = processedUrls.flat().filter((url) => url);+ logger.debug("Processed URLs.", {+ linkCount: links.length,+ });if (links.length === 0) {return {@@ -281,6 +288,8 @@ export async function performExtraction(reqSchema = await dereferenceSchema(reqSchema);}+ logger.debug("Transformed schema.", { schema: reqSchema });+// agent evaluates if the schema or the prompt has an array with big amount of items// also it checks if the schema any other properties that are not arrays// if so, it splits the results into 2 types of completions:@@ -295,6 +304,8 @@ export async function performExtraction(tokenUsage: schemaAnalysisTokenUsage,} = await analyzeSchemaAndPrompt(links, reqSchema, request.prompt ?? "");+ logger.debug("Analyzed schema.", { isMultiEntity, multiEntityKeys, reasoning, keyIndicators });+// Track schema analysis tokenstokenUsage.push(schemaAnalysisTokenUsage);@@ -305,11 +316,14 @@ export async function performExtraction(let rSchema = reqSchema;if (isMultiEntity && reqSchema) {+ logger.debug("=== MULTI-ENTITY ===");+const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(reqSchema,multiEntityKeys,);rSchema = singleAnswerSchema;+ logger.debug("Spread schemas.", { singleAnswerSchema, multiEntitySchema });await updateExtract(extractId, {status: "processing",@@ -337,6 +351,7 @@ export async function performExtraction(],});+ logger.debug("Starting multi-entity scrape...");let startScrape = Date.now();const scrapePromises = links.map((url) => {if (!docsMap.has(url)) {@@ -349,6 +364,7 @@ export async function performExtraction(timeout,},urlTraces,+ logger.child({ module: "extract", method: "scrapeDocument", url, isMultiEntity: true }),);}return docsMap.get(url);@@ -358,6 +374,10 @@ export async function performExtraction((doc): doc is Document => doc !== null,);+ logger.debug("Multi-entity scrape finished.", {+ docCount: multyEntityDocs.length,+ });+totalUrlsScraped += multyEntityDocs.length;let endScrape = Date.now();@@ -380,6 +400,8 @@ export async function performExtraction(}}+ logger.debug("Updated docsMap.", { docsMapSize: docsMap.size }); // useful for error probing+// Process docs in chunks with queue style processingconst chunkSize = 50;const timeoutCompletion = 45000; // 45 second timeout@@ -527,7 +549,7 @@ export async function performExtraction(return multiEntityCompletion.extract;} catch (error) {- logger.error(`Failed to process document: ${error}`);+ logger.error(`Failed to process document.`, { error, url: doc.metadata.url ?? doc.metadata.sourceURL! });return null;}});@@ -537,6 +559,7 @@ export async function performExtraction(multiEntityCompletions.push(...chunkResults.filter((result) => result !== null),);+ logger.debug("All multi-entity completion chunks finished.", { completionCount: multiEntityCompletions.length });}try {@@ -548,7 +571,7 @@ export async function performExtraction(multiEntityResult = mergeNullValObjs(multiEntityResult);// @nick: maybe we can add here a llm that checks if the array probably has a primary key?} catch (error) {- logger.error(`Failed to transform array to object: ${error}`);+ logger.error(`Failed to transform array to object`, { error });return {success: false,error:@@ -565,6 +588,11 @@ export async function performExtraction(rSchema.properties &&Object.keys(rSchema.properties).length > 0) {+ logger.debug("=== SINGLE PAGES ===", {+ linkCount: links.length,+ schema: rSchema,+ });+// Scrape documentsconst timeout = 60000;let singleAnswerDocs: Document[] = [];@@ -593,6 +621,7 @@ export async function performExtraction(timeout,},urlTraces,+ logger.child({ module: "extract", method: "scrapeDocument", url, isMultiEntity: false }));}return docsMap.get(url);@@ -606,12 +635,15 @@ export async function performExtraction(docsMap.set(doc.metadata.url, doc);}}+ logger.debug("Updated docsMap.", { docsMapSize: docsMap.size }); // useful for error probingconst validResults = results.filter((doc): doc is Document => doc !== null,);singleAnswerDocs.push(...validResults);totalUrlsScraped += validResults.length;++ logger.debug("Scrapes finished.", { docCount: validResults.length });} catch (error) {return {success: false,@@ -624,6 +656,7 @@ export async function performExtraction(if (docsMap.size == 0) {// All urls are invalid+ logger.error("All provided URLs are invalid!");return {success: false,error:@@ -647,8 +680,9 @@ export async function performExtraction(});// Generate completions+ logger.debug("Generating singleAnswer completions...");singleAnswerCompletions = await generateOpenAICompletions(- logger.child({ method: "extractService/generateOpenAICompletions" }),+ logger.child({ module: "extract", method: "generateOpenAICompletions" }),{mode: "llm",systemPrompt:@@ -662,6 +696,7 @@ export async function performExtraction(undefined,true,);+ logger.debug("Done generating singleAnswer completions.");// Track single answer extraction tokensif (singleAnswerCompletions) {@@ -691,7 +726,7 @@ export async function performExtraction(}let finalResult = reqSchema- ? await mixSchemaObjects(reqSchema, singleAnswerResult, multiEntityResult)+ ? await mixSchemaObjects(reqSchema, singleAnswerResult, multiEntityResult, logger.child({ method: "mixSchemaObjects" })): singleAnswerResult || multiEntityResult;// Tokenize final result to get token count@@ -785,6 +820,8 @@ export async function performExtraction(});});+ logger.debug("Done!");+return {success: true,data: finalResult ?? {},diff --git a/apps/api/src/lib/extract/helpers/mix-schema-objs.ts b/apps/api/src/lib/extract/helpers/mix-schema-objs.tsindex 5cc111eeb..e001ac736 100644--- a/apps/api/src/lib/extract/helpers/mix-schema-objs.ts+++ b/apps/api/src/lib/extract/helpers/mix-schema-objs.ts@@ -1,9 +1,13 @@+import type { Logger } from "winston";+export async function mixSchemaObjects(finalSchema: any,singleAnswerResult: any,multiEntityResult: any,+ logger: Logger) {const finalResult: any = {};+ logger.debug("Mixing schema objects.");// Recursive helper function to merge results based on schemafunction mergeResults(schema: any, singleResult: any, multiResult: any) {diff --git a/apps/api/src/lib/extract/url-processor.ts b/apps/api/src/lib/extract/url-processor.tsindex 97d214673..ab9f6f60a 100644--- a/apps/api/src/lib/extract/url-processor.ts+++ b/apps/api/src/lib/extract/url-processor.ts@@ -9,6 +9,7 @@ import { rerankLinksWithLLM } from "./reranker";import { extractConfig } from "./config";import { updateExtract } from "./extract-redis";import { ExtractStep } from "./extract-redis";+import type { Logger } from "winston";interface ProcessUrlOptions {url: string;@@ -26,6 +27,7 @@ export async function processUrl(options: ProcessUrlOptions,urlTraces: URLTrace[],updateExtractCallback: (links: string[]) => void,+ logger: Logger,): Promise<string[]> {const trace: URLTrace = {url: options.url,@@ -41,6 +43,7 @@ export async function processUrl(trace.usedInCompletion = true;return [options.url];}+ logger.warn("URL is blocked");trace.status = "error";trace.error = "URL is blocked";trace.usedInCompletion = false;@@ -63,6 +66,9 @@ export async function processUrl(}try {+ logger.debug("Running map...", {+ search: searchQuery,+ })const mapResults = await getMapResults({url: baseUrl,search: searchQuery,@@ -79,6 +85,10 @@ export async function processUrl(let mappedLinks = mapResults.mapResults as MapDocument[];let allUrls = [...mappedLinks.map((m) => m.url), ...mapResults.links];let uniqueUrls = removeDuplicateUrls(allUrls);+ logger.debug("Map finished.", {+ linkCount: allUrls.length,+ uniqueLinkCount: uniqueUrls.length,+ });// Track all discovered URLsuniqueUrls.forEach((discoveredUrl) => {@@ -96,6 +106,7 @@ export async function processUrl(// retry if only one url is returnedif (uniqueUrls.length <= 1) {+ logger.debug("Running map... (pass 2)");const retryMapResults = await getMapResults({url: baseUrl,teamId: options.teamId,@@ -111,6 +122,10 @@ export async function processUrl(mappedLinks = retryMapResults.mapResults as MapDocument[];allUrls = [...mappedLinks.map((m) => m.url), ...mapResults.links];uniqueUrls = removeDuplicateUrls(allUrls);+ logger.debug("Map finished. (pass 2)", {+ linkCount: allUrls.length,+ uniqueLinkCount: uniqueUrls.length,+ });// Track all discovered URLsuniqueUrls.forEach((discoveredUrl) => {@@ -184,6 +199,11 @@ export async function processUrl(// (link, index) => `${index + 1}. URL: ${link.url}, Title: ${link.title}, Description: ${link.description}`// );+ logger.info("Generated rephrased prompt.", {+ rephrasedPrompt+ });++ logger.info("Reranking (pass 1)...");const rerankerResult = await rerankLinksWithLLM(mappedLinks,rephrasedPrompt,@@ -191,9 +211,13 @@ export async function processUrl();mappedLinks = rerankerResult.mapDocument;let tokensUsed = rerankerResult.tokensUsed;+ logger.info("Reranked! (pass 1)", {+ linkCount: mappedLinks.length,+ });// 2nd Pass, useful for when the first pass returns too many linksif (mappedLinks.length > 100) {+ logger.info("Reranking (pass 2)...");const rerankerResult = await rerankLinksWithLLM(mappedLinks,rephrasedPrompt,@@ -201,6 +225,9 @@ export async function processUrl();mappedLinks = rerankerResult.mapDocument;tokensUsed += rerankerResult.tokensUsed;+ logger.info("Reranked! (pass 2)", {+ linkCount: mappedLinks.length,+ });}// dumpToFile(