Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL): reduce logs
Gergő Móriczd13a2e7d2617
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex b895b4fba..1f1bcc69b 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -87,7 +87,6 @@ export async function fireEngineCheckStatus(logger: Logger, jobId: string): Promlogger.debug("Scrape succeeded!", { jobId });return successParse.data;} else if (processingParse.success) {- logger.debug("Scrape is still processing", { jobId });throw new StillProcessingError(jobId);} else if (failedParse.success) {logger.debug("Scrape job failed", { status, jobId });diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex abf92fdbe..0ea54382c 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -40,7 +40,7 @@ async function performFireEngineScrape<Engine extends FireEngineScrapeRequestChrstatus = await fireEngineCheckStatus(logger.child({ method: "fireEngineCheckStatus" }), scrape.jobId)} catch (error) {if (error instanceof StillProcessingError) {- logger.debug("Scrape is still processing...");+ // nop} else if (error instanceof EngineError) {logger.debug("Fire-engine scrape job failed.", { error, jobId: scrape.jobId });throw error;diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex b1d354e61..d839f8bc2 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -115,16 +115,19 @@ export const transformerStack: Transformer[] = [];export async function executeTransformers(meta: Meta, document: Document): Promise<Document> {+ const executions: [string, number][] = [];+for (const transformer of transformerStack) {const _meta = {...meta,logger: meta.logger.child({ method: "executeTransformers/" + transformer.name }),};- meta.logger.debug("Executing transformer " + transformer.name + "...", { document });const start = Date.now();document = await transformer(_meta, document);- meta.logger.debug("Finished executing transformer " + transformer.name + " (" + (Date.now() - start) + "ms)", { document });+ executions.push([transformer.name, Date.now() - start]);}+ meta.logger.debug("Executed transformers.", { executions });+return document;}