Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL): reduce logs

Gergő Móriczd13a2e7d2617
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index b895b4fba..1f1bcc69b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -87,7 +87,6 @@ export async function fireEngineCheckStatus(logger: Logger, jobId: string): Prom
logger.debug("Scrape succeeded!", { jobId });
return successParse.data;
} else if (processingParse.success) {
- logger.debug("Scrape is still processing", { jobId });
throw new StillProcessingError(jobId);
} else if (failedParse.success) {
logger.debug("Scrape job failed", { status, jobId });
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index abf92fdbe..0ea54382c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -40,7 +40,7 @@ async function performFireEngineScrape<Engine extends FireEngineScrapeRequestChr
status = await fireEngineCheckStatus(logger.child({ method: "fireEngineCheckStatus" }), scrape.jobId)
} catch (error) {
if (error instanceof StillProcessingError) {
- logger.debug("Scrape is still processing...");
+ // nop
} else if (error instanceof EngineError) {
logger.debug("Fire-engine scrape job failed.", { error, jobId: scrape.jobId });
throw error;
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index b1d354e61..d839f8bc2 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -115,16 +115,19 @@ export const transformerStack: Transformer[] = [
];
export async function executeTransformers(meta: Meta, document: Document): Promise<Document> {
+ const executions: [string, number][] = [];
+
for (const transformer of transformerStack) {
const _meta = {
...meta,
logger: meta.logger.child({ method: "executeTransformers/" + transformer.name }),
};
- meta.logger.debug("Executing transformer " + transformer.name + "...", { document });
const start = Date.now();
document = await transformer(_meta, document);
- meta.logger.debug("Finished executing transformer " + transformer.name + " (" + (Date.now() - start) + "ms)", { document });
+ executions.push([transformer.name, Date.now() - start]);
}
+ meta.logger.debug("Executed transformers.", { executions });
+
return document;
}