Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Muv2 exp add more logs (#2421)

Thomas Kosmas07044c8215c4
diff --git a/.gitignore b/.gitignore
index 1c2469ae5..749ce29c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -51,4 +51,5 @@ apps/python-sdk/.venv/
/apps/api/running-docs/
-/apps/go-html-to-md-service/.gomodcache/
\ No newline at end of file
+/apps/go-html-to-md-service/.gomodcache/
+target/
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index a7f37f4a9..fc6fd343d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -74,8 +74,15 @@ async function scrapePDFWithRunPodMU(
Number(process.env.PDF_MU_V2_EXPERIMENT_PERCENT ?? "100")
) {
(async () => {
+ const pdfParseId = crypto.randomUUID();
const startedAt = Date.now();
const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
+ logger.info("MU v2 experiment started", {
+ scrapeId: meta.id,
+ pdfParseId,
+ url: meta.rewrittenUrl ?? meta.url,
+ maxPages,
+ });
try {
const resp = await robustFetch({
url: process.env.PDF_MU_V2_BASE_URL ?? "",
@@ -86,6 +93,7 @@ async function scrapePDFWithRunPodMU(
filename: path.basename(tempFilePath) + ".pdf",
timeout: meta.abort.scrapeTimeout(),
created_at: Date.now(),
+ id: pdfParseId,
...(maxPages !== undefined && { max_pages: maxPages }),
},
},