Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Muv2 exp add more logs (#2421)
Thomas Kosmas07044c8215c4
diff --git a/.gitignore b/.gitignoreindex 1c2469ae5..749ce29c7 100644--- a/.gitignore+++ b/.gitignore@@ -51,4 +51,5 @@ apps/python-sdk/.venv//apps/api/running-docs/-/apps/go-html-to-md-service/.gomodcache/\ No newline at end of file+/apps/go-html-to-md-service/.gomodcache/+target/\ No newline at end of filediff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex a7f37f4a9..fc6fd343d 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -74,8 +74,15 @@ async function scrapePDFWithRunPodMU(Number(process.env.PDF_MU_V2_EXPERIMENT_PERCENT ?? "100")) {(async () => {+ const pdfParseId = crypto.randomUUID();const startedAt = Date.now();const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });+ logger.info("MU v2 experiment started", {+ scrapeId: meta.id,+ pdfParseId,+ url: meta.rewrittenUrl ?? meta.url,+ maxPages,+ });try {const resp = await robustFetch({url: process.env.PDF_MU_V2_BASE_URL ?? "",@@ -86,6 +93,7 @@ async function scrapePDFWithRunPodMU(filename: path.basename(tempFilePath) + ".pdf",timeout: meta.abort.scrapeTimeout(),created_at: Date.now(),+ id: pdfParseId,...(maxPages !== undefined && { max_pages: maxPages }),},},