Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
test new mu alternative (#2263)
Thomas Kosmasa5d7c57c5c04
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 0f6442e53..949894e20 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -276,6 +276,51 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {process.env.RUNPOD_MU_API_KEY &&process.env.RUNPOD_MU_POD_ID) {+ // Minimal experiment: fire MU v2 in background if enabled; do not await+ if (+ process.env.PDF_MU_V2_EXPERIMENT === "true" &&+ process.env.PDF_MU_V2_BASE_URL+ ) {+ (async () => {+ const startedAt = Date.now();+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });+ try {+ const resp = await robustFetch({+ url: process.env.PDF_MU_V2_BASE_URL ?? "",+ method: "POST",+ body: {+ input: {+ file_content: base64Content,+ filename: path.basename(tempFilePath) + ".pdf",+ timeout: meta.abort.scrapeTimeout(),+ created_at: Date.now(),+ ...(maxPages !== undefined && { max_pages: maxPages }),+ },+ },+ logger,+ schema: z.any(),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });+ const body: any = resp as any;+ const tokensIn = body?.metadata?.["total-input-tokens"];+ const tokensOut = body?.metadata?.["total-output-tokens"];+ const pages = body?.metadata?.["pdf-total-pages"];+ const durationMs = Date.now() - startedAt;+ logger.info("MU v2 experiment completed", {+ durationMs,+ url: meta.rewrittenUrl ?? meta.url,+ tokensIn,+ tokensOut,+ pages,+ });+ } catch (error) {+ const durationMs = Date.now() - startedAt;+ logger.warn("MU v2 experiment failed", { error, durationMs });+ }+ })();+ }+ const muV1StartedAt = Date.now();try {result = await scrapePDFWithRunPodMU({@@ -288,6 +333,15 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {base64Content,maxPages,);+ const muV1DurationMs = Date.now() - muV1StartedAt;+ meta.logger+ .child({ method: "scrapePDF/MUv1Experiment" })+ .info("MU v1 completed", {+ durationMs: muV1DurationMs,+ url: meta.rewrittenUrl ?? meta.url,+ pages: effectivePageCount,+ success: true,+ });} catch (error) {if (error instanceof RemoveFeatureError ||@@ -300,6 +354,15 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {{ error },);Sentry.captureException(error);+ const muV1DurationMs = Date.now() - muV1StartedAt;+ meta.logger+ .child({ method: "scrapePDF/MUv1Experiment" })+ .info("MU v1 failed", {+ durationMs: muV1DurationMs,+ url: meta.rewrittenUrl ?? meta.url,+ pages: effectivePageCount,+ success: false,+ });}}