Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

test new mu alternative (#2263)

Thomas Kosmasa5d7c57c5c04
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 0f6442e53..949894e20 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -276,6 +276,51 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
process.env.RUNPOD_MU_API_KEY &&
process.env.RUNPOD_MU_POD_ID
) {
+ // Minimal experiment: fire MU v2 in background if enabled; do not await
+ if (
+ process.env.PDF_MU_V2_EXPERIMENT === "true" &&
+ process.env.PDF_MU_V2_BASE_URL
+ ) {
+ (async () => {
+ const startedAt = Date.now();
+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
+ try {
+ const resp = await robustFetch({
+ url: process.env.PDF_MU_V2_BASE_URL ?? "",
+ method: "POST",
+ body: {
+ input: {
+ file_content: base64Content,
+ filename: path.basename(tempFilePath) + ".pdf",
+ timeout: meta.abort.scrapeTimeout(),
+ created_at: Date.now(),
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ },
+ },
+ logger,
+ schema: z.any(),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+ const body: any = resp as any;
+ const tokensIn = body?.metadata?.["total-input-tokens"];
+ const tokensOut = body?.metadata?.["total-output-tokens"];
+ const pages = body?.metadata?.["pdf-total-pages"];
+ const durationMs = Date.now() - startedAt;
+ logger.info("MU v2 experiment completed", {
+ durationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ tokensIn,
+ tokensOut,
+ pages,
+ });
+ } catch (error) {
+ const durationMs = Date.now() - startedAt;
+ logger.warn("MU v2 experiment failed", { error, durationMs });
+ }
+ })();
+ }
+ const muV1StartedAt = Date.now();
try {
result = await scrapePDFWithRunPodMU(
{
@@ -288,6 +333,15 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
base64Content,
maxPages,
);
+ const muV1DurationMs = Date.now() - muV1StartedAt;
+ meta.logger
+ .child({ method: "scrapePDF/MUv1Experiment" })
+ .info("MU v1 completed", {
+ durationMs: muV1DurationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ pages: effectivePageCount,
+ success: true,
+ });
} catch (error) {
if (
error instanceof RemoveFeatureError ||
@@ -300,6 +354,15 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
{ error },
);
Sentry.captureException(error);
+ const muV1DurationMs = Date.now() - muV1StartedAt;
+ meta.logger
+ .child({ method: "scrapePDF/MUv1Experiment" })
+ .info("MU v1 failed", {
+ durationMs: muV1DurationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ pages: effectivePageCount,
+ success: false,
+ });
}
}