Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { Meta } from "../..";
import { config } from "../../../../config";
import { robustFetch } from "../../lib/fetch";
import { safeMarkdownToHtml } from "./markdownToHtml";
import { z } from "zod";
import path from "node:path";
import {
getPdfResultFromCache,
pdfCacheConfigured,
resolvePdfCacheKey,
savePdfResultToCache,
} from "../../../../lib/gcs-pdf-cache";
import type { PDFProcessorResult } from "./types";
import { getPDFRefresh } from "../../../../controllers/v2/types";
import { consumeRefresh } from "./fire-pdf/refresh-budget";
export async function scrapePDFWithRunPodMU(
meta: Meta,
tempFilePath: string,
base64Content: string,
maxPages?: number,
pagesProcessed?: number,
): Promise<PDFProcessorResult> {
meta.logger.debug("Processing PDF document with RunPod MU", {
tempFilePath,
});
// The cache key (sha256 of the payload, a full pass over the document) is
// computed at most once per request and reused for the read, the write
// and the log lines. This cache lives under the legacy `pdf-cache-v2/`
// prefix; the key is logged so a report can name the entry.
let cacheKeyMemo: string | undefined;
const cacheKeyOf = (): string =>
(cacheKeyMemo ??= resolvePdfCacheKey(base64Content));
// Without a bucket there is no cache: nothing is hashed and no refresh
// budget is spent. Zero-data-retention requests never read the shared
// cache, as on the fire-pdf path. `parsers: [{ type: "pdf", refresh: true }]`
// skips it too, under the same per-team budget: a MinerU-diverted request
// never reaches the fire-pdf path, so the budget has to be applied here as
// well, and the decision is taken once per request, so a fire-pdf → MU
// fallback keeps the decision the first engine got instead of spending a
// second token. Over budget, or with the limiter unavailable, the entry is
// served.
const cacheOn = pdfCacheConfigured();
if (cacheOn && !maxPages && !meta.internalOptions.zeroDataRetention) {
const cacheKey = cacheKeyOf();
let bypass = false;
if (getPDFRefresh(meta.options?.parsers)) {
const decision = await consumeRefresh(
meta.internalOptions.teamId,
meta.id,
);
bypass = decision === "allowed";
if (bypass) {
meta.logger.info("RunPod MU cache bypassed by refresh", {
tempFilePath,
cacheKey,
cacheProvider: "runpod",
});
} else {
meta.logger.warn("RunPod MU cache refresh not applied", {
tempFilePath,
cacheKey,
cacheProvider: "runpod",
decision,
perMinute: config.FIRE_PDF_CACHE_REFRESH_PER_MINUTE,
});
}
}
if (!bypass) {
try {
const cachedResult = await getPdfResultFromCache({ key: cacheKey });
if (cachedResult) {
meta.logger.info("Using cached RunPod MU result for PDF", {
tempFilePath,
cacheKey,
cacheProvider: "runpod",
});
return cachedResult;
}
} catch (error) {
meta.logger.warn(
"Error checking PDF cache, proceeding with RunPod MU",
{ error, tempFilePath, cacheKey, cacheProvider: "runpod" },
);
}
}
}
meta.abort.throwIfAborted();
meta.logger.info("Max Pdf pages", {
tempFilePath,
maxPages,
});
if (
config.PDF_MU_V2_EXPERIMENT === "true" &&
config.PDF_MU_V2_BASE_URL &&
Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT
) {
(async () => {
const pdfParseId = crypto.randomUUID();
const startedAt = Date.now();
const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
logger.info("MU v2 experiment started", {
scrapeId: meta.id,
pdfParseId,
url: meta.rewrittenUrl ?? meta.url,
maxPages,
});
try {
const resp = await robustFetch({
url: config.PDF_MU_V2_BASE_URL ?? "",
method: "POST",
headers: config.PDF_MU_V2_API_KEY
? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }
: undefined,
body: {
input: {
file_content: base64Content,
filename: path.basename(tempFilePath) + ".pdf",
timeout: meta.abort.scrapeTimeout(),
created_at: Date.now(),
id: pdfParseId,
...(maxPages !== undefined && { max_pages: maxPages }),
},
},
logger,
schema: z.any(),
mock: meta.mock,
abort: meta.abort.asSignal(),
});
const body: any = resp as any;
const tokensIn = body?.metadata?.["total-input-tokens"];
const tokensOut = body?.metadata?.["total-output-tokens"];
const pages = body?.metadata?.["pdf-total-pages"];
const durationMs = Date.now() - startedAt;
logger.info("MU v2 experiment completed", {
durationMs,
url: meta.rewrittenUrl ?? meta.url,
tokensIn,
tokensOut,
pages,
});
} catch (error) {
const durationMs = Date.now() - startedAt;
logger.warn("MU v2 experiment failed", { error, durationMs });
}
})();
}
const muV1StartedAt = Date.now();
const podStart = await robustFetch({
url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",
method: "POST",
headers: {
Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
},
body: {
input: {
file_content: base64Content,
filename: path.basename(tempFilePath) + ".pdf",
timeout: meta.abort.scrapeTimeout(),
created_at: Date.now(),
...(maxPages !== undefined && { max_pages: maxPages }),
},
},
logger: meta.logger.child({
method: "scrapePDFWithRunPodMU/runsync/robustFetch",
}),
schema: z.object({
id: z.string(),
status: z.string(),
output: z
.object({
markdown: z.string(),
})
.optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
});
let status: string = podStart.status;
let result: { markdown: string } | undefined = podStart.output;
if (status === "IN_QUEUE" || status === "IN_PROGRESS") {
do {
meta.abort.throwIfAborted();
await new Promise(resolve => setTimeout(resolve, 2500));
meta.abort.throwIfAborted();
const podStatus = await robustFetch({
url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,
method: "GET",
headers: {
Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
},
logger: meta.logger.child({
method: "scrapePDFWithRunPodMU/status/robustFetch",
}),
schema: z.object({
status: z.string(),
output: z
.object({
markdown: z.string(),
})
.optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
});
status = podStatus.status;
result = podStatus.output;
} while (status !== "COMPLETED" && status !== "FAILED");
}
if (status === "FAILED") {
const durationMs = Date.now() - muV1StartedAt;
meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
durationMs,
url: meta.rewrittenUrl ?? meta.url,
pagesProcessed,
});
throw new Error("RunPod MU failed to parse PDF");
}
if (!result) {
const durationMs = Date.now() - muV1StartedAt;
meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
durationMs,
url: meta.rewrittenUrl ?? meta.url,
pagesProcessed,
});
throw new Error("RunPod MU returned no result");
}
const processorResult = {
markdown: result.markdown,
html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),
};
if (cacheOn && !meta.internalOptions.zeroDataRetention) {
try {
await savePdfResultToCache({ key: cacheKeyOf() }, processorResult);
} catch (error) {
meta.logger.warn("Error saving PDF to cache", {
error,
tempFilePath,
});
}
}
{
const durationMs = Date.now() - muV1StartedAt;
meta.logger.child({ method: "scrapePDF/MUv1" }).info("MU v1 completed", {
durationMs,
url: meta.rewrittenUrl ?? meta.url,
pagesProcessed,
});
}
return processorResult;
}