Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add pages processed and per-page latency to selfHostedOCR logs (#3195)

Abimael Martellf00f6d3e920d
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
index 90d724b9f..1934dc171 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
@@ -216,6 +216,7 @@ export async function scrapePDFWithRunPodMU(
base64Content,
{ markdown: result.markdown, durationMs },
maxPages,
+ pagesProcessed,
);
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
index 282153b32..f34cc2487 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
@@ -35,6 +35,7 @@ export function runSelfHostedOCRExperiment(
base64Content: string,
muV1Result: { markdown: string; durationMs: number },
maxPages?: number,
+ pagesProcessed?: number,
): void {
if (
!config.PDF_OCR_EXPERIMENT_ENABLE ||
@@ -63,12 +64,14 @@ export function runSelfHostedOCRExperiment(
schema: z.object({
markdown: z.string(),
failed_pages: z.array(z.number()).nullable(),
+ pages_processed: z.number().optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
});
const ocrDurationMs = Date.now() - startedAt;
const similarity = wordSimilarity(resp.markdown, muV1Result.markdown);
+ const pages = resp.pages_processed ?? pagesProcessed;
logger.info("Self-hosted OCR experiment completed", {
scrapeId: meta.id,
@@ -79,6 +82,9 @@ export function runSelfHostedOCRExperiment(
muV1MarkdownLength: muV1Result.markdown.length,
wordSimilarity: Math.round(similarity * 1000) / 1000,
failedPages: resp.failed_pages,
+ pagesProcessed: pages,
+ ocrPerPageMs: pages ? Math.round(ocrDurationMs / pages) : undefined,
+ muV1PerPageMs: pages ? Math.round(muV1Result.durationMs / pages) : undefined,
});
} catch {
// Non-blocking: instance may be down at any time, silently skip