Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add pages processed and per-page latency to selfHostedOCR logs (#3195)
Abimael Martellf00f6d3e920d
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.tsindex 90d724b9f..1934dc171 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts@@ -216,6 +216,7 @@ export async function scrapePDFWithRunPodMU(base64Content,{ markdown: result.markdown, durationMs },maxPages,+ pagesProcessed,);}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.tsindex 282153b32..f34cc2487 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts@@ -35,6 +35,7 @@ export function runSelfHostedOCRExperiment(base64Content: string,muV1Result: { markdown: string; durationMs: number },maxPages?: number,+ pagesProcessed?: number,): void {if (!config.PDF_OCR_EXPERIMENT_ENABLE ||@@ -63,12 +64,14 @@ export function runSelfHostedOCRExperiment(schema: z.object({markdown: z.string(),failed_pages: z.array(z.number()).nullable(),+ pages_processed: z.number().optional(),}),mock: meta.mock,abort: meta.abort.asSignal(),});const ocrDurationMs = Date.now() - startedAt;const similarity = wordSimilarity(resp.markdown, muV1Result.markdown);+ const pages = resp.pages_processed ?? pagesProcessed;logger.info("Self-hosted OCR experiment completed", {scrapeId: meta.id,@@ -79,6 +82,9 @@ export function runSelfHostedOCRExperiment(muV1MarkdownLength: muV1Result.markdown.length,wordSimilarity: Math.round(similarity * 1000) / 1000,failedPages: resp.failed_pages,+ pagesProcessed: pages,+ ocrPerPageMs: pages ? Math.round(ocrDurationMs / pages) : undefined,+ muV1PerPageMs: pages ? Math.round(muV1Result.durationMs / pages) : undefined,});} catch {// Non-blocking: instance may be down at any time, silently skip