Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add self-hosted OCR A/B experiment for PDF pipeline (#3132)
Abimael Martellda55f4caded6
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 01baba795..7dbb2b5d1 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -158,6 +158,12 @@ const configSchema = z.object({PDF_MU_V2_EXPERIMENT: z.string().optional(),PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),+ // Self-Hosted OCR Experiment+ PDF_OCR_EXPERIMENT_ENABLE: z.stringbool().optional(),+ PDF_OCR_EXPERIMENT_PERCENT: z.coerce.number().min(0).max(100).default(10),+ PDF_OCR_BASE_URL: z.string().optional(),+ PDF_OCR_API_KEY: z.string().optional(),+// RunPodRUNPOD_MU_API_KEY: z.string().optional(),RUNPOD_MU_POD_ID: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.tsindex cd2d889ef..90d724b9f 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts@@ -4,6 +4,7 @@ import * as marked from "marked";import { robustFetch } from "../../lib/fetch";import { z } from "zod";import path from "node:path";+import { runSelfHostedOCRExperiment } from "./selfHostedOCR";import {getPdfResultFromCache,savePdfResultToCache,@@ -209,6 +210,14 @@ export async function scrapePDFWithRunPodMU(url: meta.rewrittenUrl ?? meta.url,pagesProcessed,});+ if (!meta.internalOptions.zeroDataRetention) {+ runSelfHostedOCRExperiment(+ meta,+ base64Content,+ { markdown: result.markdown, durationMs },+ maxPages,+ );+ }}return processorResult;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.tsnew file mode 100644index 000000000..0df99471c--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts@@ -0,0 +1,86 @@+import { Meta } from "../..";+import { config } from "../../../../config";+import { robustFetch } from "../../lib/fetch";+import { z } from "zod";++/**+ * Compute word-level Jaccard similarity between two texts.+ * Strips markdown syntax and normalises whitespace so we compare+ * the underlying data, not formatting differences.+ */+function wordSimilarity(a: string, b: string): number {+ const normalise = (s: string) =>+ s+ .replace(/[#*_`\[\]()>|~\-]/g, " ")+ .replace(/\s+/g, " ")+ .trim()+ .toLowerCase();++ const wordsA = new Set(normalise(a).split(" ").filter(Boolean));+ const wordsB = new Set(normalise(b).split(" ").filter(Boolean));++ if (wordsA.size === 0 && wordsB.size === 0) return 1;+ if (wordsA.size === 0 || wordsB.size === 0) return 0;++ let intersection = 0;+ for (const w of wordsA) {+ if (wordsB.has(w)) intersection++;+ }++ return intersection / (wordsA.size + wordsB.size - intersection);+}++export function runSelfHostedOCRExperiment(+ meta: Meta,+ base64Content: string,+ muV1Result: { markdown: string; durationMs: number },+ maxPages?: number,+): void {+ if (+ !config.PDF_OCR_EXPERIMENT_ENABLE ||+ !config.PDF_OCR_BASE_URL ||+ Math.random() * 100 >= config.PDF_OCR_EXPERIMENT_PERCENT+ ) {+ return;+ }++ (async () => {+ const startedAt = Date.now();+ const logger = meta.logger.child({ method: "scrapePDF/selfHostedOCR" });+ try {+ const resp = await robustFetch({+ url: `${config.PDF_OCR_BASE_URL}/ocr`,+ method: "POST",+ headers: config.PDF_OCR_API_KEY+ ? { Authorization: `Bearer ${config.PDF_OCR_API_KEY}` }+ : undefined,+ body: {+ pdf: base64Content,+ ...(maxPages !== undefined && { max_pages: maxPages }),+ },+ logger,+ schema: z.object({+ markdown: z.string(),+ failed_pages: z.array(z.number()).nullable(),+ }),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });+ const ocrDurationMs = Date.now() - startedAt;+ const similarity = wordSimilarity(resp.markdown, muV1Result.markdown);++ logger.info("Self-hosted OCR experiment completed", {+ scrapeId: meta.id,+ url: meta.rewrittenUrl ?? meta.url,+ ocrDurationMs,+ muV1DurationMs: muV1Result.durationMs,+ ocrMarkdownLength: resp.markdown.length,+ muV1MarkdownLength: muV1Result.markdown.length,+ wordSimilarity: Math.round(similarity * 1000) / 1000,+ failedPages: resp.failed_pages,+ });+ } catch {+ // Non-blocking: instance may be down at any time, silently skip+ }+ })();+}