Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add self-hosted OCR A/B experiment for PDF pipeline (#3132)

Abimael Martellda55f4caded6
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 01baba795..7dbb2b5d1 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -158,6 +158,12 @@ const configSchema = z.object({
PDF_MU_V2_EXPERIMENT: z.string().optional(),
PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),
+ // Self-Hosted OCR Experiment
+ PDF_OCR_EXPERIMENT_ENABLE: z.stringbool().optional(),
+ PDF_OCR_EXPERIMENT_PERCENT: z.coerce.number().min(0).max(100).default(10),
+ PDF_OCR_BASE_URL: z.string().optional(),
+ PDF_OCR_API_KEY: z.string().optional(),
+
// RunPod
RUNPOD_MU_API_KEY: z.string().optional(),
RUNPOD_MU_POD_ID: z.string().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
index cd2d889ef..90d724b9f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
@@ -4,6 +4,7 @@ import * as marked from "marked";
import { robustFetch } from "../../lib/fetch";
import { z } from "zod";
import path from "node:path";
+import { runSelfHostedOCRExperiment } from "./selfHostedOCR";
import {
getPdfResultFromCache,
savePdfResultToCache,
@@ -209,6 +210,14 @@ export async function scrapePDFWithRunPodMU(
url: meta.rewrittenUrl ?? meta.url,
pagesProcessed,
});
+ if (!meta.internalOptions.zeroDataRetention) {
+ runSelfHostedOCRExperiment(
+ meta,
+ base64Content,
+ { markdown: result.markdown, durationMs },
+ maxPages,
+ );
+ }
}
return processorResult;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
new file mode 100644
index 000000000..0df99471c
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
@@ -0,0 +1,86 @@
+import { Meta } from "../..";
+import { config } from "../../../../config";
+import { robustFetch } from "../../lib/fetch";
+import { z } from "zod";
+
+/**
+ * Compute word-level Jaccard similarity between two texts.
+ * Strips markdown syntax and normalises whitespace so we compare
+ * the underlying data, not formatting differences.
+ */
+function wordSimilarity(a: string, b: string): number {
+ const normalise = (s: string) =>
+ s
+ .replace(/[#*_`\[\]()>|~\-]/g, " ")
+ .replace(/\s+/g, " ")
+ .trim()
+ .toLowerCase();
+
+ const wordsA = new Set(normalise(a).split(" ").filter(Boolean));
+ const wordsB = new Set(normalise(b).split(" ").filter(Boolean));
+
+ if (wordsA.size === 0 && wordsB.size === 0) return 1;
+ if (wordsA.size === 0 || wordsB.size === 0) return 0;
+
+ let intersection = 0;
+ for (const w of wordsA) {
+ if (wordsB.has(w)) intersection++;
+ }
+
+ return intersection / (wordsA.size + wordsB.size - intersection);
+}
+
+export function runSelfHostedOCRExperiment(
+ meta: Meta,
+ base64Content: string,
+ muV1Result: { markdown: string; durationMs: number },
+ maxPages?: number,
+): void {
+ if (
+ !config.PDF_OCR_EXPERIMENT_ENABLE ||
+ !config.PDF_OCR_BASE_URL ||
+ Math.random() * 100 >= config.PDF_OCR_EXPERIMENT_PERCENT
+ ) {
+ return;
+ }
+
+ (async () => {
+ const startedAt = Date.now();
+ const logger = meta.logger.child({ method: "scrapePDF/selfHostedOCR" });
+ try {
+ const resp = await robustFetch({
+ url: `${config.PDF_OCR_BASE_URL}/ocr`,
+ method: "POST",
+ headers: config.PDF_OCR_API_KEY
+ ? { Authorization: `Bearer ${config.PDF_OCR_API_KEY}` }
+ : undefined,
+ body: {
+ pdf: base64Content,
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ },
+ logger,
+ schema: z.object({
+ markdown: z.string(),
+ failed_pages: z.array(z.number()).nullable(),
+ }),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+ const ocrDurationMs = Date.now() - startedAt;
+ const similarity = wordSimilarity(resp.markdown, muV1Result.markdown);
+
+ logger.info("Self-hosted OCR experiment completed", {
+ scrapeId: meta.id,
+ url: meta.rewrittenUrl ?? meta.url,
+ ocrDurationMs,
+ muV1DurationMs: muV1Result.durationMs,
+ ocrMarkdownLength: resp.markdown.length,
+ muV1MarkdownLength: muV1Result.markdown.length,
+ wordSimilarity: Math.round(similarity * 1000) / 1000,
+ failedPages: resp.failed_pages,
+ });
+ } catch {
+ // Non-blocking: instance may be down at any time, silently skip
+ }
+ })();
+}