Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add precision, recall, F1 metrics to self-hosted OCR experiment (#3253)

Abimael Martell4e0f65e39118
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
index 9401a7132..ffb68dd34 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
@@ -3,12 +3,12 @@ import { config } from "../../../../config";
import { robustFetch } from "../../lib/fetch";
import { z } from "zod";
-/**
- * Compute word-level Jaccard similarity between two texts.
- * Strips markdown syntax and normalises whitespace so we compare
- * the underlying data, not formatting differences.
- */
-function wordSimilarity(a: string, b: string): number {
+function computeSimilarityMetrics(a: string, b: string): {
+ jaccard: number;
+ precision: number;
+ recall: number;
+ f1: number;
+} {
const normalise = (s: string) =>
s
.replace(/[#*_`\[\]()>|~\-]/g, " ")
@@ -19,15 +19,29 @@ function wordSimilarity(a: string, b: string): number {
const wordsA = new Set(normalise(a).split(" ").filter(Boolean));
const wordsB = new Set(normalise(b).split(" ").filter(Boolean));
- if (wordsA.size === 0 && wordsB.size === 0) return 1;
- if (wordsA.size === 0 || wordsB.size === 0) return 0;
+ if (wordsA.size === 0 && wordsB.size === 0) {
+ return { jaccard: 1, precision: 1, recall: 1, f1: 1 };
+ }
+ if (wordsA.size === 0 || wordsB.size === 0) {
+ return { jaccard: 0, precision: 0, recall: 0, f1: 0 };
+ }
let intersection = 0;
for (const w of wordsA) {
if (wordsB.has(w)) intersection++;
}
- return intersection / (wordsA.size + wordsB.size - intersection);
+ const jaccard = intersection / (wordsA.size + wordsB.size - intersection);
+ const precision = intersection / wordsA.size; // % of our words in MinerU
+ const recall = intersection / wordsB.size; // % of MinerU words in ours
+ const f1 = (2 * precision * recall) / (precision + recall) || 0;
+
+ return {
+ jaccard: Math.round(jaccard * 1000) / 1000,
+ precision: Math.round(precision * 1000) / 1000,
+ recall: Math.round(recall * 1000) / 1000,
+ f1: Math.round(f1 * 1000) / 1000,
+ };
}
export function runSelfHostedOCRExperiment(
@@ -70,7 +84,7 @@ export function runSelfHostedOCRExperiment(
abort: meta.abort.asSignal(),
});
const ocrDurationMs = Date.now() - startedAt;
- const similarity = wordSimilarity(resp.markdown, muV1Result.markdown);
+ const similarity = computeSimilarityMetrics(resp.markdown, muV1Result.markdown);
const pages = resp.pages_processed ?? pagesProcessed;
const timeDiffMs = muV1Result.durationMs - ocrDurationMs;
const speedup = muV1Result.durationMs > 0 && ocrDurationMs > 0
@@ -86,7 +100,10 @@ export function runSelfHostedOCRExperiment(
speedup,
ocrMarkdownLength: resp.markdown.length,
muV1MarkdownLength: muV1Result.markdown.length,
- wordSimilarity: Math.round(similarity * 1000) / 1000,
+ wordSimilarity: similarity.jaccard,
+ wordSimilarityPrecision: similarity.precision,
+ wordSimilarityRecall: similarity.recall,
+ wordSimilarityF1: similarity.f1,
failedPages: resp.failed_pages,
pagesProcessed: pages,
ocrPerPageMs: pages ? Math.round(ocrDurationMs / pages) : undefined,