Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add precision, recall, F1 metrics to self-hosted OCR experiment (#3253)
Abimael Martell4e0f65e39118
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.tsindex 9401a7132..ffb68dd34 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts@@ -3,12 +3,12 @@ import { config } from "../../../../config";import { robustFetch } from "../../lib/fetch";import { z } from "zod";-/**- * Compute word-level Jaccard similarity between two texts.- * Strips markdown syntax and normalises whitespace so we compare- * the underlying data, not formatting differences.- */-function wordSimilarity(a: string, b: string): number {+function computeSimilarityMetrics(a: string, b: string): {+ jaccard: number;+ precision: number;+ recall: number;+ f1: number;+} {const normalise = (s: string) =>s.replace(/[#*_`\[\]()>|~\-]/g, " ")@@ -19,15 +19,29 @@ function wordSimilarity(a: string, b: string): number {const wordsA = new Set(normalise(a).split(" ").filter(Boolean));const wordsB = new Set(normalise(b).split(" ").filter(Boolean));- if (wordsA.size === 0 && wordsB.size === 0) return 1;- if (wordsA.size === 0 || wordsB.size === 0) return 0;+ if (wordsA.size === 0 && wordsB.size === 0) {+ return { jaccard: 1, precision: 1, recall: 1, f1: 1 };+ }+ if (wordsA.size === 0 || wordsB.size === 0) {+ return { jaccard: 0, precision: 0, recall: 0, f1: 0 };+ }let intersection = 0;for (const w of wordsA) {if (wordsB.has(w)) intersection++;}- return intersection / (wordsA.size + wordsB.size - intersection);+ const jaccard = intersection / (wordsA.size + wordsB.size - intersection);+ const precision = intersection / wordsA.size; // % of our words in MinerU+ const recall = intersection / wordsB.size; // % of MinerU words in ours+ const f1 = (2 * precision * recall) / (precision + recall) || 0;++ return {+ jaccard: Math.round(jaccard * 1000) / 1000,+ precision: Math.round(precision * 1000) / 1000,+ recall: Math.round(recall * 1000) / 1000,+ f1: Math.round(f1 * 1000) / 1000,+ };}export function runSelfHostedOCRExperiment(@@ -70,7 +84,7 @@ export function runSelfHostedOCRExperiment(abort: meta.abort.asSignal(),});const ocrDurationMs = Date.now() - startedAt;- const similarity = wordSimilarity(resp.markdown, muV1Result.markdown);+ const similarity = computeSimilarityMetrics(resp.markdown, muV1Result.markdown);const pages = resp.pages_processed ?? pagesProcessed;const timeDiffMs = muV1Result.durationMs - ocrDurationMs;const speedup = muV1Result.durationMs > 0 && ocrDurationMs > 0@@ -86,7 +100,10 @@ export function runSelfHostedOCRExperiment(speedup,ocrMarkdownLength: resp.markdown.length,muV1MarkdownLength: muV1Result.markdown.length,- wordSimilarity: Math.round(similarity * 1000) / 1000,+ wordSimilarity: similarity.jaccard,+ wordSimilarityPrecision: similarity.precision,+ wordSimilarityRecall: similarity.recall,+ wordSimilarityF1: similarity.f1,failedPages: resp.failed_pages,pagesProcessed: pages,ocrPerPageMs: pages ? Math.round(ocrDurationMs / pages) : undefined,