A fast, helpful, and open-source document parser
better approach for ligatures
Logan Markewich26c29796dfa9
diff --git a/src/engines/pdf/README.md b/src/engines/pdf/README.mdindex c64cde9..f60f113 100644--- a/src/engines/pdf/README.md+++ b/src/engines/pdf/README.md@@ -47,9 +47,15 @@ PDF uses a bottom-left origin with Y pointing up. PDF.js provides transformation- `getRotation()` - Extract rotation angle from matrix**Text Decoding:**-Handles special PDF.js markers for problematic fonts:-- Decodes `:->|>_<charCode>_<fontChar>_<|<-:` format-- Handles pipe-separated characters: `|a| |b| |c|` → `abc`+Handles special PDF.js markers for problematic ("buggy") fonts — fonts whose ToUnicode/encoding maps glyphs to control characters or Private Use Area code points:+- Our patched PDF.js emits markers in the format `:->|>_<glyphId>_<fontCharCode>@<glyphName>@<|<-:` for buggy font glyphs+- The glyph name comes from the font's `/Differences` or `/Encoding` dictionary+- `decodeBuggyFontMarkers()` resolves glyph names to Unicode via the `ADOBE_GLYPH_MAP` (a subset of the Adobe Glyph List)+- Falls back to ASCII char code for glyphs in range 32-126+- Handles underscore-separated composite glyph names (e.g., `f_i` → "fi")+- Handles `uniXXXX` glyph name convention+- Also handles pipe-separated characters: `|a| |b| |c|` → `abc`+- `stripControlChars()` maps Windows-1252 C1 range (0x80-0x9F) to proper Unicode and decomposes Unicode ligatures (U+FB00-FB06) to plain text**Garbled Font Detection:**Some PDFs have fonts with corrupted or missing ToUnicode mappings, causing PDF.js to output characters mapped to unexpected Unicode code points. The `isGarbledFontOutput()` function detects this by identifying:diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.tsindex 01eb942..5986705 100644--- a/src/engines/pdf/pdfjs.ts+++ b/src/engines/pdf/pdfjs.ts@@ -100,533 +100,149 @@ const BUGGY_FONT_MARKER_CHECK = ":->|>";const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;/**- * Common tabular figures font encoding mappings.- * Many fonts with "Differences" arrays use similar patterns for tabular digits.- * These mappings are derived from common font encoding conventions.+ * Adobe Glyph List subset: maps standard PostScript glyph names to Unicode characters.*- * Note: The same PDF can use multiple fonts with DIFFERENT glyph-to-character mappings- * for the same glyph IDs. We try all mappings and pick the best match.+ * When PDF.js detects a "buggy" font (one whose ToUnicode/encoding maps glyphs to+ * control characters or PUA code points), it emits markers containing the glyph's+ * original char code AND the glyph name from the font's /Differences or /Encoding+ * dictionary. This map resolves those glyph names to correct Unicode characters.*- * Special glyphs:- * - 42: '*' (asterisk for significance markers)- * - 150: '-' (minus sign/dash)+ * Values sourced from the getGlyphsUnicode table in pdf.worker.mjs (Adobe Glyph List).*/-const TABULAR_FIGURES_MAPPINGS: Record<number, string>[] = [- // Mapping 1: Bold/header style (e.g., census PDF header row)- // Characters: 0123456789.,- {- 17: "4",- 18: "6",- 19: "8",- 20: "5",- 21: "9",- 22: "7",- 23: "1",- 24: " ",- 25: ",",- 26: "+",- 27: "-",- 28: "3",- 29: "0",- 30: "2",- 31: ".",- 42: "*",- 150: "-",- },- // Mapping 2: Book/body style (e.g., census PDF detail rows)- // Note: Same glyph IDs but different character assignments!- {- 17: "+",- 18: "7",- 19: "-",- 20: "9",- 21: "6",- 22: "3",- 23: "1",- 24: " ",- 25: "8",- 26: "5",- 27: "4",- 28: "0",- 29: "2",- 30: ".",- 31: ",",- 42: "*",- 150: "-",- },-];--/**- * Check if all glyphs in the range would produce printable ASCII via direct char code.- * Returns true if using String.fromCharCode on these glyphs would produce valid text.- */-function canDecodeAsAscii(glyphs: number[]): boolean {- // Check if ALL glyphs would produce valid printable ASCII or common whitespace- for (const g of glyphs) {- // Printable ASCII range (space through tilde), plus tab/newline- if (!((g >= 32 && g <= 126) || g === 9 || g === 10 || g === 13)) {- return false;- }- }- return true;-}--/**- * Score a decoded string for how "number-like" it appears.- * Higher scores indicate better number formatting.- */-function scoreNumberFormat(decoded: string): number {- let score = 0;-- // Count digits - primary indicator of a number- const digitCount = (decoded.match(/[0-9]/g) || []).length;- score += digitCount * 2;-- // Bonus for matching common number patterns- // Pattern: digits with optional commas for thousands- if (/^\d{1,3}(,\d{3})*$/.test(decoded)) {- score += 5; // e.g., "248,800"- }- // Pattern: decimal number- if (/^\d+\.\d+$/.test(decoded)) {- score += 5; // e.g., "10.5"- }- // Pattern: negative number- if (/^[*-]?\d/.test(decoded)) {- score += 2; // e.g., "-1,132" or "*-0.4"- }- // Pattern: percentage or simple number- if (/^\d+$/.test(decoded)) {- score += 3; // e.g., "897"- }-- // Penalize bad patterns- // Consecutive punctuation marks (not valid in numbers)- if (/[.,]{2,}/.test(decoded)) {- score -= 10;- }- // Punctuation at start (except minus/asterisk) or end- if (/^[.,+]|[.,+]$/.test(decoded)) {- score -= 5;- }- // Comma followed by anything other than 3 digits then boundary- if (/,(?!\d{3}(?:[,.]|$))/.test(decoded)) {- score -= 3;- }- // Period not followed by digits (except at end)- if (/\.(?![0-9])/.test(decoded) && !decoded.endsWith(".")) {- score -= 3;- }-- return score;-}+const ADOBE_GLYPH_MAP: Record<string, string> = {+ // Basic Latin letters+ A: "A", B: "B", C: "C", D: "D", E: "E", F: "F", G: "G", H: "H", I: "I",+ J: "J", K: "K", L: "L", M: "M", N: "N", O: "O", P: "P", Q: "Q", R: "R",+ S: "S", T: "T", U: "U", V: "V", W: "W", X: "X", Y: "Y", Z: "Z",+ a: "a", b: "b", c: "c", d: "d", e: "e", f: "f", g: "g", h: "h", i: "i",+ j: "j", k: "k", l: "l", m: "m", n: "n", o: "o", p: "p", q: "q", r: "r",+ s: "s", t: "t", u: "u", v: "v", w: "w", x: "x", y: "y", z: "z",+ // Digits+ zero: "0", one: "1", two: "2", three: "3", four: "4",+ five: "5", six: "6", seven: "7", eight: "8", nine: "9",+ // Ligatures (Unicode presentation forms — decomposed later by stripControlChars)+ fi: "\uFB01", fl: "\uFB02", ff: "\uFB00", ffi: "\uFB03", ffl: "\uFB04",+ // Punctuation and symbols+ space: " ", period: ".", comma: ",", colon: ":", semicolon: ";",+ hyphen: "-", minus: "\u2212", slash: "/", question: "?", dollar: "$",+ parenleft: "(", parenright: ")", asterisk: "*", plus: "+", equal: "=",+ numbersign: "#", percent: "%", ampersand: "&", at: "@", exclam: "!",+ bracketleft: "[", bracketright: "]", braceleft: "{", braceright: "}",+ underscore: "_", quotedbl: "\"", quotesingle: "'", backslash: "\\",+ bar: "|", asciitilde: "~", asciicircum: "^", grave: "`", less: "<", greater: ">",+ // Typographic+ quoteright: "\u2019", quoteleft: "\u2018",+ quotedblleft: "\u201C", quotedblright: "\u201D",+ quotesinglbase: "\u201A", quotedblbase: "\u201E",+ endash: "\u2013", emdash: "\u2014",+ bullet: "\u2022", ellipsis: "\u2026",+ dagger: "\u2020", daggerdbl: "\u2021",+ guilsinglleft: "\u2039", guilsinglright: "\u203A",+ guillemotleft: "\u00AB", guillemotright: "\u00BB",+ trademark: "\u2122", registered: "\u00AE", copyright: "\u00A9",+ // Greek+ Alpha: "\u0391", Beta: "\u0392", Gamma: "\u0393", Delta: "\u2206",+ Epsilon: "\u0395", Zeta: "\u0396", Eta: "\u0397", Theta: "\u0398",+ Iota: "\u0399", Kappa: "\u039A", Lambda: "\u039B", Mu: "\u039C",+ Nu: "\u039D", Xi: "\u039E", Omicron: "\u039F", Pi: "\u03A0",+ Rho: "\u03A1", Sigma: "\u03A3", Tau: "\u03A4", Upsilon: "\u03A5",+ Phi: "\u03A6", Chi: "\u03A7", Psi: "\u03A8", Omega: "\u2126",+ alpha: "\u03B1", beta: "\u03B2", gamma: "\u03B3", delta: "\u03B4",+ epsilon: "\u03B5", zeta: "\u03B6", eta: "\u03B7", theta: "\u03B8",+ iota: "\u03B9", kappa: "\u03BA", lambda: "\u03BB", mu: "\u00B5",+ nu: "\u03BD", xi: "\u03BE", omicron: "\u03BF", pi: "\u03C0",+ rho: "\u03C1", sigma: "\u03C3", tau: "\u03C4", upsilon: "\u03C5",+ phi: "\u03C6", chi: "\u03C7", psi: "\u03C8", omega: "\u03C9",+ // Math symbols+ greaterequal: "\u2265", lessequal: "\u2264", notequal: "\u2260",+ plusminus: "\u00B1", multiply: "\u00D7", divide: "\u00F7",+ infinity: "\u221E", summation: "\u2211", integral: "\u222B",+ partialdiff: "\u2202", radical: "\u221A", approxequal: "\u2248", degree: "\u00B0",+ // Accented Latin (common)+ Aacute: "\u00C1", Agrave: "\u00C0", Acircumflex: "\u00C2", Atilde: "\u00C3",+ Adieresis: "\u00C4", Aring: "\u00C5",+ Eacute: "\u00C9", Egrave: "\u00C8", Ecircumflex: "\u00CA", Edieresis: "\u00CB",+ Iacute: "\u00CD", Igrave: "\u00CC", Icircumflex: "\u00CE", Idieresis: "\u00CF",+ Oacute: "\u00D3", Ograve: "\u00D2", Ocircumflex: "\u00D4", Otilde: "\u00D5",+ Odieresis: "\u00D6",+ Uacute: "\u00DA", Ugrave: "\u00D9", Ucircumflex: "\u00DB", Udieresis: "\u00DC",+ Ntilde: "\u00D1", Ccedilla: "\u00C7", Scaron: "\u0160", Zcaron: "\u017D",+ aacute: "\u00E1", agrave: "\u00E0", acircumflex: "\u00E2", atilde: "\u00E3",+ adieresis: "\u00E4", aring: "\u00E5",+ eacute: "\u00E9", egrave: "\u00E8", ecircumflex: "\u00EA", edieresis: "\u00EB",+ iacute: "\u00ED", igrave: "\u00EC", icircumflex: "\u00EE", idieresis: "\u00EF",+ oacute: "\u00F3", ograve: "\u00F2", ocircumflex: "\u00F4", otilde: "\u00F5",+ odieresis: "\u00F6",+ uacute: "\u00FA", ugrave: "\u00F9", ucircumflex: "\u00FB", udieresis: "\u00FC",+ ntilde: "\u00F1", ccedilla: "\u00E7", scaron: "\u0161", zcaron: "\u017E",+ ydieresis: "\u00FF",+ // Miscellaneous+ AE: "\u00C6", ae: "\u00E6", OE: "\u0152", oe: "\u0153",+ Eth: "\u00D0", eth: "\u00F0", Thorn: "\u00DE", thorn: "\u00FE",+ germandbls: "\u00DF", dotlessi: "\u0131",+ section: "\u00A7", paragraph: "\u00B6", currency: "\u00A4",+ cent: "\u00A2", sterling: "\u00A3", yen: "\u00A5", Euro: "\u20AC",+ logicalnot: "\u00AC", nbspace: "\u00A0",+};/**- * Try to decode buggy font markers using known tabular figures mappings.- * Returns the decoded string if a mapping produces valid-looking text,- * otherwise returns null to fall back to charCode decoding.- *- * Strategy:- * 1. If glyphs are in ASCII range (32-126), let the fallback handle it- * 2. If glyphs are in tabular range (17-31, plus special chars), try mappings- * 3. Score each result for how "number-like" it appears- * 4. Return the best result if it looks like a valid number+ * Resolve a glyph name to its Unicode character using the Adobe Glyph List.+ * Handles standard names, the "uniXXXX" convention, and underscore-separated+ * composite names (e.g., "f_i" → resolve "f" + "i" = "fi").*/-function tryDecodeTabularFigures(str: string): string | null {- if (!str.includes(BUGGY_FONT_MARKER_CHECK)) return null;-- // Extract all glyph IDs from the markers- const glyphs: number[] = [];- let match;- const regex = /:->\|>_(\d+)_\d+_<\|<-:/g;- while ((match = regex.exec(str)) !== null) {- glyphs.push(parseInt(match[1]));- }-- if (glyphs.length === 0) return null;+function resolveGlyphName(glyphName: string): string | null {+ if (glyphName in ADOBE_GLYPH_MAP) return ADOBE_GLYPH_MAP[glyphName];- // If these glyphs would decode fine as ASCII, don't use tabular mapping- if (canDecodeAsAscii(glyphs)) {- return null;+ // Handle "uniXXXX" convention (e.g., "uni00A0" → U+00A0)+ if (glyphName.startsWith("uni") && glyphName.length === 7) {+ const code = parseInt(glyphName.slice(3), 16);+ if (!isNaN(code) && code > 0) return String.fromCharCode(code);}- // Check if glyphs are in the tabular figures range- // Tabular figures typically use glyphs 17-31, plus special chars like 42, 150- const tabularRange = glyphs.every(- (g) =>- (g >= 17 && g <= 31) || // Core tabular figures- g === 42 || // Asterisk- g === 150 || // Minus- g === 8 ||- g === 9 ||- g === 10 // Some special chars- );-- if (!tabularRange) {- // Mixed content - not pure tabular figures- return null;- }-- // Try each mapping and pick the best result- let bestResult: string | null = null;- let bestScore = -Infinity;-- for (const mapping of TABULAR_FIGURES_MAPPINGS) {- const decoded = glyphs.map((g) => mapping[g] || "").join("");-- // Skip if there are unmapped glyphs- const unmapped = glyphs.filter((g) => !mapping[g]).length;- if (unmapped > 0) continue;-- // Score based on how "number-like" the result looks- const score = scoreNumberFormat(decoded);-- if (score > bestScore) {- bestScore = score;- bestResult = decoded;+ // Handle underscore-separated composite names (e.g., "f_i" → "fi", "f_f_i" → "ffi")+ // Some fonts use this convention instead of standard ligature names+ if (glyphName.includes("_")) {+ const parts = glyphName.split("_");+ const resolved = parts.map((p) => resolveGlyphName(p));+ if (resolved.every((r) => r !== null)) {+ return resolved.join("");}}- // Only return if we got a reasonable score (at least some digits, proper format)- if (bestResult && bestScore > 0) {- return bestResult;- }-return null;}/**- * Simple decode of buggy font markers: extract glyph charCodes.- * Used as a first pass before ligature resolution.- */-function decodeBuggyFontMarkersSimple(str: string): { text: string; codes: number[] } {- const MARKER_RE = /:->\|>_(\d+)_\d+_<\|<-:/g;- const codes: number[] = [];- let m: RegExpExecArray | null;- MARKER_RE.lastIndex = 0;- while ((m = MARKER_RE.exec(str)) !== null) {- codes.push(parseInt(m[1]));- }- return { text: codes.map((c) => String.fromCharCode(c)).join(""), codes };-}--/** Composite key for font-specific ligature mapping */-function ligatureKey(fontName: string, code: number): string {- return `${fontName}:${code}`;-}--/**- * Candidate ligature/special character replacements for control-char glyphs.- * Ordered roughly by frequency in English text.- */-const LIGATURE_CANDIDATES = ["fi", "fl", "ff", "ffi", "ffl"] as const;--/**- * Known English word fragments containing each ligature.- * Used for scoring: if inserting a candidate ligature into a context produces- * a string containing any of these fragments, it's evidence for that ligature.+ * Decode buggy font markers emitted by patched PDF.js.*- * Fragments are chosen to be discriminating — they should match the intended- * ligature but NOT others. For example, "defin" matches "fi" but NOT "ffi"- * (since "deffin" is not a valid English fragment).- */-const LIGATURE_FRAGMENTS: Record<string, string[]> = {- fi: [- // Common fi words/stems- "defin",- "specif",- "first",- "final",- "field",- "file",- "find",- "five",- "fix",- "fire",- "firm",- "fish",- "fit",- "figur",- "ficti",- "fine",- "fill",- "filt",- "fing",- "finis",- "benef",- "certif",- "classif",- "confir",- "identif",- "modif",- "notif",- "qualif",- "satisf",- "signif",- "verif",- "financ",- "fiscal",- "fiber",- "fidel",- "filib",- "fifty",- "profi",- "magni",- "manif",- "paci",- "sacri",- "artif",- "scienti",- "justi",- "ratif",- "ampli",- "clari",- "digni",- "edif",- "exempl",- "falsi",- "forti",- "glori",- "horri",- "intens",- "purif",- "simpli",- "speci",- "terri",- "unifi",- "vivifi",- ],- fl: [- "floor",- "flag",- "flat",- "flip",- "flow",- "fly",- "flock",- "float",- "fled",- "flesh",- "flex",- "flaw",- "flame",- "flash",- "flu",- "reflect",- "influe",- "confli",- "inflat",- "inflam",- "afflict",- "profli",- "overfl",- "influ",- ],- ff: [- "affect",- "afford",- "differ",- "effect",- "offer",- "buffer",- "suffer",- "staff",- "stuff",- "cliff",- "bluff",- "affair",- "offend",- "offset",- "coffee",- "toffee",- "offsp",- "daffod",- "scaffo",- "effort",- "offic", // Note: "offic" could also match ffi, but "ff"+"ic" is valid- ],- ffi: [- "offici",- "effici",- "traffi",- "suffici",- "affili",- "affida",- "graffi",- "coffin",- "muffin",- "puffin",- "affini",- "affix",- "suffix",- "daffil",- ],- ffl: [- "baffle",- "raffle",- "shuffle",- "waffle",- "scaffol",- "ruffle",- "muffle",- "sniffle",- "piffle",- "riffle",- "duffle",- "truffle",- ],-};--/**- * Accumulate ligature contexts from buggy font items into a document-level map.- * Keyed by `fontName:glyphCode` to handle multi-font documents correctly.- */-function accumulateLigatureContexts(- buggyItems: Array<{ codes: number[]; fontName: string }>,- accumulator: Map<string, Array<{ before: string; after: string }>>-): void {- for (const { codes, fontName } of buggyItems) {- for (let i = 0; i < codes.length; i++) {- const code = codes[i];- if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) continue;-- const before = codes- .slice(Math.max(0, i - 4), i)- .filter((c) => c >= 0x20)- .map((c) => String.fromCharCode(c))- .join("");- const after = codes- .slice(i + 1, i + 5)- .filter((c) => c >= 0x20)- .map((c) => String.fromCharCode(c))- .join("");-- const key = ligatureKey(fontName, code);- if (!accumulator.has(key)) accumulator.set(key, []);- accumulator.get(key)!.push({ before, after });- }- }-}--/**- * Build a mapping from control-char glyph codes to their actual characters- * using accumulated contexts across all pages.+ * Marker format: :->|>_<glyphId>_<fontCharCode>@<glyphName>@<|<-:+ * The glyph name is delimited by @ instead of _ because some fonts use+ * non-standard glyph names containing underscores (e.g., "f_i" for "fi").*- * Uses a greedy assignment strategy: score all (code, candidate) pairs,- * then assign the highest-confidence mapping first, ensuring each ligature- * is only assigned to one glyph code.+ * Resolution strategy:+ * 1. Use glyph name from font's /Differences or /Encoding dictionary+ * 2. Fall back to glyphId if it's in printable ASCII range (32-126)+ * 3. Drop the character if neither works (better than guessing)*/-function buildControlCharMappingFromContexts(- contextsByCode: Map<string, Array<{ before: string; after: string }>>-): Map<string, string> {- const mapping = new Map<string, string>();-- // Score all (key, candidate) pairs- const allScores: Array<{ code: string; candidate: string; score: number }> = [];-- for (const [code, contexts] of contextsByCode) {- for (const candidate of LIGATURE_CANDIDATES) {- let matchCount = 0;- const fragments = LIGATURE_FRAGMENTS[candidate] || [];- for (const { before, after } of contexts) {- const combined = (before + candidate + after).toLowerCase();- // Check if any known word fragment appears in the combined context- for (const frag of fragments) {- if (combined.includes(frag)) {- matchCount++;- break; // One match per context is enough- }- }- }- // Score = match ratio (0 to 1) * 100- const matchRatio = contexts.length > 0 ? matchCount / contexts.length : 0;- allScores.push({ code, candidate, score: matchRatio * 100 });- }+const BUGGY_FONT_MARKER_RE = /:->\|>_(\d+)_\d+@([^@]*)@<\|<-:/g;- // Also check for special characters (en-dash, em-dash)- let digitPairCount = 0;- let spacePairCount = 0;- for (const { before, after } of contexts) {- if (/[0-9]$/.test(before) && /^[0-9]/.test(after)) digitPairCount++;- if (/\s$/.test(before) || /^\s/.test(after)) spacePairCount++;- }- if (digitPairCount > 0 && digitPairCount >= contexts.length * 0.3) {- allScores.push({ code, candidate: "\u2013", score: digitPairCount * 4 }); // en-dash+function decodeBuggyFontMarkers(str: string): string {+ return str.replace(BUGGY_FONT_MARKER_RE, (_match, glyphIdStr: string, glyphName: string) => {+ // Priority 1: Resolve via glyph name from font metadata+ if (glyphName) {+ const resolved = resolveGlyphName(glyphName);+ if (resolved) return resolved;}- if (spacePairCount > 0 && spacePairCount >= contexts.length * 0.3) {- allScores.push({ code, candidate: "\u2014", score: spacePairCount * 4 }); // em-dash- }- }-- // Greedy assignment: assign highest-confidence mappings first.- // Enforce uniqueness per font — each ligature maps to at most one glyph code- // within the same font.- allScores.sort((a, b) => b.score - a.score);- // Track assigned candidates per font: fontName → Set<candidate>- const assignedPerFont = new Map<string, Set<string>>();-- for (const { code, candidate, score } of allScores) {- if (mapping.has(code)) continue;- if (score <= 0) continue;-- // Extract fontName from the composite key "fontName:glyphCode"- const fontName = code.substring(0, code.lastIndexOf(":"));- if (!assignedPerFont.has(fontName)) assignedPerFont.set(fontName, new Set());- const fontAssigned = assignedPerFont.get(fontName)!;-- // Only enforce uniqueness for ligatures, not special chars- if (LIGATURE_CANDIDATES.includes(candidate as (typeof LIGATURE_CANDIDATES)[number])) {- if (fontAssigned.has(candidate)) continue;- fontAssigned.add(candidate);- }-- mapping.set(code, candidate);- }- // Fallback: assign "fi" to any remaining unmapped codes- for (const code of contextsByCode.keys()) {- if (!mapping.has(code)) {- const fontName = code.substring(0, code.lastIndexOf(":"));- if (!assignedPerFont.has(fontName)) assignedPerFont.set(fontName, new Set());- const fontAssigned = assignedPerFont.get(fontName)!;-- for (const candidate of LIGATURE_CANDIDATES) {- if (!fontAssigned.has(candidate)) {- mapping.set(code, candidate);- fontAssigned.add(candidate);- break;- }- }- if (!mapping.has(code)) {- mapping.set(code, "fi");- }+ // Priority 2: If glyphId is in printable ASCII range, use it directly+ const glyphId = parseInt(glyphIdStr);+ if (glyphId >= 32 && glyphId <= 126) {+ return String.fromCharCode(glyphId);}- }-- return mapping;-}-/**- * Apply the control-char mapping to decode a buggy font text item.- */-function applyControlCharMapping(- codes: number[],- fontName: string,- mapping: Map<string, string>-): string {- const result: string[] = [];- for (const code of codes) {- if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {- result.push(String.fromCharCode(code));- } else {- const mapped = mapping.get(ligatureKey(fontName, code));- if (mapped) {- result.push(mapped);- }- // Unknown control chars are dropped- }- }- return result.join("");+ // Priority 3: Drop unresolvable characters+ return "";+ });}/**@@ -849,19 +465,7 @@ export class PdfJsEngine implements PdfEngine {private currentPdfPath: string | null = null;private currentPdfData: Uint8Array | null = null;- /**- * Document-level accumulator for buggy font ligature contexts.- * Keyed by `fontName:glyphCode` to handle multi-font documents.- * Persists across extractPage calls so the mapping improves with more data.- */- private ligatureContextAccumulator = new Map<string, Array<{ before: string; after: string }>>();- private resolvedLigatureMap = new Map<string, string>();-async loadDocument(input: string | Uint8Array, password?: string): Promise<PdfDocument> {- // Reset ligature cache for each new document- this.ligatureContextAccumulator = new Map();- this.resolvedLigatureMap = new Map();-let data: Uint8Array;if (typeof input === "string") {data = new Uint8Array(await fs.readFile(input));@@ -931,21 +535,8 @@ export class PdfJsEngine implements PdfEngine {const viewportHeight = viewport.height;const viewportTransform = viewport.transform;- // Pre-process all items: compute geometry and identify buggy font items- interface PreprocessedItem {- item: PdfJsTextItem;- left: number;- top: number;- width: number;- height: number;- rotation: number;- isBuggyFont: boolean;- tabularDecoded?: string;- buggyDecoded?: { text: string; codes: number[] };- }-- const preprocessed: PreprocessedItem[] = [];- const buggyFontItems: Array<{ codes: number[]; fontName: string }> = [];+ const textItems: TextItem[] = [];+ const garbledTextRegions: BoundingBox[] = [];for (const item of textContent.items) {// Skip items with zero dimensions@@ -976,58 +567,11 @@ export class PdfJsEngine implements PdfEngine {let rotation = getRotation(cm);if (rotation < 0) rotation += 360;- const pre: PreprocessedItem = {- item,- left,- top,- width,- height,- rotation,- isBuggyFont: false,- };-- if (item.str.includes(BUGGY_FONT_MARKER_CHECK)) {- const tabularDecoded = tryDecodeTabularFigures(item.str);- if (tabularDecoded) {- pre.tabularDecoded = tabularDecoded;- } else {- pre.isBuggyFont = true;- const decoded = decodeBuggyFontMarkersSimple(item.str);- pre.buggyDecoded = decoded;- buggyFontItems.push({ codes: decoded.codes, fontName: item.fontName || "" });- }- }-- preprocessed.push(pre);- }-- // Accumulate contexts from this page's buggy font items into the document-level cache,- // then rebuild the mapping using ALL accumulated contexts across all pages seen so far.- if (buggyFontItems.length > 0) {- accumulateLigatureContexts(buggyFontItems, this.ligatureContextAccumulator);- this.resolvedLigatureMap = buildControlCharMappingFromContexts(- this.ligatureContextAccumulator- );- }- const controlCharMap = this.resolvedLigatureMap;-- // Second pass: decode all items using the resolved mapping- const textItems: TextItem[] = [];- const garbledTextRegions: BoundingBox[] = [];-- for (const pre of preprocessed) {- let decodedStr: string;-- if (pre.tabularDecoded) {- decodedStr = pre.tabularDecoded;- } else if (pre.isBuggyFont && pre.buggyDecoded) {- decodedStr = applyControlCharMapping(- pre.buggyDecoded.codes,- pre.item.fontName || "",- controlCharMap- );- } else {- decodedStr = pre.item.str;+ // Decode buggy font markers using glyph names from font metadata+ let decodedStr = item.str;+ if (decodedStr.includes(BUGGY_FONT_MARKER_CHECK)) {+ BUGGY_FONT_MARKER_RE.lastIndex = 0;+ decodedStr = decodeBuggyFontMarkers(decodedStr);}// Handle pipe-separated characters: " |a| |r| |X| " -> "arX"@@ -1041,7 +585,7 @@ export class PdfJsEngine implements PdfEngine {// Skip garbled text from fonts with corrupted ToUnicode mappingsif (isGarbledFontOutput(decodedStr)) {- garbledTextRegions.push({ x: pre.left, y: pre.top, width: pre.width, height: pre.height });+ garbledTextRegions.push({ x: left, y: top, width, height });continue;}@@ -1050,17 +594,17 @@ export class PdfJsEngine implements PdfEngine {textItems.push({str: decodedStr,- x: pre.left,- y: pre.top,- width: pre.width,- height: pre.height,- w: pre.width,- h: pre.height,- r: pre.rotation,- fontName: pre.item.fontName,+ x: left,+ y: top,+ width,+ height,+ w: width,+ h: height,+ r: rotation,+ fontName: item.fontName,fontSize: Math.sqrt(- pre.item.transform[0] * pre.item.transform[0] +- pre.item.transform[1] * pre.item.transform[1]+ item.transform[0] * item.transform[0] ++ item.transform[1] * item.transform[1]),confidence: 1.0,});@@ -1103,51 +647,6 @@ export class PdfJsEngine implements PdfEngine {};}- /**- * Pre-scan pages for buggy font markers to build the ligature mapping- * before doing full extraction. This is a lightweight pass that only reads- * text content — no image extraction, OCR, or coordinate transforms.- */- private async prescanForLigatures(doc: PdfDocument, pageNumbers: number[]): Promise<void> {- const pdfDocument = (doc as PdfJsExtendedDocument)._pdfDocument;- // Sample up to 20 pages spread evenly across the document- const sampleSize = Math.min(pageNumbers.length, 20);- const step = Math.max(1, Math.floor(pageNumbers.length / sampleSize));- const samplePages = pageNumbers.filter((_, i) => i % step === 0).slice(0, sampleSize);-- let hasBuggyFonts = false;-- for (const pageNum of samplePages) {- const page = await pdfDocument.getPage(pageNum);- const tc = await page.getTextContent();-- const buggyItems: Array<{ codes: number[]; fontName: string }> = [];- for (const item of tc.items) {- if (typeof item.str !== "string") continue;- if (item.str.includes(BUGGY_FONT_MARKER_CHECK)) {- const tabularDecoded = tryDecodeTabularFigures(item.str);- if (!tabularDecoded) {- const decoded = decodeBuggyFontMarkersSimple(item.str);- buggyItems.push({ codes: decoded.codes, fontName: item.fontName || "" });- hasBuggyFonts = true;- }- }- }-- if (buggyItems.length > 0) {- accumulateLigatureContexts(buggyItems, this.ligatureContextAccumulator);- }-- await page.cleanup();- }-- if (hasBuggyFonts) {- this.resolvedLigatureMap = buildControlCharMappingFromContexts(- this.ligatureContextAccumulator- );- }- }-async extractAllPages(doc: PdfDocument,maxPages?: number,@@ -1166,9 +665,6 @@ export class PdfJsEngine implements PdfEngine {pageNumbers = Array.from({ length: numPages }, (_, i) => i + 1);}- // Pre-scan pages to build ligature mapping before full extraction- await this.prescanForLigatures(doc, pageNumbers);-for (const pageNum of pageNumbers) {if (maxPages && pages.length >= maxPages) {break;diff --git a/src/vendor/pdfjs/pdf.worker.mjs b/src/vendor/pdfjs/pdf.worker.mjsindex d95a664..20d367b 100644--- a/src/vendor/pdfjs/pdf.worker.mjs+++ b/src/vendor/pdfjs/pdf.worker.mjs@@ -31572,7 +31572,8 @@ class PartialEvaluator {}saveLastChar(glyphUnicode);if (fontIsBuggy) {- textChunk.str.push(`:->|>_${glyph.originalCharCode}_${glyph.fontChar.charCodeAt(0) || glyph.unicode.charCodeAt(0) || 0}_<|<-:`);+ const _glyphName = font.differences?.[glyph.originalCharCode] || font.defaultEncoding?.[glyph.originalCharCode] || '';+ textChunk.str.push(`:->|>_${glyph.originalCharCode}_${glyph.fontChar.charCodeAt(0) || glyph.unicode.charCodeAt(0) || 0}@${_glyphName}@<|<-:`);} else {textChunk.str.push(glyphUnicode);}