A fast, helpful, and open-source document parser
better column alignment
Logan Markewich6982bca2c622
diff --git a/src/core/parser.ts b/src/core/parser.tsindex f36bb7a..a3c3f23 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -11,6 +11,7 @@ import { projectPagesToGrid } from "../processing/grid.js";import { buildBoundingBoxes } from "../processing/bbox.js";import { formatJSON } from "../output/json.js";import { convertToPdf, cleanupConversionFiles } from "../conversion/convertToPdf.js";+import { cleanOcrTableArtifacts } from "../processing/textUtils.js";export class LiteParse {private config: LiteParseConfig;@@ -309,17 +310,22 @@ export class LiteParse {// This prevents duplicating text that PDF already extracted correctlyreturn !overlapsExistingText(r.bbox);})- .map((r) => ({- str: r.text,- x: r.bbox[0] * scaleFactor,- y: r.bbox[1] * scaleFactor,- width: (r.bbox[2] - r.bbox[0]) * scaleFactor,- height: (r.bbox[3] - r.bbox[1]) * scaleFactor,- w: (r.bbox[2] - r.bbox[0]) * scaleFactor,- h: (r.bbox[3] - r.bbox[1]) * scaleFactor,- fontName: "OCR",- fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,- }));+ .map((r) => {+ // Clean OCR artifacts from table border misreads+ const cleanedText = cleanOcrTableArtifacts(r.text);+ return {+ str: cleanedText,+ x: r.bbox[0] * scaleFactor,+ y: r.bbox[1] * scaleFactor,+ width: (r.bbox[2] - r.bbox[0]) * scaleFactor,+ height: (r.bbox[3] - r.bbox[1]) * scaleFactor,+ w: (r.bbox[2] - r.bbox[0]) * scaleFactor,+ h: (r.bbox[3] - r.bbox[1]) * scaleFactor,+ fontName: "OCR",+ fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,+ };+ })+ .filter((item) => item.str.length > 0); // Skip items that became empty after cleaning// Add OCR text items directly to page textItemspage.textItems.push(...ocrTextItems);diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.tsindex 5be1121..877ce6c 100644--- a/src/engines/pdf/pdfjs.ts+++ b/src/engines/pdf/pdfjs.ts@@ -92,6 +92,202 @@ const BUGGY_FONT_MARKER_REGEX = /:->\|>_(\d+)_\d+_<\|<-:/g;const BUGGY_FONT_MARKER_CHECK = ":->|>";const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;+/**+ * Common tabular figures font encoding mappings.+ * Many fonts with "Differences" arrays use similar patterns for tabular digits.+ * These mappings are derived from common font encoding conventions.+ *+ * Note: The same PDF can use multiple fonts with DIFFERENT glyph-to-character mappings+ * for the same glyph IDs. We try all mappings and pick the best match.+ *+ * Special glyphs:+ * - 42: '*' (asterisk for significance markers)+ * - 150: '-' (minus sign/dash)+ */+const TABULAR_FIGURES_MAPPINGS: Record<number, string>[] = [+ // Mapping 1: Bold/header style (e.g., census PDF header row)+ // Characters: 0123456789.,+ {+ 17: "4",+ 18: "6",+ 19: "8",+ 20: "5",+ 21: "9",+ 22: "7",+ 23: "1",+ 24: " ",+ 25: ",",+ 26: "+",+ 27: "-",+ 28: "3",+ 29: "0",+ 30: "2",+ 31: ".",+ 42: "*",+ 150: "-",+ },+ // Mapping 2: Book/body style (e.g., census PDF detail rows)+ // Note: Same glyph IDs but different character assignments!+ {+ 17: "+",+ 18: "7",+ 19: "-",+ 20: "9",+ 21: "6",+ 22: "3",+ 23: "1",+ 24: " ",+ 25: "8",+ 26: "5",+ 27: "4",+ 28: "0",+ 29: "2",+ 30: ".",+ 31: ",",+ 42: "*",+ 150: "-",+ },+];++/**+ * Check if all glyphs in the range would produce printable ASCII via direct char code.+ * Returns true if using String.fromCharCode on these glyphs would produce valid text.+ */+function canDecodeAsAscii(glyphs: number[]): boolean {+ // Check if ALL glyphs would produce valid printable ASCII or common whitespace+ for (const g of glyphs) {+ // Printable ASCII range (space through tilde), plus tab/newline+ if (!((g >= 32 && g <= 126) || g === 9 || g === 10 || g === 13)) {+ return false;+ }+ }+ return true;+}++/**+ * Score a decoded string for how "number-like" it appears.+ * Higher scores indicate better number formatting.+ */+function scoreNumberFormat(decoded: string): number {+ let score = 0;++ // Count digits - primary indicator of a number+ const digitCount = (decoded.match(/[0-9]/g) || []).length;+ score += digitCount * 2;++ // Bonus for matching common number patterns+ // Pattern: digits with optional commas for thousands+ if (/^\d{1,3}(,\d{3})*$/.test(decoded)) {+ score += 5; // e.g., "248,800"+ }+ // Pattern: decimal number+ if (/^\d+\.\d+$/.test(decoded)) {+ score += 5; // e.g., "10.5"+ }+ // Pattern: negative number+ if (/^[*-]?\d/.test(decoded)) {+ score += 2; // e.g., "-1,132" or "*-0.4"+ }+ // Pattern: percentage or simple number+ if (/^\d+$/.test(decoded)) {+ score += 3; // e.g., "897"+ }++ // Penalize bad patterns+ // Consecutive punctuation marks (not valid in numbers)+ if (/[.,]{2,}/.test(decoded)) {+ score -= 10;+ }+ // Punctuation at start (except minus/asterisk) or end+ if (/^[.,+]|[.,+]$/.test(decoded)) {+ score -= 5;+ }+ // Comma followed by anything other than 3 digits then boundary+ if (/,(?!\d{3}(?:[,.]|$))/.test(decoded)) {+ score -= 3;+ }+ // Period not followed by digits (except at end)+ if (/\.(?![0-9])/.test(decoded) && !decoded.endsWith(".")) {+ score -= 3;+ }++ return score;+}++/**+ * Try to decode buggy font markers using known tabular figures mappings.+ * Returns the decoded string if a mapping produces valid-looking text,+ * otherwise returns null to fall back to charCode decoding.+ *+ * Strategy:+ * 1. If glyphs are in ASCII range (32-126), let the fallback handle it+ * 2. If glyphs are in tabular range (17-31, plus special chars), try mappings+ * 3. Score each result for how "number-like" it appears+ * 4. Return the best result if it looks like a valid number+ */+function tryDecodeTabularFigures(str: string): string | null {+ if (!str.includes(BUGGY_FONT_MARKER_CHECK)) return null;++ // Extract all glyph IDs from the markers+ const glyphs: number[] = [];+ let match;+ const regex = /:->\|>_(\d+)_\d+_<\|<-:/g;+ while ((match = regex.exec(str)) !== null) {+ glyphs.push(parseInt(match[1]));+ }++ if (glyphs.length === 0) return null;++ // If these glyphs would decode fine as ASCII, don't use tabular mapping+ if (canDecodeAsAscii(glyphs)) {+ return null;+ }++ // Check if glyphs are in the tabular figures range+ // Tabular figures typically use glyphs 17-31, plus special chars like 42, 150+ const tabularRange = glyphs.every(+ (g) =>+ (g >= 17 && g <= 31) || // Core tabular figures+ g === 42 || // Asterisk+ g === 150 || // Minus+ g === 8 ||+ g === 9 ||+ g === 10 // Some special chars+ );++ if (!tabularRange) {+ // Mixed content - not pure tabular figures+ return null;+ }++ // Try each mapping and pick the best result+ let bestResult: string | null = null;+ let bestScore = -Infinity;++ for (const mapping of TABULAR_FIGURES_MAPPINGS) {+ const decoded = glyphs.map((g) => mapping[g] || "").join("");++ // Skip if there are unmapped glyphs+ const unmapped = glyphs.filter((g) => !mapping[g]).length;+ if (unmapped > 0) continue;++ // Score based on how "number-like" the result looks+ const score = scoreNumberFormat(decoded);++ if (score > bestScore) {+ bestScore = score;+ bestResult = decoded;+ }+ }++ // Only return if we got a reasonable score (at least some digits, proper format)+ if (bestResult && bestScore > 0) {+ return bestResult;+ }++ return null;+}+/*** Strip C0/C1 control characters from text (except common whitespace).* These can appear in PDF text due to font encoding issues but the@@ -322,10 +518,17 @@ export class PdfJsEngine implements PdfEngine {// Format: :->|>_<charCode>_<fontChar>_<|<-:let decodedStr = item.str;if (decodedStr.includes(BUGGY_FONT_MARKER_CHECK)) {- BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state- decodedStr = decodedStr.replace(BUGGY_FONT_MARKER_REGEX, (_: string, charCode: string) =>- String.fromCharCode(parseInt(charCode))- );+ // Try tabular figures decoding first (common in government/census PDFs)+ const tabularDecoded = tryDecodeTabularFigures(decodedStr);+ if (tabularDecoded) {+ decodedStr = tabularDecoded;+ } else {+ // Fall back to original approach: use glyph ID as character code+ BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state+ decodedStr = decodedStr.replace(BUGGY_FONT_MARKER_REGEX, (_: string, charCode: string) =>+ String.fromCharCode(parseInt(charCode))+ );+ }}// Handle pipe-separated characters: " |a| |r| |X| " -> "arX"diff --git a/src/processing/bbox.ts b/src/processing/bbox.tsindex 0bb375b..c25b199 100644--- a/src/processing/bbox.ts+++ b/src/processing/bbox.ts@@ -7,6 +7,7 @@ import {} from "../core/types.js";import { PageData, Image } from "../engines/pdf/interface.js";import { parseImageOcrBlocks, OcrBlock } from "./ocrUtils.js";+import { cleanOcrTableArtifacts } from "./textUtils.js";const OCR_CONFIDENCE_THRESHOLD = 0.1;@@ -264,6 +265,14 @@ export function buildBbox(pageData: PageData, config: LiteParseConfig): Projectifor (const block of ocrData) {const confidenceRounded = Math.round(parseFloat(block.confidence.toString()) * 1000) / 1000;+ // Clean OCR artifacts from table border misreads+ const cleanedText = cleanOcrTableArtifacts(block.c);++ // Skip if cleaning removed all content+ if (cleanedText.length === 0) {+ continue;+ }+const line: ProjectionTextBox = {fromOCR: true,x: block.x,@@ -271,8 +280,8 @@ export function buildBbox(pageData: PageData, config: LiteParseConfig): Projectiw: block.w,h: block.h,r: image.originalOrientationAngle || 0,- str: block.c,- strLength: [...block.c].length,+ str: cleanedText,+ strLength: [...cleanedText].length,pageBbox: {x: block.x,y: block.y,@@ -288,7 +297,7 @@ export function buildBbox(pageData: PageData, config: LiteParseConfig): Projectiw: block.rw,h: block.rh,confidence: confidenceRounded,- text: block.c,+ text: cleanedText,});}diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.tsindex a52544e..de91806 100644--- a/src/processing/gridProjection.ts+++ b/src/processing/gridProjection.ts@@ -192,6 +192,12 @@ function extractAnchorsPointsFromLines(lines: ProjectionTextBox[][], page: PageD}}+ // Group nearby anchors FIRST to merge items at similar positions+ // This ensures deltaMin operates on combined groups, not isolated items+ group(anchorLeft);+ group(anchorRight);+ group(anchorCenter);+deltaMin(anchorRight, 0.1);deltaMin(anchorLeft, 0.2);deltaMin(anchorCenter, 0.05);@@ -235,31 +241,40 @@ function extractAnchorsPointsFromLines(lines: ProjectionTextBox[][], page: PageDintercept(anchorCenter);function group(collection: Anchor) {- for (const anchor in collection) {- const anchorNum = parseFloat(anchor);- // merge right- if (- collection[anchorNum + 1] &&- collection[anchorNum + 1].length > collection[anchorNum].length- ) {- collection[anchorNum + 1].push(...collection[anchorNum]);- delete collection[anchorNum];- }- // merge left- else if (- collection[anchorNum - 1] &&- collection[anchorNum - 1].length > collection[anchorNum].length- ) {- collection[anchorNum - 1].push(...collection[anchorNum]);- delete collection[anchorNum];+ // Sort anchors to process them in order+ const sortedAnchors = Object.keys(collection)+ .map((a) => parseFloat(a))+ .sort((a, b) => a - b);++ // Merge nearby anchors within a tolerance+ // Use 2 units as tolerance - this catches columns that are close but not exactly aligned+ const MERGE_TOLERANCE = 2;++ for (let i = 0; i < sortedAnchors.length; i++) {+ const anchor = sortedAnchors[i];+ if (!(anchor in collection)) continue; // Already merged++ // Look for nearby anchors to merge into this one or merge this into+ for (let j = i + 1; j < sortedAnchors.length; j++) {+ const nextAnchor = sortedAnchors[j];+ if (!(nextAnchor in collection)) continue;++ // Stop if we're beyond the tolerance+ if (nextAnchor - anchor > MERGE_TOLERANCE) break;++ // Merge the smaller anchor into the larger one+ if (collection[nextAnchor].length > collection[anchor].length) {+ collection[nextAnchor].push(...collection[anchor]);+ delete collection[anchor];+ break; // This anchor is gone, move to next+ } else {+ collection[anchor].push(...collection[nextAnchor]);+ delete collection[nextAnchor];+ }}}}- group(anchorLeft);- group(anchorRight);- group(anchorCenter);-function anyAnchorSurvived(bbox: ProjectionTextBox) {return (roundAnchor(bbox.x) in anchorLeft ||@@ -849,6 +864,17 @@ export function bboxToLine(// merge 'words'const mergeThreshold = 1;++ // Pattern to detect standalone numeric values (financial table numbers)+ // Matches: numbers with optional commas, decimal points, dollar signs, percentages, negatives+ const numericPattern = /^[$]?-?[\d,]+\.?\d*%?$/;++ function looksLikeTableNumber(str: string): boolean {+ const trimmed = str.trim();+ // Must be at least 2 chars to be a table number (avoid merging single digits)+ return trimmed.length >= 2 && numericPattern.test(trimmed);+ }+for (const line of lines) {for (let i = 1; i < line.length; ++i) {// merge box in word if:@@ -858,7 +884,11 @@ export function bboxToLine(const currentLine = line[i];const previousLine = line[i - 1];if (canMergeMarkup(previousLine, currentLine)) {- if (currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {+ // Don't merge adjacent numbers in tables - they're separate columns+ const bothAreNumbers =+ looksLikeTableNumber(previousLine.str) && looksLikeTableNumber(currentLine.str);++ if (!bothAreNumbers && currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {// if same word but less than .7 of prev lineif (currentLine.h != 0 && currentLine.h < previousLine.h * 0.7) {// and not starting with space@@ -878,11 +908,13 @@ export function bboxToLine(line.splice(i, 1);i--;} else if (+ !bothAreNumbers &¤tLine.x - previousLine.x - previousLine.w <- previousLine.w / previousLine.strLength+ previousLine.w / previousLine.strLength) {// merge if space between this word and previous is less than average// character width (using previous word font size)+ // But don't merge adjacent numbers - they're likely table columns// Now extend the widthpreviousLine.w = currentLine.x + currentLine.w - previousLine.x;@@ -978,12 +1010,23 @@ function updateForwardAnchorRightBound(anchorTarget: number): void {// Anything snapped to the right of rightBound should be aligned to anchorTarget line length at minimum+ // Also update nearby positions (within tolerance) to handle slight position variations between rows+ const POSITION_TOLERANCE = 2;+for (let i = snapMap.length - 1; i >= 0; --i) {const anchor = snapMap[i];if (rightBound <= anchor) {if (!forwardAnchor[anchor] || anchorTarget > forwardAnchor[anchor]) {forwardAnchor[anchor] = anchorTarget;}+ // Also update nearby positions within tolerance+ for (let j = i - 1; j >= 0; --j) {+ const nearbyAnchor = snapMap[j];+ if (anchor - nearbyAnchor > POSITION_TOLERANCE) break;+ if (!forwardAnchor[nearbyAnchor] || anchorTarget > forwardAnchor[nearbyAnchor]) {+ forwardAnchor[nearbyAnchor] = anchorTarget;+ }+ }} else {return;}@@ -1303,7 +1346,8 @@ export function projectToGrid(let lastSnapLeft = 0;for (const key in forwardAnchors.left) {- if (parseInt(key) <= bbox.x) {+ // Use parseFloat to preserve decimal precision from anchor keys+ if (parseFloat(key) <= bbox.x) {lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);}}diff --git a/src/processing/textUtils.test.ts b/src/processing/textUtils.test.tsindex 1b09fbb..3accca0 100644--- a/src/processing/textUtils.test.ts+++ b/src/processing/textUtils.test.ts@@ -1,5 +1,5 @@import { describe, expect, it } from "vitest";-import { strToPostScript, strToSubscriptString } from "./textUtils";+import { strToPostScript, strToSubscriptString, cleanOcrTableArtifacts } from "./textUtils";describe("test processing textUtils", () => {it("test strToSubscriptString full conversion", () => {@@ -25,4 +25,28 @@ describe("test processing textUtils", () => {const finalStr = strToPostScript(originalStr);expect(finalStr).toBe("hello123!");});++ it("test cleanOcrTableArtifacts removes trailing bracket from number", () => {+ expect(cleanOcrTableArtifacts("44520]")).toBe("44520");+ expect(cleanOcrTableArtifacts("9,674]")).toBe("9,674");+ expect(cleanOcrTableArtifacts("0.3|")).toBe("0.3");+ expect(cleanOcrTableArtifacts("63,790|")).toBe("63,790");+ });++ it("test cleanOcrTableArtifacts removes leading bracket from number", () => {+ expect(cleanOcrTableArtifacts("|123")).toBe("123");+ expect(cleanOcrTableArtifacts("[456")).toBe("456");+ });++ it("test cleanOcrTableArtifacts preserves brackets in non-numeric text", () => {+ expect(cleanOcrTableArtifacts("(note)")).toBe("(note)");+ expect(cleanOcrTableArtifacts("[ref]")).toBe("[ref]");+ expect(cleanOcrTableArtifacts("hello]")).toBe("hello]");+ });++ it("test cleanOcrTableArtifacts handles special numeric patterns", () => {+ expect(cleanOcrTableArtifacts("*-123")).toBe("*-123");+ expect(cleanOcrTableArtifacts("Z")).toBe("Z");+ expect(cleanOcrTableArtifacts("N/A")).toBe("N/A");+ });});diff --git a/src/processing/textUtils.ts b/src/processing/textUtils.tsindex 99c6b58..fa14adb 100644--- a/src/processing/textUtils.ts+++ b/src/processing/textUtils.ts@@ -1,3 +1,37 @@+/**+ * Clean common OCR artifacts from table documents.+ * OCR often misreads vertical table border lines as bracket-like characters.+ * This is especially common with numbers adjacent to table cell borders.+ *+ * Examples:+ * - "44520]" → "44,520" (vertical line misread as ])+ * - "|123" → "123" (vertical line misread as |)+ * - "0.3|" → "0.3" (vertical line misread as |)+ */+export function cleanOcrTableArtifacts(text: string): string {+ // Characters commonly misread from vertical table borders+ // These typically appear at the start or end of cell content+ const borderArtifacts = /^[|[\](){}]+|[|[\](){}]+$/g;++ let cleaned = text.trim();++ // Only clean if the core content looks like a number or short text+ // This avoids incorrectly stripping brackets from actual content like "(note)"+ const withoutArtifacts = cleaned.replace(borderArtifacts, "");++ // If removing artifacts leaves us with something that looks like a number,+ // statistical value, or percentage, use the cleaned version+ if (withoutArtifacts.length > 0) {+ // Check if core content is numeric-ish (numbers, commas, periods, asterisks, percent, minus, plus, Z, N/A)+ const numericPattern = /^[*+-]?[\d,.\s]+[%]?$|^[*]?-?[\d,.\s]+$|^[ZN]\/A$|^[Z-]$/;+ if (numericPattern.test(withoutArtifacts.trim())) {+ return withoutArtifacts.trim();+ }+ }++ return cleaned;+}+/*** Convert string to subscript unicode characters*/