Skip to content

A fast, helpful, and open-source document parser

better column alignment

Logan Markewich6982bca2c622
diff --git a/src/core/parser.ts b/src/core/parser.ts
index f36bb7a..a3c3f23 100644
--- a/src/core/parser.ts
+++ b/src/core/parser.ts
@@ -11,6 +11,7 @@ import { projectPagesToGrid } from "../processing/grid.js";
import { buildBoundingBoxes } from "../processing/bbox.js";
import { formatJSON } from "../output/json.js";
import { convertToPdf, cleanupConversionFiles } from "../conversion/convertToPdf.js";
+import { cleanOcrTableArtifacts } from "../processing/textUtils.js";
export class LiteParse {
private config: LiteParseConfig;
@@ -309,17 +310,22 @@ export class LiteParse {
// This prevents duplicating text that PDF already extracted correctly
return !overlapsExistingText(r.bbox);
})
- .map((r) => ({
- str: r.text,
- x: r.bbox[0] * scaleFactor,
- y: r.bbox[1] * scaleFactor,
- width: (r.bbox[2] - r.bbox[0]) * scaleFactor,
- height: (r.bbox[3] - r.bbox[1]) * scaleFactor,
- w: (r.bbox[2] - r.bbox[0]) * scaleFactor,
- h: (r.bbox[3] - r.bbox[1]) * scaleFactor,
- fontName: "OCR",
- fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,
- }));
+ .map((r) => {
+ // Clean OCR artifacts from table border misreads
+ const cleanedText = cleanOcrTableArtifacts(r.text);
+ return {
+ str: cleanedText,
+ x: r.bbox[0] * scaleFactor,
+ y: r.bbox[1] * scaleFactor,
+ width: (r.bbox[2] - r.bbox[0]) * scaleFactor,
+ height: (r.bbox[3] - r.bbox[1]) * scaleFactor,
+ w: (r.bbox[2] - r.bbox[0]) * scaleFactor,
+ h: (r.bbox[3] - r.bbox[1]) * scaleFactor,
+ fontName: "OCR",
+ fontSize: (r.bbox[3] - r.bbox[1]) * scaleFactor,
+ };
+ })
+ .filter((item) => item.str.length > 0); // Skip items that became empty after cleaning
// Add OCR text items directly to page textItems
page.textItems.push(...ocrTextItems);
diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.ts
index 5be1121..877ce6c 100644
--- a/src/engines/pdf/pdfjs.ts
+++ b/src/engines/pdf/pdfjs.ts
@@ -92,6 +92,202 @@ const BUGGY_FONT_MARKER_REGEX = /:->\|>_(\d+)_\d+_<\|<-:/g;
const BUGGY_FONT_MARKER_CHECK = ":->|>";
const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;
+/**
+ * Common tabular figures font encoding mappings.
+ * Many fonts with "Differences" arrays use similar patterns for tabular digits.
+ * These mappings are derived from common font encoding conventions.
+ *
+ * Note: The same PDF can use multiple fonts with DIFFERENT glyph-to-character mappings
+ * for the same glyph IDs. We try all mappings and pick the best match.
+ *
+ * Special glyphs:
+ * - 42: '*' (asterisk for significance markers)
+ * - 150: '-' (minus sign/dash)
+ */
+const TABULAR_FIGURES_MAPPINGS: Record<number, string>[] = [
+ // Mapping 1: Bold/header style (e.g., census PDF header row)
+ // Characters: 0123456789.,
+ {
+ 17: "4",
+ 18: "6",
+ 19: "8",
+ 20: "5",
+ 21: "9",
+ 22: "7",
+ 23: "1",
+ 24: " ",
+ 25: ",",
+ 26: "+",
+ 27: "-",
+ 28: "3",
+ 29: "0",
+ 30: "2",
+ 31: ".",
+ 42: "*",
+ 150: "-",
+ },
+ // Mapping 2: Book/body style (e.g., census PDF detail rows)
+ // Note: Same glyph IDs but different character assignments!
+ {
+ 17: "+",
+ 18: "7",
+ 19: "-",
+ 20: "9",
+ 21: "6",
+ 22: "3",
+ 23: "1",
+ 24: " ",
+ 25: "8",
+ 26: "5",
+ 27: "4",
+ 28: "0",
+ 29: "2",
+ 30: ".",
+ 31: ",",
+ 42: "*",
+ 150: "-",
+ },
+];
+
+/**
+ * Check if all glyphs in the range would produce printable ASCII via direct char code.
+ * Returns true if using String.fromCharCode on these glyphs would produce valid text.
+ */
+function canDecodeAsAscii(glyphs: number[]): boolean {
+ // Check if ALL glyphs would produce valid printable ASCII or common whitespace
+ for (const g of glyphs) {
+ // Printable ASCII range (space through tilde), plus tab/newline
+ if (!((g >= 32 && g <= 126) || g === 9 || g === 10 || g === 13)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+/**
+ * Score a decoded string for how "number-like" it appears.
+ * Higher scores indicate better number formatting.
+ */
+function scoreNumberFormat(decoded: string): number {
+ let score = 0;
+
+ // Count digits - primary indicator of a number
+ const digitCount = (decoded.match(/[0-9]/g) || []).length;
+ score += digitCount * 2;
+
+ // Bonus for matching common number patterns
+ // Pattern: digits with optional commas for thousands
+ if (/^\d{1,3}(,\d{3})*$/.test(decoded)) {
+ score += 5; // e.g., "248,800"
+ }
+ // Pattern: decimal number
+ if (/^\d+\.\d+$/.test(decoded)) {
+ score += 5; // e.g., "10.5"
+ }
+ // Pattern: negative number
+ if (/^[*-]?\d/.test(decoded)) {
+ score += 2; // e.g., "-1,132" or "*-0.4"
+ }
+ // Pattern: percentage or simple number
+ if (/^\d+$/.test(decoded)) {
+ score += 3; // e.g., "897"
+ }
+
+ // Penalize bad patterns
+ // Consecutive punctuation marks (not valid in numbers)
+ if (/[.,]{2,}/.test(decoded)) {
+ score -= 10;
+ }
+ // Punctuation at start (except minus/asterisk) or end
+ if (/^[.,+]|[.,+]$/.test(decoded)) {
+ score -= 5;
+ }
+ // Comma followed by anything other than 3 digits then boundary
+ if (/,(?!\d{3}(?:[,.]|$))/.test(decoded)) {
+ score -= 3;
+ }
+ // Period not followed by digits (except at end)
+ if (/\.(?![0-9])/.test(decoded) && !decoded.endsWith(".")) {
+ score -= 3;
+ }
+
+ return score;
+}
+
+/**
+ * Try to decode buggy font markers using known tabular figures mappings.
+ * Returns the decoded string if a mapping produces valid-looking text,
+ * otherwise returns null to fall back to charCode decoding.
+ *
+ * Strategy:
+ * 1. If glyphs are in ASCII range (32-126), let the fallback handle it
+ * 2. If glyphs are in tabular range (17-31, plus special chars), try mappings
+ * 3. Score each result for how "number-like" it appears
+ * 4. Return the best result if it looks like a valid number
+ */
+function tryDecodeTabularFigures(str: string): string | null {
+ if (!str.includes(BUGGY_FONT_MARKER_CHECK)) return null;
+
+ // Extract all glyph IDs from the markers
+ const glyphs: number[] = [];
+ let match;
+ const regex = /:->\|>_(\d+)_\d+_<\|<-:/g;
+ while ((match = regex.exec(str)) !== null) {
+ glyphs.push(parseInt(match[1]));
+ }
+
+ if (glyphs.length === 0) return null;
+
+ // If these glyphs would decode fine as ASCII, don't use tabular mapping
+ if (canDecodeAsAscii(glyphs)) {
+ return null;
+ }
+
+ // Check if glyphs are in the tabular figures range
+ // Tabular figures typically use glyphs 17-31, plus special chars like 42, 150
+ const tabularRange = glyphs.every(
+ (g) =>
+ (g >= 17 && g <= 31) || // Core tabular figures
+ g === 42 || // Asterisk
+ g === 150 || // Minus
+ g === 8 ||
+ g === 9 ||
+ g === 10 // Some special chars
+ );
+
+ if (!tabularRange) {
+ // Mixed content - not pure tabular figures
+ return null;
+ }
+
+ // Try each mapping and pick the best result
+ let bestResult: string | null = null;
+ let bestScore = -Infinity;
+
+ for (const mapping of TABULAR_FIGURES_MAPPINGS) {
+ const decoded = glyphs.map((g) => mapping[g] || "").join("");
+
+ // Skip if there are unmapped glyphs
+ const unmapped = glyphs.filter((g) => !mapping[g]).length;
+ if (unmapped > 0) continue;
+
+ // Score based on how "number-like" the result looks
+ const score = scoreNumberFormat(decoded);
+
+ if (score > bestScore) {
+ bestScore = score;
+ bestResult = decoded;
+ }
+ }
+
+ // Only return if we got a reasonable score (at least some digits, proper format)
+ if (bestResult && bestScore > 0) {
+ return bestResult;
+ }
+
+ return null;
+}
+
/**
* Strip C0/C1 control characters from text (except common whitespace).
* These can appear in PDF text due to font encoding issues but the
@@ -322,10 +518,17 @@ export class PdfJsEngine implements PdfEngine {
// Format: :->|>_<charCode>_<fontChar>_<|<-:
let decodedStr = item.str;
if (decodedStr.includes(BUGGY_FONT_MARKER_CHECK)) {
- BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state
- decodedStr = decodedStr.replace(BUGGY_FONT_MARKER_REGEX, (_: string, charCode: string) =>
- String.fromCharCode(parseInt(charCode))
- );
+ // Try tabular figures decoding first (common in government/census PDFs)
+ const tabularDecoded = tryDecodeTabularFigures(decodedStr);
+ if (tabularDecoded) {
+ decodedStr = tabularDecoded;
+ } else {
+ // Fall back to original approach: use glyph ID as character code
+ BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state
+ decodedStr = decodedStr.replace(BUGGY_FONT_MARKER_REGEX, (_: string, charCode: string) =>
+ String.fromCharCode(parseInt(charCode))
+ );
+ }
}
// Handle pipe-separated characters: " |a| |r| |X| " -> "arX"
diff --git a/src/processing/bbox.ts b/src/processing/bbox.ts
index 0bb375b..c25b199 100644
--- a/src/processing/bbox.ts
+++ b/src/processing/bbox.ts
@@ -7,6 +7,7 @@ import {
} from "../core/types.js";
import { PageData, Image } from "../engines/pdf/interface.js";
import { parseImageOcrBlocks, OcrBlock } from "./ocrUtils.js";
+import { cleanOcrTableArtifacts } from "./textUtils.js";
const OCR_CONFIDENCE_THRESHOLD = 0.1;
@@ -264,6 +265,14 @@ export function buildBbox(pageData: PageData, config: LiteParseConfig): Projecti
for (const block of ocrData) {
const confidenceRounded = Math.round(parseFloat(block.confidence.toString()) * 1000) / 1000;
+ // Clean OCR artifacts from table border misreads
+ const cleanedText = cleanOcrTableArtifacts(block.c);
+
+ // Skip if cleaning removed all content
+ if (cleanedText.length === 0) {
+ continue;
+ }
+
const line: ProjectionTextBox = {
fromOCR: true,
x: block.x,
@@ -271,8 +280,8 @@ export function buildBbox(pageData: PageData, config: LiteParseConfig): Projecti
w: block.w,
h: block.h,
r: image.originalOrientationAngle || 0,
- str: block.c,
- strLength: [...block.c].length,
+ str: cleanedText,
+ strLength: [...cleanedText].length,
pageBbox: {
x: block.x,
y: block.y,
@@ -288,7 +297,7 @@ export function buildBbox(pageData: PageData, config: LiteParseConfig): Projecti
w: block.rw,
h: block.rh,
confidence: confidenceRounded,
- text: block.c,
+ text: cleanedText,
});
}
diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.ts
index a52544e..de91806 100644
--- a/src/processing/gridProjection.ts
+++ b/src/processing/gridProjection.ts
@@ -192,6 +192,12 @@ function extractAnchorsPointsFromLines(lines: ProjectionTextBox[][], page: PageD
}
}
+ // Group nearby anchors FIRST to merge items at similar positions
+ // This ensures deltaMin operates on combined groups, not isolated items
+ group(anchorLeft);
+ group(anchorRight);
+ group(anchorCenter);
+
deltaMin(anchorRight, 0.1);
deltaMin(anchorLeft, 0.2);
deltaMin(anchorCenter, 0.05);
@@ -235,31 +241,40 @@ function extractAnchorsPointsFromLines(lines: ProjectionTextBox[][], page: PageD
intercept(anchorCenter);
function group(collection: Anchor) {
- for (const anchor in collection) {
- const anchorNum = parseFloat(anchor);
- // merge right
- if (
- collection[anchorNum + 1] &&
- collection[anchorNum + 1].length > collection[anchorNum].length
- ) {
- collection[anchorNum + 1].push(...collection[anchorNum]);
- delete collection[anchorNum];
- }
- // merge left
- else if (
- collection[anchorNum - 1] &&
- collection[anchorNum - 1].length > collection[anchorNum].length
- ) {
- collection[anchorNum - 1].push(...collection[anchorNum]);
- delete collection[anchorNum];
+ // Sort anchors to process them in order
+ const sortedAnchors = Object.keys(collection)
+ .map((a) => parseFloat(a))
+ .sort((a, b) => a - b);
+
+ // Merge nearby anchors within a tolerance
+ // Use 2 units as tolerance - this catches columns that are close but not exactly aligned
+ const MERGE_TOLERANCE = 2;
+
+ for (let i = 0; i < sortedAnchors.length; i++) {
+ const anchor = sortedAnchors[i];
+ if (!(anchor in collection)) continue; // Already merged
+
+ // Look for nearby anchors to merge into this one or merge this into
+ for (let j = i + 1; j < sortedAnchors.length; j++) {
+ const nextAnchor = sortedAnchors[j];
+ if (!(nextAnchor in collection)) continue;
+
+ // Stop if we're beyond the tolerance
+ if (nextAnchor - anchor > MERGE_TOLERANCE) break;
+
+ // Merge the smaller anchor into the larger one
+ if (collection[nextAnchor].length > collection[anchor].length) {
+ collection[nextAnchor].push(...collection[anchor]);
+ delete collection[anchor];
+ break; // This anchor is gone, move to next
+ } else {
+ collection[anchor].push(...collection[nextAnchor]);
+ delete collection[nextAnchor];
+ }
}
}
}
- group(anchorLeft);
- group(anchorRight);
- group(anchorCenter);
-
function anyAnchorSurvived(bbox: ProjectionTextBox) {
return (
roundAnchor(bbox.x) in anchorLeft ||
@@ -849,6 +864,17 @@ export function bboxToLine(
// merge 'words'
const mergeThreshold = 1;
+
+ // Pattern to detect standalone numeric values (financial table numbers)
+ // Matches: numbers with optional commas, decimal points, dollar signs, percentages, negatives
+ const numericPattern = /^[$]?-?[\d,]+\.?\d*%?$/;
+
+ function looksLikeTableNumber(str: string): boolean {
+ const trimmed = str.trim();
+ // Must be at least 2 chars to be a table number (avoid merging single digits)
+ return trimmed.length >= 2 && numericPattern.test(trimmed);
+ }
+
for (const line of lines) {
for (let i = 1; i < line.length; ++i) {
// merge box in word if:
@@ -858,7 +884,11 @@ export function bboxToLine(
const currentLine = line[i];
const previousLine = line[i - 1];
if (canMergeMarkup(previousLine, currentLine)) {
- if (currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {
+ // Don't merge adjacent numbers in tables - they're separate columns
+ const bothAreNumbers =
+ looksLikeTableNumber(previousLine.str) && looksLikeTableNumber(currentLine.str);
+
+ if (!bothAreNumbers && currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {
// if same word but less than .7 of prev line
if (currentLine.h != 0 && currentLine.h < previousLine.h * 0.7) {
// and not starting with space
@@ -878,11 +908,13 @@ export function bboxToLine(
line.splice(i, 1);
i--;
} else if (
+ !bothAreNumbers &&
currentLine.x - previousLine.x - previousLine.w <
- previousLine.w / previousLine.strLength
+ previousLine.w / previousLine.strLength
) {
// merge if space between this word and previous is less than average
// character width (using previous word font size)
+ // But don't merge adjacent numbers - they're likely table columns
// Now extend the width
previousLine.w = currentLine.x + currentLine.w - previousLine.x;
@@ -978,12 +1010,23 @@ function updateForwardAnchorRightBound(
anchorTarget: number
): void {
// Anything snapped to the right of rightBound should be aligned to anchorTarget line length at minimum
+ // Also update nearby positions (within tolerance) to handle slight position variations between rows
+ const POSITION_TOLERANCE = 2;
+
for (let i = snapMap.length - 1; i >= 0; --i) {
const anchor = snapMap[i];
if (rightBound <= anchor) {
if (!forwardAnchor[anchor] || anchorTarget > forwardAnchor[anchor]) {
forwardAnchor[anchor] = anchorTarget;
}
+ // Also update nearby positions within tolerance
+ for (let j = i - 1; j >= 0; --j) {
+ const nearbyAnchor = snapMap[j];
+ if (anchor - nearbyAnchor > POSITION_TOLERANCE) break;
+ if (!forwardAnchor[nearbyAnchor] || anchorTarget > forwardAnchor[nearbyAnchor]) {
+ forwardAnchor[nearbyAnchor] = anchorTarget;
+ }
+ }
} else {
return;
}
@@ -1303,7 +1346,8 @@ export function projectToGrid(
let lastSnapLeft = 0;
for (const key in forwardAnchors.left) {
- if (parseInt(key) <= bbox.x) {
+ // Use parseFloat to preserve decimal precision from anchor keys
+ if (parseFloat(key) <= bbox.x) {
lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);
}
}
diff --git a/src/processing/textUtils.test.ts b/src/processing/textUtils.test.ts
index 1b09fbb..3accca0 100644
--- a/src/processing/textUtils.test.ts
+++ b/src/processing/textUtils.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { strToPostScript, strToSubscriptString } from "./textUtils";
+import { strToPostScript, strToSubscriptString, cleanOcrTableArtifacts } from "./textUtils";
describe("test processing textUtils", () => {
it("test strToSubscriptString full conversion", () => {
@@ -25,4 +25,28 @@ describe("test processing textUtils", () => {
const finalStr = strToPostScript(originalStr);
expect(finalStr).toBe("hello123!");
});
+
+ it("test cleanOcrTableArtifacts removes trailing bracket from number", () => {
+ expect(cleanOcrTableArtifacts("44520]")).toBe("44520");
+ expect(cleanOcrTableArtifacts("9,674]")).toBe("9,674");
+ expect(cleanOcrTableArtifacts("0.3|")).toBe("0.3");
+ expect(cleanOcrTableArtifacts("63,790|")).toBe("63,790");
+ });
+
+ it("test cleanOcrTableArtifacts removes leading bracket from number", () => {
+ expect(cleanOcrTableArtifacts("|123")).toBe("123");
+ expect(cleanOcrTableArtifacts("[456")).toBe("456");
+ });
+
+ it("test cleanOcrTableArtifacts preserves brackets in non-numeric text", () => {
+ expect(cleanOcrTableArtifacts("(note)")).toBe("(note)");
+ expect(cleanOcrTableArtifacts("[ref]")).toBe("[ref]");
+ expect(cleanOcrTableArtifacts("hello]")).toBe("hello]");
+ });
+
+ it("test cleanOcrTableArtifacts handles special numeric patterns", () => {
+ expect(cleanOcrTableArtifacts("*-123")).toBe("*-123");
+ expect(cleanOcrTableArtifacts("Z")).toBe("Z");
+ expect(cleanOcrTableArtifacts("N/A")).toBe("N/A");
+ });
});
diff --git a/src/processing/textUtils.ts b/src/processing/textUtils.ts
index 99c6b58..fa14adb 100644
--- a/src/processing/textUtils.ts
+++ b/src/processing/textUtils.ts
@@ -1,3 +1,37 @@
+/**
+ * Clean common OCR artifacts from table documents.
+ * OCR often misreads vertical table border lines as bracket-like characters.
+ * This is especially common with numbers adjacent to table cell borders.
+ *
+ * Examples:
+ * - "44520]" → "44,520" (vertical line misread as ])
+ * - "|123" → "123" (vertical line misread as |)
+ * - "0.3|" → "0.3" (vertical line misread as |)
+ */
+export function cleanOcrTableArtifacts(text: string): string {
+ // Characters commonly misread from vertical table borders
+ // These typically appear at the start or end of cell content
+ const borderArtifacts = /^[|[\](){}]+|[|[\](){}]+$/g;
+
+ let cleaned = text.trim();
+
+ // Only clean if the core content looks like a number or short text
+ // This avoids incorrectly stripping brackets from actual content like "(note)"
+ const withoutArtifacts = cleaned.replace(borderArtifacts, "");
+
+ // If removing artifacts leaves us with something that looks like a number,
+ // statistical value, or percentage, use the cleaned version
+ if (withoutArtifacts.length > 0) {
+ // Check if core content is numeric-ish (numbers, commas, periods, asterisks, percent, minus, plus, Z, N/A)
+ const numericPattern = /^[*+-]?[\d,.\s]+[%]?$|^[*]?-?[\d,.\s]+$|^[ZN]\/A$|^[Z-]$/;
+ if (numericPattern.test(withoutArtifacts.trim())) {
+ return withoutArtifacts.trim();
+ }
+ }
+
+ return cleaned;
+}
+
/**
* Convert string to subscript unicode characters
*/