A fast, helpful, and open-source document parser
fixes
Logan Markewich39178c5d86ff
diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.tsindex dd0ec39..abd160e 100644--- a/src/engines/pdf/pdfjs.ts+++ b/src/engines/pdf/pdfjs.ts@@ -92,6 +92,25 @@ const BUGGY_FONT_MARKER_REGEX = /:->\|>_(\d+)_\d+_<\|<-:/g;const BUGGY_FONT_MARKER_CHECK = ":->|>";const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;+/**+ * Strip C0/C1 control characters from text (except common whitespace).+ * These can appear in PDF text due to font encoding issues but the+ * surrounding text may still be valid.+ */+function stripControlChars(str: string): string {+ let result = "";+ for (const char of str) {+ const code = char.charCodeAt(0);+ // Skip C0 controls (except tab, newline, carriage return) and C1 controls+ if ((code >= 0x00 && code <= 0x1f && code !== 0x09 && code !== 0x0a && code !== 0x0d) ||+ (code >= 0x80 && code <= 0x9f)) {+ continue;+ }+ result += char;+ }+ return result;+}+/*** Detect garbled text from fonts with corrupted ToUnicode mappings.*@@ -101,7 +120,7 @@ const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;* 1. Private Use Area (PUA) characters - fonts often map glyphs here* 2. Mix of unrelated scripts (Arabic + Latin Extended in English text)* 3. Rare/obscure Unicode blocks appearing in normal text- * 4. Control characters or specials+ * 4. Control characters (when text is predominantly control chars)** Returns true if the string appears to be garbled font output.*/@@ -113,12 +132,20 @@ function isGarbledFontOutput(str: string): boolean {let latinExtendedCount = 0;let basicLatinLetterCount = 0;let suspiciousCount = 0; // Other suspicious Unicode ranges+ let controlCharCount = 0; // C0/C1 control characters+ let normalCharCount = 0; // Normal printable charactersfor (const char of str) {const code = char.charCodeAt(0);+ // C0 control characters (0x00-0x1F) except common whitespace (tab, newline, carriage return)+ // C1 control characters (0x80-0x9F)+ if ((code >= 0x00 && code <= 0x1f && code !== 0x09 && code !== 0x0a && code !== 0x0d) ||+ (code >= 0x80 && code <= 0x9f)) {+ controlCharCount++;+ }// Private Use Area (U+E000-U+F8FF) - almost always garbled- if (code >= 0xe000 && code <= 0xf8ff) {+ else if (code >= 0xe000 && code <= 0xf8ff) {privateUseCount++;}// Arabic block (0x600-0x6FF) and Arabic Extended (0x750-0x77F, 0x8A0-0x8FF)@@ -137,6 +164,7 @@ function isGarbledFontOutput(str: string): boolean {// Basic Latin letters (a-z, A-Z)else if ((code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a)) {basicLatinLetterCount++;+ normalCharCount++;}// Suspicious ranges that rarely appear in normal text:// - Syriac (0x700-0x74F)@@ -156,10 +184,20 @@ function isGarbledFontOutput(str: string): boolean {) {suspiciousCount++;}+ // Normal printable characters (digits, punctuation, common symbols, space)+ else if ((code >= 0x20 && code <= 0x7e) || code === 0x09 || code === 0x0a || code === 0x0d) {+ normalCharCount++;+ }}const totalChars = str.length;+ // Text is predominantly control characters - definitely garbled+ // This catches cases like more_hard_2.pdf where text is entirely control chars+ if (controlCharCount > 0 && controlCharCount > normalCharCount) {+ return true;+ }+// Private Use Area characters are almost always garbled fontsif (privateUseCount >= 2) {return true;@@ -303,6 +341,10 @@ export class PdfJsEngine implements PdfEngine {continue;}+ // Strip any remaining control characters from valid text+ // (e.g., form feed chars that sneak into ligatures like "fi")+ decodedStr = stripControlChars(decodedStr);+textItems.push({str: decodedStr,x: left,diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.tsindex 4f6f2f1..828b33a 100644--- a/src/processing/gridProjection.ts+++ b/src/processing/gridProjection.ts@@ -511,43 +511,52 @@ function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: nreturn;}- textBbox.sort((a, b) => {- return a.x - b.x;- });-- // Horizontal Axis- let bboxGroup: ProjectionTextBox[][] = [];- let currentGroup: ProjectionTextBox[] = [];- let previousBbox: ProjectionTextBox | null = null;+ // Group ALL items by rotation value (not by consecutive items)+ // This ensures rotated text blocks stay together even when their X coordinates+ // overlap with non-rotated content (e.g., rotated table + footer at same X positions)+ const groupsByRotation: { [key: number]: ProjectionTextBox[] } = {};for (const bbox of textBbox) {- if (previousBbox) {- if (bbox.r == previousBbox.r) {- currentGroup.push(bbox);- } else {- // sort currentGroup by bbox.y- currentGroup.sort((a, b) => a.y - b.y);- bboxGroup.push(currentGroup);- currentGroup = [bbox];- }- } else {- currentGroup.push(bbox);+ const r = bbox.r || 0;+ if (!groupsByRotation[r]) {+ groupsByRotation[r] = [];}- previousBbox = bbox;+ groupsByRotation[r].push(bbox);}- if (currentGroup.length) {- currentGroup.sort((a, b) => a.y - b.y);- bboxGroup.push(currentGroup);++ // Build bboxGroup array from rotation groups, sorted by X position of group+ let bboxGroup: ProjectionTextBox[][] = [];+ for (const rotation in groupsByRotation) {+ const group = groupsByRotation[rotation];+ // Sort each group by Y for proper reading order+ group.sort((a, b) => a.y - b.y);+ bboxGroup.push(group);}+ // Sort groups by their minimum X position to maintain left-to-right order+ bboxGroup.sort((a, b) => {+ const minXA = Math.min(...a.map((item) => item.x));+ const minXB = Math.min(...b.map((item) => item.x));+ return minXA - minXB;+ });+// NOTE/ WARNING: height and width of bbox are NOT rotated beforehand!for (const [index, group] of bboxGroup.entries()) {if (group[0].r == 90 || group[0].r == 270) {- // check in all bbox if they are not group[0].r and have overlap in x axis+ // Check if there are non-rotated items that actually overlap visually (both X and Y)+ // with the rotated group. X-only overlap is not sufficient because items could+ // be in completely different parts of the page (e.g., rotated table + footer).let globalOverlap = false;for (const bbox of textBbox) {if (bbox.r != group[0].r) {const overlap = group.find(- (b) => b.x >= bbox.x && b.x <= bbox.x + bbox.w && bbox.r != b.r+ (b) =>+ // Check X overlap+ b.x >= bbox.x &&+ b.x <= bbox.x + bbox.w &&+ // Also check Y overlap - items must actually be near each other vertically+ b.y < bbox.y + bbox.h &&+ b.y + b.h > bbox.y &&+ bbox.r != b.r);if (overlap) {globalOverlap = true;@@ -565,7 +574,7 @@ function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: nbbox.r = 0;bbox.rotated = true;}- } else if (!globalOverlap) {+ } else {// insert the bbox group in the Y axis after previous group and before next group.// move Next group by current group height (width as not rotated yet).const groupMaxX = Math.max(...group.map((v) => v.x + v.w));@@ -577,25 +586,52 @@ function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: n// pageHeight is radical but garantie no issue of allignementdeltaY = previousGroupMaxY + pageHeight;}- // clockwize rotation+ // clockwise rotation (90 degrees)+ // - Text reads top-to-bottom in page space+ // - Y position in page space -> X position after de-rotation+ // - X position in page space -> Y position after de-rotation (row)if (group[0].r == 90) {for (const bbox of group) {- // w not h here as bbox w/h are not flipped- const x = Math.round(bbox.ry ?? bbox.y);- const y = (bbox.rx ?? bbox.x) + deltaY;- bbox.x = Math.round(x);- bbox.y = y;+ const newX = Math.round(bbox.y);+ const newY = bbox.x + deltaY;+ // Swap width and height since text orientation changes+ const newW = bbox.h;+ const newH = bbox.w;+ bbox.x = newX;+ bbox.y = newY;+ bbox.w = newW;+ bbox.h = newH;bbox.r = 0;bbox.rotated = true;}}- // counter clockwize rotation+ // counter clockwize rotation (text reads bottom-to-top)+ // For 270-degree rotation, text at higher Y positions should be+ // at lower X positions after de-rotation (left-to-right reading order)if (group[0].r == 270) {+ // For 270-degree counter-clockwise rotation:+ // - Text reads bottom-to-top in page space+ // - Y position in page space -> X position after de-rotation (inverted)+ // - X position in page space -> Y position after de-rotation (row)+ // - w and h need to be swapped since they represent visual dimensions+ // For 270-degree rotation: h is the extent along reading direction (string width)+ const maxY = Math.max(...group.map((b) => b.y + b.h));+for (const bbox of group) {- const x = Math.round(bbox.ry ?? bbox.y);- const y = (bbox.rx ?? bbox.x) + deltaY;- bbox.x = Math.round(x);- bbox.y = y;+ // Transform coordinates:+ // - new_x = distance from right edge of rotated block (inverted Y)+ // Use h (string width in original coords) for the extent+ // - new_y = row position (from original X)+ const newX = Math.round(maxY - bbox.y - bbox.h);+ // Use exact X for Y (will be grouped by bboxToLine's Y_SORT_TOLERANCE)+ const newY = bbox.x + deltaY;+ // Swap width and height since text orientation changes+ const newW = bbox.h;+ const newH = bbox.w;+ bbox.x = newX;+ bbox.y = newY;+ bbox.w = newW;+ bbox.h = newH;bbox.r = 0;bbox.rotated = true;}@@ -624,32 +660,12 @@ function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: nreturn a.y - b.y;});- // Vertical Axis- bboxGroup = [];- currentGroup = [];- previousBbox = null as ProjectionTextBox | null;- for (const bbox of textBbox) {- if (previousBbox) {- if (bbox.r == previousBbox.r) {- currentGroup.push(bbox);- } else {- // sort currentGroup by bbox.y- currentGroup.sort((a, b) => a.x - b.x);- bboxGroup.push(currentGroup);- currentGroup = [bbox];- }- } else {- currentGroup.push(bbox);- }- previousBbox = bbox;- }- if (currentGroup.length) {- currentGroup.sort((a, b) => a.x - b.x);- bboxGroup.push(currentGroup);- }-+ // Handle 180-degree rotated text (upside down)+ // Since we already grouped by rotation, we can iterate the existing groupsfor (const group of bboxGroup) {if (group[0].r == 180) {+ // Sort by X for proper reading order+ group.sort((a, b) => a.x - b.x);// Switch upside downfor (const bbox of group) {bbox.x = Math.round(bbox.ry ?? bbox.y);@@ -668,8 +684,11 @@ export function bboxToLine(pageWidth?: number): ProjectionTextBox[][] {// Y-tolerance for sorting: items within this threshold are considered same line- // This handles floating point precision issues between columns (e.g., 334.7400 vs 334.7399)- const Y_SORT_TOLERANCE = 2.0;+ // This handles:+ // 1. Floating point precision issues between columns (e.g., 334.7400 vs 334.7399)+ // 2. Subscripts/superscripts which are typically offset by 3-5 units from their base characters+ // Using a fraction of medianHeight to scale with document font size.+ const Y_SORT_TOLERANCE = Math.max(medianHeight * 0.5, 5.0);// Note: We keep whitespace items as they may be needed for proper word separation.// The spacing calculation handles gaps between items.@@ -778,9 +797,12 @@ export function bboxToLine(const overlapLenght =Math.min(currentLineItemBbox.x + currentLineItemBbox.w, bbox.x + bbox.w) -Math.max(currentLineItemBbox.x, bbox.x);- // Use a minimum threshold of 3px to tolerate small overlaps common in PDFs- // due to character spacing/kerning, while still detecting true collisions- if (overlapLenght > Math.max(medianWidth / 3, 3)) {+ // Use a minimum threshold to tolerate small overlaps common in PDFs due to:+ // - character spacing/kerning+ // - floating-point precision issues+ // - adjacent items with slightly overlapping bounding boxes+ // We want to detect true collisions (same text rendered twice) not adjacent text+ if (overlapLenght > Math.max(medianWidth / 3, 5)) {lineCollide = true;break;}@@ -791,10 +813,15 @@ export function bboxToLine(const bboxIsMargin = bbox.isMarginLineNumber === true;const marginMismatch = currentLineHasMargin !== bboxIsMargin;+ // For rotated text, use Y-tolerance based merging since heights may be inconsistent+ const yTolerance = bbox.rotated ? Math.max(medianHeight * 2, 20) : 0;+ const yWithinTolerance = bbox.rotated && Math.abs(bbox.y - lineMinY) < yTolerance;+if (!lineCollide &&!marginMismatch &&- ((bbox.y + bbox.h * 0.5 >= lineMinY && bbox.y + bbox.h * 0.5 <= lineMaxY) ||+ (yWithinTolerance ||+ (bbox.y + bbox.h * 0.5 >= lineMinY && bbox.y + bbox.h * 0.5 <= lineMaxY) ||(bbox.y >= lineMinY && bbox.y <= lineMaxY))) {currentLine.push(bbox);