Skip to content

A fast, helpful, and open-source document parser

remove english-specific changes

Logan Markewicha16e5b6b1d3c
diff --git a/src/core/types.ts b/src/core/types.ts
index b1bd5d1..0ceaee6 100644
--- a/src/core/types.ts
+++ b/src/core/types.ts
@@ -81,7 +81,6 @@ export interface ProjectionTextBox {
forceUnsnapped?: boolean;
rotated?: boolean;
d?: number; // Delta for rotation handling
- isWordContinuation?: boolean; // True if this item continues from previous word
}
export interface Coordinates {
diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.ts
index d5dfac8..613ae6b 100644
--- a/src/processing/gridProjection.ts
+++ b/src/processing/gridProjection.ts
@@ -1201,80 +1201,10 @@ export function projectToGrid(
const xDelta = bbox.x - (prevBbox.x + prevBbox.w);
const prevCharWidth = prevBbox.w / prevBbox.strLength;
- // Check if items appear to be part of the same word
- // Heuristics:
- // 1. Very small gap (< 30% of char width) = definitely same word
- // 2. Prev ends with letter, current starts with lowercase, AND
- // current looks like a word fragment
- const prevStr = prevBbox.str.trimEnd();
- const currStr = bbox.str.trimStart();
- const prevEndsWithLetter = /[a-zA-Z]$/.test(prevStr);
- const currStartsWithLowercase = /^[a-z]/.test(currStr);
-
- // Common short standalone words that should NOT be merged
- const standaloneWords = new Set([
- "a",
- "i",
- "of",
- "to",
- "in",
- "on",
- "is",
- "it",
- "be",
- "we",
- "an",
- "or",
- "as",
- "at",
- "so",
- "no",
- "my",
- "he",
- "do",
- "by",
- "if",
- "up",
- "me",
- "go",
- "us",
- "am",
- ]);
-
- const verySmallGap = xDelta < prevCharWidth * 0.3;
- const currWord = currStr.toLowerCase().split(/\s/)[0];
-
- // Check if current item looks like a word fragment:
- // - Short word (<=3 chars) that's not a standalone word
- // - Starts with doubled consonant (rr, ll, mm, nn, pp, tt, etc.) - unlikely to begin a word
- // - Starts with certain letter patterns that indicate mid-word suffixes
- const isShortFragment =
- currWord.length <= 3 && !standaloneWords.has(currWord) && currStartsWithLowercase;
- // Only truly doubled consonants - these almost never start English words
- const startsWithDoubledConsonant = /^(bb|cc|dd|ff|gg|hh|jj|kk|ll|mm|nn|pp|qq|rr|ss|tt|vv|ww|xx|zz)/i.test(currWord);
- // Common suffixes that indicate mid-word fragments
- const startsWithSuffix =
- /^(tion|sion|ness|ment|able|ible|ful|less|ing|ous|ive|ers|est|ly|ed|en|al|ic|y)$/i.test(
- currWord
- );
- const isWordFragment =
- isShortFragment || startsWithDoubledConsonant || startsWithSuffix;
-
- // Looks like word continuation if:
- // - Gap is very small, OR
- // - Prev ends with letter AND current looks like a word fragment
- const looksLikeWordContinuation =
- verySmallGap || (prevEndsWithLetter && isWordFragment && currStartsWithLowercase);
-
- // add a space (unless it looks like word continuation)
- bbox.shouldSpace = looksLikeWordContinuation ? 0 : 1;
-
- // Mark word continuations to skip floating anchor alignment
- // and reset shouldSpace to 0
- if (looksLikeWordContinuation) {
- bbox.isWordContinuation = true;
- bbox.shouldSpace = 0;
- } else if (xDelta > prevCharWidth * 2) {
+ // add a space
+ bbox.shouldSpace = 1;
+
+ if (xDelta > prevCharWidth * 2) {
// Check if both items are in the same column based on gap size
// If gap is less than 10% of page width, treat as same column
// This works for any number of columns
@@ -1376,13 +1306,7 @@ export function projectToGrid(
targetX = lineMax;
}
- // For word continuations, place them right after the previous item
- // with no extra spacing beyond shouldSpace
- if (bbox.isWordContinuation) {
- targetX = rawLines[lineIndex].trimEnd().length + (bbox.shouldSpace ?? 0);
- }
-
- if (!bbox.forceUnsnapped && !bbox.isWordContinuation) {
+ if (!bbox.forceUnsnapped) {
const floatingAnchor = forwardAnchors.floating[Math.round(bbox.x)];
if (floatingAnchor && targetX < floatingAnchor) {
// Limit floating anchor adjustment to avoid excessive gaps in justified text