A fast, helpful, and open-source document parser
format
Logan Markewichdba4b4cf5c47
diff --git a/README.md b/README.mdindex a03bebe..99a9a03 100644--- a/README.md+++ b/README.md@@ -87,9 +87,14 @@ liteparse parse document.pdf --target-pages "1-5,10,15-20"# Parse without OCRliteparse parse document.pdf --no-ocr+```++### Batch Parsing++You can also parse an entire directory of documents:-# Use custom config file-liteparse parse document.pdf --config config.json+```bash+liteparse batch-parse ./input-directory ./output-directory```### Generate Screenshots@@ -133,7 +138,7 @@ console.log(result.text);#### Parse Command```-liteparse parse <file> [options]+$ liteparse parse <file> [options]Options:-o, --output <file> Output file path@@ -152,10 +157,33 @@ Options:--config <file> Config file (JSON)```+#### Batch Parse Command++```+$ liteparse batch-parse --help++Parse multiple documents in batch mode (reuses PDF engine for efficiency)++Options:+ --format <format> Output format: json|text (default: "text")+ --ocr-server-url <url> HTTP OCR server URL (uses Tesseract if not provided)+ --no-ocr Disable OCR+ --ocr-language <lang> OCR language(s) (default: "en")+ --max-pages <n> Max pages to parse per file (default: "1000")+ --dpi <dpi> DPI for rendering (default: "150")+ --no-tables Disable table detection+ --no-precise-bbox Disable precise bounding boxes+ --recursive Recursively search input directory+ --extension <ext> Only process files with this extension (e.g., ".pdf")+ --config <file> Config file (JSON)+ -q, --quiet Suppress progress output+ -h, --help display help for command+```+#### Screenshot Command```-liteparse screenshot <file> [options]+$ liteparse screenshot <file> [options]Options:-o, --output-dir <dir> Output directory for screenshots (default: "./screenshots")diff --git a/cli/parse.ts b/cli/parse.tsindex 627e425..3787212 100644--- a/cli/parse.ts+++ b/cli/parse.ts@@ -1,35 +1,35 @@-import { Command } from 'commander';-import fs from 'fs/promises';-import { existsSync, readdirSync, statSync } from 'fs';-import path from 'path';-import { LiteParse } from '../src/core/parser.js';-import { LiteParseConfig, OutputFormat } from '../src/core/types.js';-import { performance } from 'perf_hooks';+import { Command } from "commander";+import fs from "fs/promises";+import { existsSync, readdirSync, statSync } from "fs";+import path from "path";+import { LiteParse } from "../src/core/parser.js";+import { LiteParseConfig, OutputFormat } from "../src/core/types.js";+import { performance } from "perf_hooks";const program = new Command();program- .name('liteparse')- .description('OSS document parsing tool (supports PDF, DOCX, XLSX, images, and more)')- .version('1.0.0');+ .name("liteparse")+ .description("OSS document parsing tool (supports PDF, DOCX, XLSX, images, and more)")+ .version("1.0.0");program- .command('parse <file>')- .description('Parse a document file (PDF, DOCX, XLSX, PPTX, images, etc.)')- .option('-o, --output <file>', 'Output file path')- .option('--format <format>', 'Output format: json|text', 'text')- .option('--ocr-server-url <url>', 'HTTP OCR server URL (uses Tesseract if not provided)')- .option('--no-ocr', 'Disable OCR')- .option('--ocr-language <lang>', 'OCR language(s)', 'en')- .option('--max-pages <n>', 'Max pages to parse', '1000')- .option('--target-pages <pages>', 'Target pages (e.g., "1-5,10,15-20")')- .option('--dpi <dpi>', 'DPI for rendering', '150')- .option('--no-tables', 'Disable table detection')- .option('--no-precise-bbox', 'Disable precise bounding boxes')- .option('--skip-diagonal-text', 'Skip diagonal text')- .option('--preserve-small-text', 'Preserve very small text')- .option('--config <file>', 'Config file (JSON)')- .option('-q, --quiet', 'Suppress progress output')+ .command("parse <file>")+ .description("Parse a document file (PDF, DOCX, XLSX, PPTX, images, etc.)")+ .option("-o, --output <file>", "Output file path")+ .option("--format <format>", "Output format: json|text", "text")+ .option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)")+ .option("--no-ocr", "Disable OCR")+ .option("--ocr-language <lang>", "OCR language(s)", "en")+ .option("--max-pages <n>", "Max pages to parse", "1000")+ .option("--target-pages <pages>", 'Target pages (e.g., "1-5,10,15-20")')+ .option("--dpi <dpi>", "DPI for rendering", "150")+ .option("--no-tables", "Disable table detection")+ .option("--no-precise-bbox", "Disable precise bounding boxes")+ .option("--skip-diagonal-text", "Skip diagonal text")+ .option("--preserve-small-text", "Preserve very small text")+ .option("--config <file>", "Config file (JSON)")+ .option("-q, --quiet", "Suppress progress output").action(async (file: string, options: any) => {try {const quiet = options.quiet || false;@@ -48,7 +48,7 @@ programconsole.error(`Error: Config file not found: ${options.config}`);process.exit(1);}- const configData = await fs.readFile(options.config, 'utf-8');+ const configData = await fs.readFile(options.config, "utf-8");config = JSON.parse(configData);}@@ -77,10 +77,10 @@ program// Format output based on formatlet output: string;switch (config.outputFormat) {- case 'json':+ case "json":output = JSON.stringify(result.json, null, 2);break;- case 'text':+ case "text":default:output = result.text;break;@@ -106,14 +106,14 @@ program});program- .command('screenshot <file>')- .description('Generate screenshots of PDF pages')- .option('-o, --output-dir <dir>', 'Output directory for screenshots', './screenshots')- .option('--pages <pages>', 'Page numbers to screenshot (e.g., "1,3,5" or "1-5")')- .option('--dpi <dpi>', 'DPI for rendering', '150')- .option('--format <format>', 'Image format: png|jpg', 'png')- .option('--config <file>', 'Config file (JSON)')- .option('-q, --quiet', 'Suppress progress output')+ .command("screenshot <file>")+ .description("Generate screenshots of PDF pages")+ .option("-o, --output-dir <dir>", "Output directory for screenshots", "./screenshots")+ .option("--pages <pages>", 'Page numbers to screenshot (e.g., "1,3,5" or "1-5")')+ .option("--dpi <dpi>", "DPI for rendering", "150")+ .option("--format <format>", "Image format: png|jpg", "png")+ .option("--config <file>", "Config file (JSON)")+ .option("-q, --quiet", "Suppress progress output").action(async (file: string, options: any) => {try {const quiet = options.quiet || false;@@ -132,7 +132,7 @@ programconsole.error(`Error: Config file not found: ${options.config}`);process.exit(1);}- const configData = await fs.readFile(options.config, 'utf-8');+ const configData = await fs.readFile(options.config, "utf-8");config = JSON.parse(configData);}@@ -183,29 +183,61 @@ program// Supported file extensions for batch parsingconst SUPPORTED_EXTENSIONS = new Set([- '.pdf',- '.doc', '.docx', '.docm', '.dot', '.dotm', '.dotx', '.odt', '.ott',- '.ppt', '.pptx', '.pptm', '.pot', '.potm', '.potx', '.odp', '.otp',- '.xls', '.xlsx', '.xlsm', '.xlsb', '.ods', '.ots', '.csv', '.tsv',- '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg',- '.rtf', '.pages', '.key', '.numbers',+ ".pdf",+ ".doc",+ ".docx",+ ".docm",+ ".dot",+ ".dotm",+ ".dotx",+ ".odt",+ ".ott",+ ".ppt",+ ".pptx",+ ".pptm",+ ".pot",+ ".potm",+ ".potx",+ ".odp",+ ".otp",+ ".xls",+ ".xlsx",+ ".xlsm",+ ".xlsb",+ ".ods",+ ".ots",+ ".csv",+ ".tsv",+ ".jpg",+ ".jpeg",+ ".png",+ ".gif",+ ".bmp",+ ".tiff",+ ".tif",+ ".webp",+ ".svg",+ ".rtf",+ ".pages",+ ".key",+ ".numbers",]);program- .command('batch-parse <input-dir> <output-dir>')- .description('Parse multiple documents in batch mode')- .option('--format <format>', 'Output format: json|text', 'text')- .option('--ocr-server-url <url>', 'HTTP OCR server URL (uses Tesseract if not provided)')- .option('--no-ocr', 'Disable OCR')- .option('--ocr-language <lang>', 'OCR language(s)', 'en')- .option('--max-pages <n>', 'Max pages to parse per file', '1000')- .option('--dpi <dpi>', 'DPI for rendering', '150')- .option('--no-tables', 'Disable table detection')- .option('--no-precise-bbox', 'Disable precise bounding boxes')- .option('--recursive', 'Recursively search input directory')- .option('--extension <ext>', 'Only process files with this extension (e.g., ".pdf")')- .option('--config <file>', 'Config file (JSON)')- .option('-q, --quiet', 'Suppress progress output')+ .command("batch-parse <input-dir> <output-dir>")+ .description("Parse multiple documents in batch mode")+ .option("--format <format>", "Output format: json|text", "text")+ .option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)")+ .option("--no-ocr", "Disable OCR")+ .option("--ocr-language <lang>", "OCR language(s)", "en")+ .option("--max-pages <n>", "Max pages to parse per file", "1000")+ .option("--dpi <dpi>", "DPI for rendering", "150")+ .option("--no-tables", "Disable table detection")+ .option("--no-precise-bbox", "Disable precise bounding boxes")+ .option("--recursive", "Recursively search input directory")+ .option("--extension <ext>", 'Only process files with this extension (e.g., ".pdf")')+ .option("--config <file>", "Config file (JSON)")+ .option("-q, --quiet", "Suppress progress output").action(async (inputDir: string, outputDir: string, options: any) => {try {const quiet = options.quiet || false;@@ -232,7 +264,7 @@ programconst files = findFiles(inputDir, options.recursive, options.extension);if (files.length === 0) {- console.error('No supported files found in input directory');+ console.error("No supported files found in input directory");process.exit(1);}@@ -247,7 +279,7 @@ programconsole.error(`Error: Config file not found: ${options.config}`);process.exit(1);}- const configData = await fs.readFile(options.config, 'utf-8');+ const configData = await fs.readFile(options.config, "utf-8");config = JSON.parse(configData);}@@ -270,15 +302,12 @@ program// Process fileslet successCount = 0;let errorCount = 0;- const outputExt = options.format === 'json' ? '.json' : '.txt';+ const outputExt = options.format === "json" ? ".json" : ".txt";for (let i = 0; i < files.length; i++) {const file = files[i];const relativePath = path.relative(inputDir, file);- const outputPath = path.join(- outputDir,- relativePath.replace(/\.[^.]+$/, outputExt)- );+ const outputPath = path.join(outputDir, relativePath.replace(/\.[^.]+$/, outputExt));// Create output subdirectory if neededconst outputSubdir = path.dirname(outputPath);@@ -292,7 +321,7 @@ program// Format outputlet output: string;- if (options.format === 'json') {+ if (options.format === "json") {output = JSON.stringify(result.json, null, 2);} else {output = result.text;@@ -303,7 +332,9 @@ programif (!quiet) {const fileTime = (performance.now() - fileStart).toFixed(0);- console.error(`[${i + 1}/${files.length}] ✓ ${relativePath} (${result.pages.length} pages, ${fileTime}ms)`);+ console.error(+ `[${i + 1}/${files.length}] ✓ ${relativePath} (${result.pages.length} pages, ${fileTime}ms)`+ );}} catch (error: any) {errorCount++;@@ -314,10 +345,11 @@ program}const totalTime = ((performance.now() - startTime) / 1000).toFixed(2);- const avgTime = files.length > 0 ? ((performance.now() - startTime) / files.length).toFixed(0) : 0;+ const avgTime =+ files.length > 0 ? ((performance.now() - startTime) / files.length).toFixed(0) : 0;if (!quiet) {- console.error('');+ console.error("");console.error(`Batch complete: ${successCount} succeeded, ${errorCount} failed`);console.error(`Total time: ${totalTime}s (avg ${avgTime}ms/file)`);console.error(`Output: ${outputDir}`);@@ -377,12 +409,12 @@ function findFiles(dir: string, recursive: boolean, filterExt?: string): string[*/function parsePageNumbers(pagesStr: string): number[] {const pages: number[] = [];- const parts = pagesStr.split(',');+ const parts = pagesStr.split(",");for (const part of parts) {const trimmed = part.trim();- if (trimmed.includes('-')) {- const [start, end] = trimmed.split('-').map((n) => parseInt(n.trim()));+ if (trimmed.includes("-")) {+ const [start, end] = trimmed.split("-").map((n) => parseInt(n.trim()));for (let i = start; i <= end; i++) {pages.push(i);}diff --git a/package.json b/package.jsonindex a47d838..80ada6b 100644--- a/package.json+++ b/package.json@@ -24,7 +24,13 @@"scripts": {"build": "tsc && npm run copy-vendor","copy-vendor": "mkdir -p dist/src/vendor && cp -r src/vendor/pdfjs dist/src/vendor/",- "dev": "tsc --watch"+ "dev": "tsc --watch",+ "test": "vitest run",+ "test:watch": "vitest",+ "lint": "eslint src cli",+ "lint:fix": "eslint src cli --fix",+ "format": "prettier --write \"src/**/*.ts\" \"cli/**/*.ts\"",+ "format:check": "prettier --check \"src/**/*.ts\" \"cli/**/*.ts\""},"dependencies": {"@hyzyla/pdfium": "^2.1.9",@@ -38,8 +44,15 @@"zod": "^3.23.0"},"devDependencies": {+ "@eslint/js": "^10.0.1","@types/node": "^22.0.0",- "typescript": "~5.9.2"+ "eslint": "^10.0.0",+ "eslint-config-prettier": "^10.1.8",+ "eslint-plugin-prettier": "^5.5.5",+ "prettier": "^3.8.1",+ "typescript": "~5.9.2",+ "typescript-eslint": "^8.56.0",+ "vitest": "^4.0.18"},"engines": {"node": ">=18.0.0"@@ -62,4 +75,4 @@"homepage": "https://github.com/run-llama/liteparse#readme","author": "","license": "Apache-2.0"-}\ No newline at end of file+}diff --git a/src/conversion/convertToPdf.ts b/src/conversion/convertToPdf.tsindex 8d5ac4d..ea0eea3 100644--- a/src/conversion/convertToPdf.ts+++ b/src/conversion/convertToPdf.ts@@ -1,7 +1,7 @@-import { promises as fs } from 'fs';-import { spawn } from 'child_process';-import path from 'path';-import os from 'os';+import { promises as fs } from "fs";+import { spawn } from "child_process";+import path from "path";+import os from "os";export interface ConversionResult {pdfPath: string;@@ -14,57 +14,57 @@ export interface ConversionError {}export interface ConversionPassthrough {- content: string+ content: string;}// File extension categoriesexport const officeExtensions = [- '.doc',- '.docx',- '.docm',- '.dot',- '.dotm',- '.dotx',- '.odt',- '.ott',- '.ppt',- '.pptx',- '.pptm',- '.pot',- '.potm',- '.potx',- '.odp',- '.otp',- '.rtf',- '.pages',- '.key',+ ".doc",+ ".docx",+ ".docm",+ ".dot",+ ".dotm",+ ".dotx",+ ".odt",+ ".ott",+ ".ppt",+ ".pptx",+ ".pptm",+ ".pot",+ ".potm",+ ".potx",+ ".odp",+ ".otp",+ ".rtf",+ ".pages",+ ".key",];export const spreadsheetExtensions = [- '.xls',- '.xlsx',- '.xlsm',- '.xlsb',- '.ods',- '.ots',- '.csv',- '.tsv',- '.numbers',+ ".xls",+ ".xlsx",+ ".xlsm",+ ".xlsb",+ ".ods",+ ".ots",+ ".csv",+ ".tsv",+ ".numbers",];export const imageExtensions = [- '.jpg',- '.jpeg',- '.png',- '.gif',- '.bmp',- '.tiff',- '.tif',- '.webp',- '.svg',+ ".jpg",+ ".jpeg",+ ".png",+ ".gif",+ ".bmp",+ ".tiff",+ ".tif",+ ".webp",+ ".svg",];-export const htmlExtensions = ['.htm', '.html', '.xhtml'];+export const htmlExtensions = [".htm", ".html", ".xhtml"];/*** Guess file extension from file content@@ -77,60 +77,51 @@ export async function guessFileExtension(filePath: string): Promise<string> {// Read first few bytes to detect file typeconst buffer = Buffer.alloc(16);- const fd = await fs.open(filePath, 'r');+ const fd = await fs.open(filePath, "r");await fd.read(buffer, 0, 16, 0);await fd.close();// PDF: %PDF- if (buffer.toString('utf-8', 0, 4) === '%PDF') {- return '.pdf';+ if (buffer.toString("utf-8", 0, 4) === "%PDF") {+ return ".pdf";}// PNG: 89 50 4E 47- if (- buffer[0] === 0x89 &&- buffer[1] === 0x50 &&- buffer[2] === 0x4e &&- buffer[3] === 0x47- ) {- return '.png';+ if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {+ return ".png";}// JPEG: FF D8 FFif (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {- return '.jpg';+ return ".jpg";}// ZIP-based formats (docx, xlsx, etc): PKif (buffer[0] === 0x50 && buffer[1] === 0x4b) {// Could be docx, xlsx, pptx, odt, etc.- return '.docx'; // Default to docx for now+ return ".docx"; // Default to docx for now}- return ext || '.pdf';+ return ext || ".pdf";}/*** Execute command with timeout*/-async function executeCommand(- command: string,- args: string[],- timeoutMs = 60000-): Promise<string> {+async function executeCommand(command: string, args: string[], timeoutMs = 60000): Promise<string> {return new Promise((resolve, reject) => {const proc = spawn(command, args, {- stdio: ['ignore', 'pipe', 'pipe'],+ stdio: ["ignore", "pipe", "pipe"],});- let stdout = '';- let stderr = '';+ let stdout = "";+ let stderr = "";- proc.stdout?.on('data', (data) => {+ proc.stdout?.on("data", (data) => {stdout += data.toString();});- proc.stderr?.on('data', (data) => {+ proc.stderr?.on("data", (data) => {stderr += data.toString();});@@ -139,7 +130,7 @@ async function executeCommand(reject(new Error(`Command timeout after ${timeoutMs}ms`));}, timeoutMs);- proc.on('close', (code) => {+ proc.on("close", (code) => {clearTimeout(timeout);if (code === 0) {resolve(stdout);@@ -148,7 +139,7 @@ async function executeCommand(}});- proc.on('error', (err) => {+ proc.on("error", (err) => {clearTimeout(timeout);reject(err);});@@ -160,7 +151,7 @@ async function executeCommand(*/async function isCommandAvailable(command: string): Promise<boolean> {try {- await executeCommand('which', [command], 5000);+ await executeCommand("which", [command], 5000);return true;} catch {return false;@@ -170,28 +161,17 @@ async function isCommandAvailable(command: string): Promise<boolean> {/*** Convert office documents using LibreOffice*/-async function convertOfficeDocument(- filePath: string,- outputDir: string-): Promise<string> {- const hasLibreOffice = await isCommandAvailable('libreoffice');+async function convertOfficeDocument(filePath: string, outputDir: string): Promise<string> {+ const hasLibreOffice = await isCommandAvailable("libreoffice");if (!hasLibreOffice) {throw new Error(- 'LibreOffice is not installed. Please install LibreOffice to convert office documents. On macOS: brew install --cask libreoffice, On Ubuntu: apt-get install libreoffice'+ "LibreOffice is not installed. Please install LibreOffice to convert office documents. On macOS: brew install --cask libreoffice, On Ubuntu: apt-get install libreoffice");}await executeCommand(- 'libreoffice',- [- '--headless',- '--invisible',- '--convert-to',- 'pdf',- '--outdir',- outputDir,- filePath,- ],+ "libreoffice",+ ["--headless", "--invisible", "--convert-to", "pdf", "--outdir", outputDir, filePath],120000 // 2 minutes timeout);@@ -204,21 +184,18 @@ async function convertOfficeDocument(await fs.access(pdfPath);return pdfPath;} catch {- throw new Error('LibreOffice conversion succeeded but output PDF not found');+ throw new Error("LibreOffice conversion succeeded but output PDF not found");}}/*** Convert images to PDF using ImageMagick*/-async function convertImageToPdf(- filePath: string,- outputDir: string-): Promise<string> {- const hasConvert = await isCommandAvailable('convert');+async function convertImageToPdf(filePath: string, outputDir: string): Promise<string> {+ const hasConvert = await isCommandAvailable("convert");if (!hasConvert) {throw new Error(- 'ImageMagick is not installed. Please install ImageMagick to convert images. On macOS: brew install imagemagick, On Ubuntu: apt-get install imagemagick'+ "ImageMagick is not installed. Please install ImageMagick to convert images. On macOS: brew install imagemagick, On Ubuntu: apt-get install imagemagick");}@@ -226,15 +203,8 @@ async function convertImageToPdf(const pdfPath = path.join(outputDir, `${baseName}.pdf`);await executeCommand(- 'convert',- [- filePath,- '-density',- '150',- '-units',- 'PixelsPerInch',- pdfPath,- ],+ "convert",+ [filePath, "-density", "150", "-units", "PixelsPerInch", pdfPath],60000);@@ -254,7 +224,7 @@ export async function convertToPdf(} catch {return {message: `File not found: ${filePath}`,- code: 'FILE_NOT_FOUND',+ code: "FILE_NOT_FOUND",};}@@ -262,7 +232,7 @@ export async function convertToPdf(const extension = await guessFileExtension(filePath);// If already PDF, return as-is- if (extension === '.pdf') {+ if (extension === ".pdf") {return {pdfPath: filePath,originalExtension: extension,@@ -270,7 +240,7 @@ export async function convertToPdf(}// Create temp directory for output- const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'liteparse-'));+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "liteparse-"));// Convert based on file typelet pdfPath: string;@@ -284,11 +254,10 @@ export async function convertToPdf(} else if (imageExtensions.includes(extension)) {console.error(`Converting image: ${path.basename(filePath)}`);pdfPath = await convertImageToPdf(filePath, tmpDir);- }- else {+ } else {// Unsupported format// Assume its a text-based format and pass through text directly- const content = await fs.readFile(filePath, 'utf-8');+ const content = await fs.readFile(filePath, "utf-8");return {content,};@@ -301,7 +270,7 @@ export async function convertToPdf(} catch (error) {return {message: error instanceof Error ? error.message : String(error),- code: 'CONVERSION_ERROR',+ code: "CONVERSION_ERROR",};}}diff --git a/src/core/config.ts b/src/core/config.tsindex 1eef499..fdd9d1a 100644--- a/src/core/config.ts+++ b/src/core/config.ts@@ -1,9 +1,9 @@-import { LiteParseConfig } from './types.js';+import { LiteParseConfig } from "./types.js";export const DEFAULT_CONFIG: LiteParseConfig = {// OCR - defaults to in-process Tesseract for zero-setup experience// If ocrServerUrl is provided, uses HTTP OCR instead- ocrLanguage: 'en',+ ocrLanguage: "en",ocrEnabled: true,ocrServerUrl: undefined, // If set, uses HTTP OCR; otherwise uses Tesseract@@ -13,7 +13,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {dpi: 150,// Output- outputFormat: 'json',+ outputFormat: "json",includeImages: true,includeCharts: true,@@ -25,9 +25,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {preserveLayoutAlignmentAcrossPages: false,};-export function mergeConfig(- userConfig: Partial<LiteParseConfig>-): LiteParseConfig {+export function mergeConfig(userConfig: Partial<LiteParseConfig>): LiteParseConfig {return {...DEFAULT_CONFIG,...userConfig,diff --git a/src/core/parser.ts b/src/core/parser.tsindex 6a89d8e..3046db3 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -1,19 +1,16 @@-import { LiteParseConfig, ParseResult, ScreenshotResult } from './types.js';-import { mergeConfig } from './config.js';-import { PdfEngine } from '../engines/pdf/interface.js';-import { PdfJsEngine } from '../engines/pdf/pdfjs.js';-import { PdfiumRenderer } from '../engines/pdf/pdfium-renderer.js';-import { OcrEngine } from '../engines/ocr/interface.js';-import { TesseractEngine } from '../engines/ocr/tesseract.js';-import { HttpOcrEngine } from '../engines/ocr/http-simple.js';-import { projectPagesToGrid } from '../processing/grid.js';-import { buildBoundingBoxes } from '../processing/bbox.js';-import { detectTables } from '../processing/tables.js';-import { formatJSON } from '../output/json.js';-import {- convertToPdf,- cleanupConversionFiles,-} from '../conversion/convertToPdf.js';+import { LiteParseConfig, ParseResult, ScreenshotResult } from "./types.js";+import { mergeConfig } from "./config.js";+import { PdfEngine } from "../engines/pdf/interface.js";+import { PdfJsEngine } from "../engines/pdf/pdfjs.js";+import { PdfiumRenderer } from "../engines/pdf/pdfium-renderer.js";+import { OcrEngine } from "../engines/ocr/interface.js";+import { TesseractEngine } from "../engines/ocr/tesseract.js";+import { HttpOcrEngine } from "../engines/ocr/http-simple.js";+import { projectPagesToGrid } from "../processing/grid.js";+import { buildBoundingBoxes } from "../processing/bbox.js";+import { detectTables } from "../processing/tables.js";+import { formatJSON } from "../output/json.js";+import { convertToPdf, cleanupConversionFiles } from "../conversion/convertToPdf.js";export class LiteParse {private config: LiteParseConfig;@@ -25,7 +22,7 @@ export class LiteParse {this.config = mergeConfig(userConfig);// Initialize PDF engine- this.pdfEngine = new PdfJsEngine()+ this.pdfEngine = new PdfJsEngine();// Initialize OCR engine// Auto-detect: use HTTP OCR if URL provided, otherwise use Tesseract@@ -53,13 +50,13 @@ export class LiteParse {log(`Processing file: ${filePath}`);const conversionResult = await convertToPdf(filePath);- if ('code' in conversionResult) {+ if ("code" in conversionResult) {// Conversion errorthrow new Error(`Conversion failed: ${conversionResult.message}`);}// Return early for text-based passthrough formats- if ('content' in conversionResult) {+ if ("content" in conversionResult) {log(`File is a text-based format. Returning content directly.`);return {pages: [],@@ -114,13 +111,13 @@ export class LiteParse {}// Build final text- const fullText = processedPages.map((p) => p.text).join('\n\n');+ const fullText = processedPages.map((p) => p.text).join("\n\n");// Close PDF documentawait this.pdfEngine.close(doc);// Cleanup OCR engine if it's Tesseract (to free memory)- if (this.ocrEngine && 'terminate' in this.ocrEngine) {+ if (this.ocrEngine && "terminate" in this.ocrEngine) {await (this.ocrEngine as TesseractEngine).terminate();}@@ -136,10 +133,10 @@ export class LiteParse {// Format based on output formatswitch (this.config.outputFormat) {- case 'json':+ case "json":result.json = JSON.parse(formatJSON(result));break;- case 'text':+ case "text":// Already in text formatbreak;}@@ -179,11 +176,7 @@ export class LiteParse {}log(`Rendering page ${pageNum}...`);- const imageBuffer = await renderer.renderPageToBuffer(- filePath,- pageNum,- this.config.dpi- );+ const imageBuffer = await renderer.renderPageToBuffer(filePath, pageNum, this.config.dpi);// Get page dimensionsconst pageData = await this.pdfEngine.extractPage(doc, pageNum);@@ -208,14 +201,10 @@ export class LiteParse {/*** Run OCR on pages that need it*/- private async runOCR(- doc: any,- pages: any[],- log: (msg: string) => void- ): Promise<void> {+ private async runOCR(doc: any, pages: any[], log: (msg: string) => void): Promise<void> {if (!this.ocrEngine) return;- log('Running OCR on pages...');+ log("Running OCR on pages...");for (let i = 0; i < pages.length; i++) {const page = pages[i];@@ -237,14 +226,11 @@ export class LiteParse {);// Save temporary image file- const fs = await import('fs/promises');- const path = await import('path');- const os = await import('os');+ const fs = await import("fs/promises");+ const path = await import("path");+ const os = await import("os");const tmpDir = os.tmpdir();- const tmpImagePath = path.join(- tmpDir,- `page_${page.pageNum}_ocr.png`- );+ const tmpImagePath = path.join(tmpDir, `page_${page.pageNum}_ocr.png`);await fs.writeFile(tmpImagePath, imageBuffer);// Run OCR@@ -264,16 +250,14 @@ export class LiteParse {y: r.bbox[1],width: r.bbox[2] - r.bbox[0],height: r.bbox[3] - r.bbox[1],- fontName: 'OCR',+ fontName: "OCR",fontSize: r.bbox[3] - r.bbox[1],fromOcr: true,}));// Add OCR text items directly to page textItemspage.textItems.push(...ocrTextItems);- log(- ` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`- );+ log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);}// Clean up temp filediff --git a/src/core/types.ts b/src/core/types.tsindex 5b47aea..4b174a5 100644--- a/src/core/types.ts+++ b/src/core/types.ts@@ -1,6 +1,6 @@-import { DetectedTable } from '../processing/tables.js';+import { DetectedTable } from "../processing/tables.js";-export type OutputFormat = 'json' | 'text';+export type OutputFormat = "json" | "text";export interface LiteParseConfig {// OCR@@ -34,8 +34,8 @@ export interface TextItem {y: number;width: number;height: number;- w: number; // Alias for width- h: number; // Alias for height+ w: number; // Alias for width+ h: number; // Alias for heightfontName?: string;fontSize?: number;r?: number; // Rotation angle in degrees (0, 90, 180, 270)@@ -70,7 +70,7 @@ export interface ProjectionTextBox {fromOCR?: boolean;// Projection metadata- snap?: 'left' | 'right' | 'center';+ snap?: "left" | "right" | "center";leftAnchor?: string;rightAnchor?: string;centerAnchor?: string;diff --git a/src/engines/ocr/http-simple.ts b/src/engines/ocr/http-simple.tsindex c5c9350..ee87ee8 100644--- a/src/engines/ocr/http-simple.ts+++ b/src/engines/ocr/http-simple.ts@@ -1,7 +1,7 @@-import axios from 'axios';-import FormData from 'form-data';-import fs from 'fs';-import { OcrEngine, OcrOptions, OcrResult } from './interface.js';+import axios from "axios";+import FormData from "form-data";+import fs from "fs";+import { OcrEngine, OcrOptions, OcrResult } from "./interface.js";/*** HTTP-based OCR engine that conforms to LiteParse OCR API specification.@@ -14,26 +14,21 @@ import { OcrEngine, OcrOptions, OcrResult } from './interface.js';* See ocr/easyocr/ and ocr/paddleocr/ for example server implementations.*/export class HttpOcrEngine implements OcrEngine {- name = 'http-ocr';+ name = "http-ocr";private serverUrl: string;constructor(serverUrl: string) {this.serverUrl = serverUrl;}- async recognize(- imagePath: string,- options: OcrOptions- ): Promise<OcrResult[]> {+ async recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]> {try {// Prepare requestconst formData = new FormData();- formData.append('file', fs.createReadStream(imagePath));+ formData.append("file", fs.createReadStream(imagePath));- const language = Array.isArray(options.language)- ? options.language[0]- : options.language;- formData.append('language', language);+ const language = Array.isArray(options.language) ? options.language[0] : options.language;+ formData.append("language", language);// Make HTTP requestconst response = await axios.post(this.serverUrl, formData, {@@ -45,7 +40,7 @@ export class HttpOcrEngine implements OcrEngine {const { results } = response.data;if (!Array.isArray(results)) {- console.warn('OCR server response missing results array:', response.data);+ console.warn("OCR server response missing results array:", response.data);return [];}@@ -70,10 +65,7 @@ export class HttpOcrEngine implements OcrEngine {}}- async recognizeBatch(- imagePaths: string[],- options: OcrOptions- ): Promise<OcrResult[][]> {+ async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {const results: OcrResult[][] = [];for (const imagePath of imagePaths) {const result = await this.recognize(imagePath, options);diff --git a/src/engines/ocr/interface.ts b/src/engines/ocr/interface.tsindex 1d2c44c..5759084 100644--- a/src/engines/ocr/interface.ts+++ b/src/engines/ocr/interface.ts@@ -1,10 +1,7 @@export interface OcrEngine {name: string;recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]>;- recognizeBatch(- imagePaths: string[],- options: OcrOptions- ): Promise<OcrResult[][]>;+ recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]>;}export interface OcrOptions {diff --git a/src/engines/ocr/tesseract.ts b/src/engines/ocr/tesseract.tsindex 684822b..793d76a 100644--- a/src/engines/ocr/tesseract.ts+++ b/src/engines/ocr/tesseract.ts@@ -1,13 +1,12 @@-import { createWorker, Worker } from 'tesseract.js';-import { OcrEngine, OcrOptions, OcrResult } from './interface.js';-+import { createWorker, Worker } from "tesseract.js";+import { OcrEngine, OcrOptions, OcrResult } from "./interface.js";export class TesseractEngine implements OcrEngine {- name = 'tesseract';+ name = "tesseract";private worker?: Worker;private currentLanguage?: string;- async initialize(language: string = 'eng'): Promise<void> {+ async initialize(language: string = "eng"): Promise<void> {if (this.worker && this.currentLanguage === language) {return; // Already initialized for this language}@@ -21,10 +20,7 @@ export class TesseractEngine implements OcrEngine {this.currentLanguage = language;}- async recognize(- imagePath: string,- options: OcrOptions- ): Promise<OcrResult[]> {+ async recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]> {// Handle language - tesseract.js uses language codes like 'eng', 'fra', 'deu'const language = this.normalizeLanguage(Array.isArray(options.language) ? options.language[0] : options.language@@ -34,7 +30,7 @@ export class TesseractEngine implements OcrEngine {await this.initialize(language);if (!this.worker) {- throw new Error('Tesseract worker not initialized');+ throw new Error("Tesseract worker not initialized");}try {@@ -46,12 +42,12 @@ export class TesseractEngine implements OcrEngine {// Convert to our OcrResult formatconst results: OcrResult[] = words.map((word) => ({text: word.text,- bbox: [- word.bbox.x0,- word.bbox.y0,- word.bbox.x1,- word.bbox.y1,- ] as [number, number, number, number],+ bbox: [word.bbox.x0, word.bbox.y0, word.bbox.x1, word.bbox.y1] as [+ number,+ number,+ number,+ number,+ ],confidence: word.confidence / 100, // Tesseract returns 0-100, we want 0-1}));@@ -63,10 +59,7 @@ export class TesseractEngine implements OcrEngine {}}- async recognizeBatch(- imagePaths: string[],- options: OcrOptions- ): Promise<OcrResult[][]> {+ async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {const results: OcrResult[][] = [];for (let i = 0; i < imagePaths.length; i++) {@@ -92,22 +85,22 @@ export class TesseractEngine implements OcrEngine {*/private normalizeLanguage(lang: string): string {const languageMap: Record<string, string> = {- en: 'eng',- fr: 'fra',- de: 'deu',- es: 'spa',- it: 'ita',- pt: 'por',- ru: 'rus',- zh: 'chi_sim',- 'zh-cn': 'chi_sim',- 'zh-tw': 'chi_tra',- ja: 'jpn',- ko: 'kor',- ar: 'ara',- hi: 'hin',- th: 'tha',- vi: 'vie',+ en: "eng",+ fr: "fra",+ de: "deu",+ es: "spa",+ it: "ita",+ pt: "por",+ ru: "rus",+ zh: "chi_sim",+ "zh-cn": "chi_sim",+ "zh-tw": "chi_tra",+ ja: "jpn",+ ko: "kor",+ ar: "ara",+ hi: "hin",+ th: "tha",+ vi: "vie",};const normalized = lang.toLowerCase().trim();diff --git a/src/engines/pdf/interface.ts b/src/engines/pdf/interface.tsindex 5c8c034..d7d86cd 100644--- a/src/engines/pdf/interface.ts+++ b/src/engines/pdf/interface.ts@@ -1,19 +1,11 @@-import { TextItem } from '../../core/types.js';+import { TextItem } from "../../core/types.js";export interface PdfEngine {name: string;loadDocument(filePath: string): Promise<PdfDocument>;extractPage(doc: PdfDocument, pageNum: number): Promise<PageData>;- extractAllPages(- doc: PdfDocument,- maxPages?: number,- targetPages?: string- ): Promise<PageData[]>;- renderPageImage(- doc: PdfDocument,- pageNum: number,- dpi: number- ): Promise<Buffer>;+ extractAllPages(doc: PdfDocument, maxPages?: number, targetPages?: string): Promise<PageData[]>;+ renderPageImage(doc: PdfDocument, pageNum: number, dpi: number): Promise<Buffer>;close(doc: PdfDocument): Promise<void>;}@@ -34,7 +26,7 @@ export interface PageData {}export interface Path {- type: 'rectangle' | 'line' | 'curve';+ type: "rectangle" | "line" | "curve";points: number[][];color?: string;width?: number;@@ -65,7 +57,7 @@ export interface Image {export type EasyOcrResultLine = [[[number, number], [number, number], [number, number], [number, number]],string,- string | number+ string | number,];export interface Annotation {diff --git a/src/engines/pdf/pdfium-renderer.ts b/src/engines/pdf/pdfium-renderer.tsindex 89d4719..e5c0705 100644--- a/src/engines/pdf/pdfium-renderer.ts+++ b/src/engines/pdf/pdfium-renderer.ts@@ -1,6 +1,6 @@-import { PDFiumLibrary, type PDFiumPageRenderOptions } from '@hyzyla/pdfium';-import sharp from 'sharp';-import { promises as fs } from 'fs';+import { PDFiumLibrary, type PDFiumPageRenderOptions } from "@hyzyla/pdfium";+import sharp from "sharp";+import { promises as fs } from "fs";/*** PDFium-based PDF screenshot renderer@@ -23,7 +23,7 @@ export class PdfiumRenderer {await this.init();if (!this.pdfium) {- throw new Error('PDFium not initialized');+ throw new Error("PDFium not initialized");}// Read PDF filediff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.tsindex 83db1d2..786a756 100644--- a/src/engines/pdf/pdfjs.ts+++ b/src/engines/pdf/pdfjs.ts@@ -1,22 +1,15 @@-import fs from 'node:fs/promises';-import { fileURLToPath } from 'node:url';-import { dirname, join } from 'node:path';-import {- PdfEngine,- PdfDocument,- PageData,- Path,- Image,- Annotation,-} from './interface.js';-import { TextItem } from '../../core/types.js';-import { PdfiumRenderer } from './pdfium-renderer.js';+import fs from "node:fs/promises";+import { fileURLToPath } from "node:url";+import { dirname, join } from "node:path";+import { PdfEngine, PdfDocument, PageData, Path, Image, Annotation } from "./interface.js";+import { TextItem } from "../../core/types.js";+import { PdfiumRenderer } from "./pdfium-renderer.js";// Dynamic import of PDF.jsconst __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);// From dist/src/engines/pdf/ we need to go up to dist/src/vendor/pdfjs-const PDFJS_DIR = join(__dirname, '../../vendor/pdfjs');+const PDFJS_DIR = join(__dirname, "../../vendor/pdfjs");// Import PDF.js dynamicallyawait import(`${PDFJS_DIR}/pdf.mjs`);@@ -45,17 +38,20 @@ function multiplyMatrices(m1: number[], m2: number[]): number[] {m1[0] * m2[2] + m1[2] * m2[3],m1[1] * m2[2] + m1[3] * m2[3],m1[0] * m2[4] + m1[2] * m2[5] + m1[4],- m1[1] * m2[4] + m1[3] * m2[5] + m1[5]+ m1[1] * m2[4] + m1[3] * m2[5] + m1[5],];}/*** Apply transformation matrix to a point*/-function applyTransformation(point: { x: number; y: number }, transform: number[]): { x: number; y: number } {+function applyTransformation(+ point: { x: number; y: number },+ transform: number[]+): { x: number; y: number } {return {x: point.x * transform[0] + point.y * transform[2] + transform[4],- y: point.x * transform[1] + point.y * transform[3] + transform[5]+ y: point.x * transform[1] + point.y * transform[3] + transform[5],};}@@ -85,11 +81,11 @@ function singularValueDecompose2dScale(m: number[]): { x: number; y: number } {// Pre-compiled regex patterns for string decoding (avoid recompilation per item)const BUGGY_FONT_MARKER_REGEX = /:->|>_(\d+)_\d+_<|<-:/g;-const BUGGY_FONT_MARKER_CHECK = ':->|>';+const BUGGY_FONT_MARKER_CHECK = ":->|>";const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;export class PdfJsEngine implements PdfEngine {- name = 'pdfjs';+ name = "pdfjs";private pdfiumRenderer: PdfiumRenderer | null = null;private currentPdfPath: string | null = null;@@ -146,10 +142,7 @@ export class PdfJsEngine implements PdfEngine {const scale = singularValueDecompose2dScale(item.transform);// Get upper-right corner- const ur = applyTransformation(- { x: item.width / scale.x, y: item.height / scale.y },- cm- );+ const ur = applyTransformation({ x: item.width / scale.x, y: item.height / scale.y }, cm);// Calculate final bounding box in viewport spaceconst left = Math.min(ll.x, ur.x);@@ -175,19 +168,18 @@ export class PdfJsEngine implements PdfEngine {let decodedStr = item.str;if (decodedStr.includes(BUGGY_FONT_MARKER_CHECK)) {BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state- decodedStr = decodedStr.replace(- BUGGY_FONT_MARKER_REGEX,- (_: string, charCode: string) => String.fromCharCode(parseInt(charCode))+ decodedStr = decodedStr.replace(BUGGY_FONT_MARKER_REGEX, (_: string, charCode: string) =>+ String.fromCharCode(parseInt(charCode)));}// Handle pipe-separated characters: " |a| |r| |X| " -> "arX"// Some PDFs encode text with characters separated by pipes and spaces- if (decodedStr.includes('|')) {+ if (decodedStr.includes("|")) {PIPE_PATTERN_REGEX.lastIndex = 0; // Reset regex stateconst matches = [...decodedStr.matchAll(PIPE_PATTERN_REGEX)];if (matches.length > 0) {- decodedStr = matches.map(m => m[1]).join('');+ decodedStr = matches.map((m) => m[1]).join("");}}@@ -233,10 +225,7 @@ export class PdfJsEngine implements PdfEngine {maxPages?: number,targetPages?: string): Promise<PageData[]> {- const numPages = Math.min(- doc.numPages,- maxPages || doc.numPages- );+ const numPages = Math.min(doc.numPages, maxPages || doc.numPages);const pages: PageData[] = [];@@ -259,25 +248,17 @@ export class PdfJsEngine implements PdfEngine {return pages;}- async renderPageImage(- _doc: PdfDocument,- pageNum: number,- dpi: number- ): Promise<Buffer> {+ async renderPageImage(_doc: PdfDocument, pageNum: number, dpi: number): Promise<Buffer> {// Use PDFium for rendering (more robust with inline images)if (!this.currentPdfPath) {- throw new Error('PDF path not available for rendering');+ throw new Error("PDF path not available for rendering");}if (!this.pdfiumRenderer) {this.pdfiumRenderer = new PdfiumRenderer();}- return await this.pdfiumRenderer.renderPageToBuffer(- this.currentPdfPath,- pageNum,- dpi- );+ return await this.pdfiumRenderer.renderPageToBuffer(this.currentPdfPath, pageNum, dpi);}async close(doc: PdfDocument): Promise<void> {@@ -296,13 +277,13 @@ export class PdfJsEngine implements PdfEngine {private parseTargetPages(targetPages: string, maxPages: number): number[] {const pages: number[] = [];- const parts = targetPages.split(',');+ const parts = targetPages.split(",");for (const part of parts) {const trimmed = part.trim();- if (trimmed.includes('-')) {+ if (trimmed.includes("-")) {// Range: "1-5"- const [start, end] = trimmed.split('-').map((n) => parseInt(n.trim()));+ const [start, end] = trimmed.split("-").map((n) => parseInt(n.trim()));for (let i = start; i <= Math.min(end, maxPages); i++) {if (i >= 1) {pages.push(i);diff --git a/src/index.ts b/src/index.tsindex 7d9b3e9..662ecab 100644--- a/src/index.ts+++ b/src/index.ts@@ -1,6 +1,6 @@#!/usr/bin/env node-import { program } from '../cli/parse.js';+import { program } from "../cli/parse.js";// Run the CLIprogram.parse(process.argv);diff --git a/src/lib.ts b/src/lib.tsindex b324b42..36d0d6a 100644--- a/src/lib.ts+++ b/src/lib.ts@@ -1,4 +1,4 @@-export { LiteParse } from './core/parser.js';+export { LiteParse } from "./core/parser.js";export type {LiteParseConfig,OutputFormat,@@ -8,4 +8,4 @@ export type {TextItem,// ImageItem,// TableItem-} from './core/types.js';+} from "./core/types.js";diff --git a/src/output/json.ts b/src/output/json.tsindex ad3973a..2ee156f 100644--- a/src/output/json.ts+++ b/src/output/json.ts@@ -1,4 +1,4 @@-import { ParseResult, ParsedPage } from '../core/types.js';+import { ParseResult, ParsedPage } from "../core/types.js";/*** Build JSON output from parsed pagesdiff --git a/src/output/text.ts b/src/output/text.tsindex 680ca22..60d9567 100644--- a/src/output/text.ts+++ b/src/output/text.ts@@ -1,4 +1,4 @@-import { ParseResult, ParsedPage } from '../core/types.js';+import { ParseResult, ParsedPage } from "../core/types.js";/*** Format pages as plain text@@ -9,7 +9,7 @@ export function formatText(result: ParseResult): string {return header + page.text;});- return pageTexts.join('\n\n');+ return pageTexts.join("\n\n");}/**diff --git a/src/processing/bbox.ts b/src/processing/bbox.tsindex 26165ef..b11eaaa 100644--- a/src/processing/bbox.ts+++ b/src/processing/bbox.ts@@ -4,9 +4,9 @@ import {ProjectionTextBox,OcrData,LiteParseConfig,-} from '../core/types.js';-import { PageData, Image } from '../engines/pdf/interface.js';-import { parseImageOcrBlocks, OcrBlock } from './ocrUtils.js';+} from "../core/types.js";+import { PageData, Image } from "../engines/pdf/interface.js";+import { parseImageOcrBlocks, OcrBlock } from "./ocrUtils.js";const OCR_CONFIDENCE_THRESHOLD = 0.1;@@ -46,8 +46,7 @@ export function filterImagesForOCR(): Image[] {// Filter images that start with g_ or pattern_ (generated/pattern images)let filtered = images.filter(- (image) =>- !image.type?.startsWith('g_') && !image.type?.startsWith('pattern_')+ (image) => !image.type?.startsWith("g_") && !image.type?.startsWith("pattern_"));// Limit to max images per page, keeping the largest ones@@ -59,7 +58,7 @@ export function filterImagesForOCR(// Apply additional filtering criteriafiltered = filtered.filter((image) => {// Ignore layout extracted images- if (image.type?.includes('layout_')) {+ if (image.type?.includes("layout_")) {return false;}@@ -173,10 +172,7 @@ function filterOcrBlocksOverlappingWithText(totalOverlapArea += overlapArea;// Condition 2: Reject if OCR block covers more than 50% of any single text item- if (- textItemArea > 0 &&- overlapArea / textItemArea >= OCR_OVERLAP_THRESHOLD- ) {+ if (textItemArea > 0 && overlapArea / textItemArea >= OCR_OVERLAP_THRESHOLD) {return false;}}@@ -196,10 +192,7 @@ function filterOcrBlocksOverlappingWithText(* Build projection text boxes from page data, including OCR results* This is the complete implementation from buildBbox.ts*/-export function buildBbox(- pageData: PageData,- config: LiteParseConfig-): ProjectionTextBox[] {+export function buildBbox(pageData: PageData, config: LiteParseConfig): ProjectionTextBox[] {const lines: ProjectionTextBox[] = [];// Process all extracted text items@@ -244,7 +237,6 @@ export function buildBbox(}));for (const image of imagesToProcess) {-// Parse OCR blocks from imagelet ocrData = parseImageOcrBlocks(image);@@ -258,8 +250,7 @@ export function buildBbox(const ocrParsed: OcrData[] = [];for (const block of ocrData) {- const confidenceRounded =- Math.round(parseFloat(block.confidence.toString()) * 1000) / 1000;+ const confidenceRounded = Math.round(parseFloat(block.confidence.toString()) * 1000) / 1000;const line: ProjectionTextBox = {fromOCR: true,@@ -305,7 +296,7 @@ export function buildBoundingBoxes(textItems: TextItem[]): BoundingBox[] {const bboxes: BoundingBox[] = [];for (const item of textItems) {- if (item.str.trim() === '') {+ if (item.str.trim() === "") {continue;}diff --git a/src/processing/cleanText.ts b/src/processing/cleanText.tsindex 551ec6e..d651909 100644--- a/src/processing/cleanText.ts+++ b/src/processing/cleanText.ts@@ -1,4 +1,4 @@-import { ParsedPage, LiteParseConfig } from '../core/types.js';+import { ParsedPage, LiteParseConfig } from "../core/types.js";/*** Detect and remove margins from a single page.@@ -9,7 +9,7 @@ import { ParsedPage, LiteParseConfig } from '../core/types.js';* - Right margin (trailing whitespace on each line)*/function detectAndRemoveMarginOnPage(page: ParsedPage): void {- const lines = page.text.split('\n');+ const lines = page.text.split("\n");let minX: number | undefined = undefined;let minY: number | undefined = undefined;@@ -39,7 +39,7 @@ function detectAndRemoveMarginOnPage(page: ParsedPage): void {// If page is entirely empty, just returnif (minX === undefined || minY === undefined || maxY === undefined) {- page.text = '';+ page.text = "";return;}@@ -60,7 +60,7 @@ function detectAndRemoveMarginOnPage(page: ParsedPage): void {newLines.push(line.slice(minX));}- page.text = newLines.join('\n');+ page.text = newLines.join("\n");}/**@@ -76,15 +76,12 @@ function detectAndRemoveMargin(pages: ParsedPage[]): void {/*** Clean raw text output - removes margins, null characters*/-export function cleanRawText(- pages: ParsedPage[],- _config: LiteParseConfig-): void {+export function cleanRawText(pages: ParsedPage[], _config: LiteParseConfig): void {// Remove margins (per-page)detectAndRemoveMargin(pages);// Remove null charactersfor (const page of pages) {- page.text = page.text.replace(/\u0000/g, ' ');+ page.text = page.text.replace(/\u0000/g, " ");}}diff --git a/src/processing/grid.ts b/src/processing/grid.tsindex 8bb52b2..abb2747 100644--- a/src/processing/grid.ts+++ b/src/processing/grid.ts@@ -1,7 +1,7 @@-import { PageData } from '../engines/pdf/interface.js';-import { ParsedPage, LiteParseConfig } from '../core/types.js';-import { projectPagesToGrid as projectPagesToGridComplete } from './gridProjection.js';-import { DEFAULT_CONFIG } from '../core/config.js';+import { PageData } from "../engines/pdf/interface.js";+import { ParsedPage, LiteParseConfig } from "../core/types.js";+import { projectPagesToGrid as projectPagesToGridComplete } from "./gridProjection.js";+import { DEFAULT_CONFIG } from "../core/config.js";/*** Projects text items onto a grid for spatial text extractiondiff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.tsindex 2d47b5a..4b05817 100644--- a/src/processing/gridProjection.ts+++ b/src/processing/gridProjection.ts@@ -1,1519 +1,1626 @@-import { strToSubscriptString, strToPostScript } from './textUtils.js';-import { buildBbox } from './bbox.js';-import { cleanRawText } from './cleanText.js';-import { ProjectionTextBox, Coordinates, LiteParseConfig, ParsedPage } from '../core/types.js';-import { PageData } from '../engines/pdf/interface.js';+import { strToSubscriptString, strToPostScript } from "./textUtils.js";+import { buildBbox } from "./bbox.js";+import { cleanRawText } from "./cleanText.js";+import { ProjectionTextBox, Coordinates, LiteParseConfig, ParsedPage } from "../core/types.js";+import { PageData } from "../engines/pdf/interface.js";-import { getVerticalAndHorizontalLinesFromPagePaths, VerticalLine } from './pathUtils.js';-import { applyMarkupTags } from './markupUtils.js';+import { getVerticalAndHorizontalLinesFromPagePaths, VerticalLine } from "./pathUtils.js";+import { applyMarkupTags } from "./markupUtils.js";// Minimum spaces between unsnapped bboxes (likely justified textconst FLOATING_SPACES = 2;// Minimum spaces between snapped columnsconst COLUMN_SPACES = 4;-type Snap = {- bbox: ProjectionTextBox;- lineIndex: number;- boxIndex: number;-}+ bbox: ProjectionTextBox;+ lineIndex: number;+ boxIndex: number;+};type Anchor = {- [key: number]: ProjectionTextBox[];-}+ [key: number]: ProjectionTextBox[];+};type Anchors = {- anchorLeft: Anchor;- anchorRight: Anchor;- anchorCenter: Anchor;+ anchorLeft: Anchor;+ anchorRight: Anchor;+ anchorCenter: Anchor;};type ForwardAnchor = { [key: string]: number };type PrevAnchors = {- forwardAnchorLeft: ForwardAnchor;- forwardAnchorCenter: ForwardAnchor;- forwardAnchorRight: ForwardAnchor;-}+ forwardAnchorLeft: ForwardAnchor;+ forwardAnchorCenter: ForwardAnchor;+ forwardAnchorRight: ForwardAnchor;+};type PageForwardAnchors = {- left: ForwardAnchor,- right: ForwardAnchor,- center: ForwardAnchor,- floating: ForwardAnchor,-}+ left: ForwardAnchor;+ right: ForwardAnchor;+ center: ForwardAnchor;+ floating: ForwardAnchor;+};type SnapMaps = {- left: number[],- right: number[],- center: number[],- floating: number[],-}+ left: number[];+ right: number[];+ center: number[];+ floating: number[];+};type LineRange = {- start: number,- end: number,-}+ start: number;+ end: number;+};type TextBoxSize = {- width: number,- height: number,-}+ width: number;+ height: number;+};function roundAnchor(anchor: number): number {- // group anchor x-coord by nearest 1/4 unit- return Math.round(anchor * 4) / 4;+ // group anchor x-coord by nearest 1/4 unit+ return Math.round(anchor * 4) / 4;}// 2pt @ PDF 72 DPI -> 8px @ 300DPIconst SMALL_FONT_SIZE_THRESHOLD = 2;function isSmallTextLine(line: ProjectionTextBox[]): boolean {- // check for line where >50% of the text is very small- const smallText = line.filter(item => item.h < SMALL_FONT_SIZE_THRESHOLD);- if (smallText.length / line.length > .5) {- return true;- }- return false;+ // check for line where >50% of the text is very small+ const smallText = line.filter((item) => item.h < SMALL_FONT_SIZE_THRESHOLD);+ if (smallText.length / line.length > 0.5) {+ return true;+ }+ return false;}-function filterUnprojectableText(config: LiteParseConfig, line: ProjectionTextBox[]): ProjectionTextBox[] {- // extract all text (OSS always uses fast mode)- if (line.length === 0) {- return line;- }-- let filteredLine = line;- if (!config.preserveVerySmallText && isSmallTextLine(line)) {- // remove very small text lines- filteredLine = filteredLine.filter(item => item.h >= SMALL_FONT_SIZE_THRESHOLD);- }- return filteredLine;+function filterUnprojectableText(+ config: LiteParseConfig,+ line: ProjectionTextBox[]+): ProjectionTextBox[] {+ // extract all text (OSS always uses fast mode)+ if (line.length === 0) {+ return line;+ }++ let filteredLine = line;+ if (!config.preserveVerySmallText && isSmallTextLine(line)) {+ // remove very small text lines+ filteredLine = filteredLine.filter((item) => item.h >= SMALL_FONT_SIZE_THRESHOLD);+ }+ return filteredLine;}function canSnapLine(config: LiteParseConfig, line: ProjectionTextBox[]): boolean {- // force lines that will likely break projection to be unsnapped floating text- // currently this includes:- // - lines of entirely small text- //- // NOTE: this assumes undesirable text has already been filtered before projection- // (i.e. parse mode based removal of text should be done before this in filterUnprojectableText())- if (line.length === 0) {- return true;- }-- if (!config.preserveVerySmallText && isSmallTextLine(line)) {- return false;- }+ // force lines that will likely break projection to be unsnapped floating text+ // currently this includes:+ // - lines of entirely small text+ //+ // NOTE: this assumes undesirable text has already been filtered before projection+ // (i.e. parse mode based removal of text should be done before this in filterUnprojectableText())+ if (line.length === 0) {return true;+ }++ if (!config.preserveVerySmallText && isSmallTextLine(line)) {+ return false;+ }+ return true;}function fixSparseBlocks(blocks: LineRange[], rawLines: string[]) {- // compress whitespace in blocks containing very sparse lines (>70% whitespace)- const regexp = new RegExp(`\\s{${COLUMN_SPACES},}`, "g");- for (const block of blocks) {- let total = 0;- let whitespace = 0;- for (let i = block.start; i < block.end; ++i) {- if (!rawLines[i]) {- continue;- }- rawLines[i] = rawLines[i].trimEnd();- const line = rawLines[i];- if (line.length === 0) {- continue;- }- total += line.length;- whitespace += line.match(/\s/g)?.length || 0- }- if (total >= 500 && whitespace / total > 0.8) {- for (let i = block.start; i < block.end; ++i) {- const line = rawLines[i];- if (!line || line.length === 0) {- continue;- }- rawLines[i] = line.replace(regexp, " ".repeat(FLOATING_SPACES))- }+ // compress whitespace in blocks containing very sparse lines (>70% whitespace)+ const regexp = new RegExp(`\\s{${COLUMN_SPACES},}`, "g");+ for (const block of blocks) {+ let total = 0;+ let whitespace = 0;+ for (let i = block.start; i < block.end; ++i) {+ if (!rawLines[i]) {+ continue;+ }+ rawLines[i] = rawLines[i].trimEnd();+ const line = rawLines[i];+ if (line.length === 0) {+ continue;+ }+ total += line.length;+ whitespace += line.match(/\s/g)?.length || 0;+ }+ if (total >= 500 && whitespace / total > 0.8) {+ for (let i = block.start; i < block.end; ++i) {+ const line = rawLines[i];+ if (!line || line.length === 0) {+ continue;}+ rawLines[i] = line.replace(regexp, " ".repeat(FLOATING_SPACES));+ }}+ }}-function extractAnchorsPointsFromLines(lines: ProjectionTextBox[][], page: PageData, verticalLines: VerticalLine[]): Anchors {- const pageHeight = page.height;-- let anchorLeft: Anchor = {};- let anchorRight: Anchor = {};- let anchorCenter: Anchor = {};-- // Add anchors from vertical lines to ensure horizontal gaps- // This creates left and right anchors at vertical line positions- for (let vLine of verticalLines) {- const leftAnchor = roundAnchor(vLine.x);- const rightAnchor = roundAnchor(vLine.x + vLine.width);-- if (!anchorLeft[leftAnchor]) {- anchorLeft[leftAnchor] = [];- }- if (!anchorRight[rightAnchor]) {- anchorRight[rightAnchor] = [];- }-- // Create dummy bboxes spanning the vertical line height to establish the anchor- const dummyBboxLeft = {- x: vLine.x,- y: vLine.y1,- w: 0,- h: vLine.height,- str: '',- strLength: 0- } as ProjectionTextBox;-- const dummyBboxRight = {- x: vLine.x + vLine.width,- y: vLine.y1,- w: 0,- h: vLine.height,- str: '',- strLength: 0- } as ProjectionTextBox;-- anchorLeft[leftAnchor].push(dummyBboxLeft);- anchorRight[rightAnchor].push(dummyBboxRight);+function extractAnchorsPointsFromLines(+ lines: ProjectionTextBox[][],+ page: PageData,+ verticalLines: VerticalLine[]+): Anchors {+ const pageHeight = page.height;++ const anchorLeft: Anchor = {};+ const anchorRight: Anchor = {};+ const anchorCenter: Anchor = {};++ // Add anchors from vertical lines to ensure horizontal gaps+ // This creates left and right anchors at vertical line positions+ for (const vLine of verticalLines) {+ const leftAnchor = roundAnchor(vLine.x);+ const rightAnchor = roundAnchor(vLine.x + vLine.width);++ if (!anchorLeft[leftAnchor]) {+ anchorLeft[leftAnchor] = [];}-- for (let line of lines) {- for (let bbox of line) {- let anchor = roundAnchor(bbox.x);- if (!anchorLeft[anchor]) {- anchorLeft[anchor] = [];- }- anchorLeft[anchor].push(bbox);-- anchor = roundAnchor(bbox.x + bbox.w);- if (!anchorRight[anchor]) {- anchorRight[anchor] = [];- }- anchorRight[anchor].push(bbox);-- let center = Math.round(bbox.x + bbox.w / 2);- if (!anchorCenter[center]) {- anchorCenter[center] = [];- }- anchorCenter[center].push(bbox);- }+ if (!anchorRight[rightAnchor]) {+ anchorRight[rightAnchor] = [];}- function deltaMin(collection: Anchor, delta: number) {- for (let anchor in collection) {- let maxDelta = pageHeight * delta;- for (let i = 0; i < collection[anchor].length; i++) {- let shouldKeep = false;- if (i > 0) {- if (collection[anchor][i].y - collection[anchor][i - 1].y < maxDelta) {- shouldKeep = true;- }- }- if (i < collection[anchor].length - 1) {- if (collection[anchor][i + 1].y - collection[anchor][i].y < maxDelta) {- shouldKeep = true;- }- }+ // Create dummy bboxes spanning the vertical line height to establish the anchor+ const dummyBboxLeft = {+ x: vLine.x,+ y: vLine.y1,+ w: 0,+ h: vLine.height,+ str: "",+ strLength: 0,+ } as ProjectionTextBox;++ const dummyBboxRight = {+ x: vLine.x + vLine.width,+ y: vLine.y1,+ w: 0,+ h: vLine.height,+ str: "",+ strLength: 0,+ } as ProjectionTextBox;++ anchorLeft[leftAnchor].push(dummyBboxLeft);+ anchorRight[rightAnchor].push(dummyBboxRight);+ }++ for (const line of lines) {+ for (const bbox of line) {+ let anchor = roundAnchor(bbox.x);+ if (!anchorLeft[anchor]) {+ anchorLeft[anchor] = [];+ }+ anchorLeft[anchor].push(bbox);++ anchor = roundAnchor(bbox.x + bbox.w);+ if (!anchorRight[anchor]) {+ anchorRight[anchor] = [];+ }+ anchorRight[anchor].push(bbox);++ const center = Math.round(bbox.x + bbox.w / 2);+ if (!anchorCenter[center]) {+ anchorCenter[center] = [];+ }+ anchorCenter[center].push(bbox);+ }+ }++ function deltaMin(collection: Anchor, delta: number) {+ for (const anchor in collection) {+ const maxDelta = pageHeight * delta;+ for (let i = 0; i < collection[anchor].length; i++) {+ let shouldKeep = false;+ if (i > 0) {+ if (collection[anchor][i].y - collection[anchor][i - 1].y < maxDelta) {+ shouldKeep = true;+ }+ }+ if (i < collection[anchor].length - 1) {+ if (collection[anchor][i + 1].y - collection[anchor][i].y < maxDelta) {+ shouldKeep = true;+ }+ }- if (!shouldKeep) {- collection[anchor].splice(i--, 1);- }- }+ if (!shouldKeep) {+ collection[anchor].splice(i--, 1);}+ }}-- deltaMin(anchorRight, 0.1);- deltaMin(anchorLeft, 0.2);- deltaMin(anchorCenter, 0.05);-- function intercept(collection: Anchor) {- for (let anchor in collection) {- let shouldKeep = false;- for (let i = 0; i < collection[anchor].length; i++) {- if (i > 0) {- let intercept = false;- // check intercept- let a1 = collection[anchor][i - 1];- let a2 = collection[anchor][i];- for (let line of lines) {- if (line.length > 0 && line[0].y > a1.y && line[0].y < a2.y) {- for (let item of line) {- if (item.x < parseInt(anchor) && item.x + item.w > parseInt(anchor)) {- intercept = true;- break;- }- }- if (intercept) {- break;- }- }- }- if (!intercept) {- shouldKeep = true;- break;- }+ }++ deltaMin(anchorRight, 0.1);+ deltaMin(anchorLeft, 0.2);+ deltaMin(anchorCenter, 0.05);++ function intercept(collection: Anchor) {+ for (const anchor in collection) {+ let shouldKeep = false;+ for (let i = 0; i < collection[anchor].length; i++) {+ if (i > 0) {+ let intercept = false;+ // check intercept+ const a1 = collection[anchor][i - 1];+ const a2 = collection[anchor][i];+ for (const line of lines) {+ if (line.length > 0 && line[0].y > a1.y && line[0].y < a2.y) {+ for (const item of line) {+ if (item.x < parseInt(anchor) && item.x + item.w > parseInt(anchor)) {+ intercept = true;+ break;}+ }+ if (intercept) {+ break;+ }}- if (!shouldKeep) {- delete collection[anchor];- }+ }+ if (!intercept) {+ shouldKeep = true;+ break;+ }}+ }+ if (!shouldKeep) {+ delete collection[anchor];+ }}+ }++ intercept(anchorRight);+ intercept(anchorLeft);+ intercept(anchorCenter);++ function group(collection: Anchor) {+ for (const anchor in collection) {+ const anchorNum = parseFloat(anchor);+ // merge right+ if (+ collection[anchorNum + 1] &&+ collection[anchorNum + 1].length > collection[anchorNum].length+ ) {+ collection[anchorNum + 1].push(...collection[anchorNum]);+ delete collection[anchorNum];+ }+ // merge left+ else if (+ collection[anchorNum - 1] &&+ collection[anchorNum - 1].length > collection[anchorNum].length+ ) {+ collection[anchorNum - 1].push(...collection[anchorNum]);+ delete collection[anchorNum];+ }+ }+ }++ group(anchorLeft);+ group(anchorRight);+ group(anchorCenter);++ function anyAnchorSurvived(bbox: ProjectionTextBox) {+ return (+ roundAnchor(bbox.x) in anchorLeft ||+ roundAnchor(bbox.x + bbox.w) in anchorRight ||+ Math.round(bbox.x + bbox.w / 2) in anchorCenter+ );+ }++ // Try seeing if a floating bbox would align well with a surviving anchor on a line immediately above or below it+ function tryAlignFloating(+ collection: Anchor,+ ANCHOR_MARGIN: number,+ refXFromBbox: (bbox: ProjectionTextBox) => number,+ anchorValFromBbox: (bbox: ProjectionTextBox) => number+ ) {+ for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {+ const line = lines[lineIndex];+ for (const bbox of line) {+ // Only consider floating bboxes+ if (anyAnchorSurvived(bbox)) {+ continue;+ }+ // Check the lines before and after+ const candidateLines: ProjectionTextBox[][] = [];+ if (lineIndex > 0) {+ candidateLines.push(lines[lineIndex - 1]);+ }+ if (lineIndex < lines.length - 1) {+ candidateLines.push(lines[lineIndex + 1]);+ }- intercept(anchorRight);- intercept(anchorLeft);- intercept(anchorCenter);-- function group(collection: Anchor) {- for (let anchor in collection) {- const anchorNum = parseFloat(anchor);- // merge right- if (collection[anchorNum + 1] && collection[anchorNum + 1].length > collection[anchorNum].length) {- collection[anchorNum + 1].push(...collection[anchorNum]);- delete collection[anchorNum];+ // Check candidate lines for:+ // Possible alignment+ // Being within the margin+ // Being the closest of the candidates+ let candidateAnchor: string = "";+ let prevDiff = ANCHOR_MARGIN + 1;+ for (const candLine of candidateLines) {+ for (const candBBox of candLine) {+ const candAnchorVal = anchorValFromBbox(candBBox);+ if (!(candAnchorVal in collection)) {+ continue;}- // merge left- else if (collection[anchorNum - 1] && collection[anchorNum - 1].length > collection[anchorNum].length) {- collection[anchorNum - 1].push(...collection[anchorNum]);- delete collection[anchorNum];+ const xDiff = Math.abs(candAnchorVal - refXFromBbox(bbox));+ if (xDiff <= ANCHOR_MARGIN && xDiff < prevDiff) {+ candidateAnchor = candAnchorVal.toString();+ prevDiff = xDiff;}+ }}- }-- group(anchorLeft);- group(anchorRight);- group(anchorCenter);+ // No candidate found+ if (candidateAnchor.length == 0) {+ continue;+ }- function anyAnchorSurvived(bbox: ProjectionTextBox) {- return roundAnchor(bbox.x) in anchorLeft ||- roundAnchor(bbox.x + bbox.w) in anchorRight ||- Math.round(bbox.x + bbox.w / 2) in anchorCenter;+ // Candidate found - update the anchor's bbox list+ collection[parseFloat(candidateAnchor)].push(bbox);+ }}-- // Try seeing if a floating bbox would align well with a surviving anchor on a line immediately above or below it- function tryAlignFloating(collection: Anchor, ANCHOR_MARGIN: number, refXFromBbox: (bbox: ProjectionTextBox) => number, anchorValFromBbox: (bbox: ProjectionTextBox) => number) {- for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {- const line = lines[lineIndex];- for (let bbox of line) {- // Only consider floating bboxes- if (anyAnchorSurvived(bbox)) {- continue;- }- // Check the lines before and after- let candidateLines: ProjectionTextBox[][] = [];- if (lineIndex > 0) {- candidateLines.push(lines[lineIndex-1]);- }- if (lineIndex < lines.length-1) {- candidateLines.push(lines[lineIndex+1]);- }-- // Check candidate lines for:- // Possible alignment- // Being within the margin- // Being the closest of the candidates- let candidateAnchor: string = "";- let prevDiff = ANCHOR_MARGIN + 1;- for (let candLine of candidateLines) {- for (let candBBox of candLine) {- const candAnchorVal = anchorValFromBbox(candBBox);- if (!(candAnchorVal in collection)) {- continue;- }- const xDiff = Math.abs(candAnchorVal - refXFromBbox(bbox));- if (xDiff <= ANCHOR_MARGIN && xDiff < prevDiff) {- candidateAnchor = candAnchorVal.toString();- prevDiff = xDiff;- }- }- }-- // No candidate found- if (candidateAnchor.length == 0) {- continue;- }-- // Candidate found - update the anchor's bbox list- collection[parseFloat(candidateAnchor)].push(bbox);- }- }+ }++ // Try to left-align floating bboxes+ tryAlignFloating(+ anchorLeft,+ 2,+ (bbox) => bbox.x,+ (bbox) => roundAnchor(bbox.x)+ );++ // Sort the anchors' lists of bboxes by y-value+ function sortAnchor(collection: Anchor) {+ for (const anchor in collection) {+ collection[anchor].sort((a, b) => a.y - b.y);}-- // Try to left-align floating bboxes- tryAlignFloating(anchorLeft, 2, (bbox) => bbox.x, (bbox) => roundAnchor(bbox.x));-- // Sort the anchors' lists of bboxes by y-value- function sortAnchor(collection: Anchor) {- for (let anchor in collection) {- collection[anchor].sort((a, b) => a.y - b.y)- }+ }++ sortAnchor(anchorLeft);+ sortAnchor(anchorRight);+ sortAnchor(anchorCenter);++ // deduplicate+ const duplicates = [];+ for (const anchor in anchorLeft) {+ for (const item of anchorLeft[anchor]) {+ item.snap = "left";+ item.leftAnchor = anchor;}-- sortAnchor(anchorLeft);- sortAnchor(anchorRight);- sortAnchor(anchorCenter);--- // deduplicate- let duplicates = [];- for (let anchor in anchorLeft) {- for (let item of anchorLeft[anchor]) {- item.snap = 'left';- item.leftAnchor = anchor;- }+ }++ for (const anchor in anchorRight) {+ for (const item of anchorRight[anchor]) {+ if (item.snap) {+ item.isDup = true;+ duplicates.push(item);+ }+ item.snap = "right";+ item.rightAnchor = anchor;}-- for (let anchor in anchorRight) {- for (let item of anchorRight[anchor]) {- if (item.snap) {- item.isDup = true;- duplicates.push(item);- }- item.snap = 'right';- item.rightAnchor = anchor;- }+ }++ for (const anchor in anchorCenter) {+ for (const item of anchorCenter[anchor]) {+ if (item.snap && !item.isDup) {+ item.isDup = true;+ duplicates.push(item);+ }+ item.snap = "center";+ item.centerAnchor = anchor;}+ }- for (let anchor in anchorCenter) {- for (let item of anchorCenter[anchor]) {- if (item.snap && !item.isDup) {- item.isDup = true;- duplicates.push(item);- }- item.snap = 'center';- item.centerAnchor = anchor;- }+ function anchorCounts(item: ProjectionTextBox): number[] {+ let leftCount = 0;+ if (item.leftAnchor) {+ const key = parseFloat(item.leftAnchor);+ leftCount = anchorLeft[key] ? anchorLeft[key].length : 0;}-- function anchorCounts(item: ProjectionTextBox): number[] {- let leftCount = 0;- if (item.leftAnchor) {- const key = parseFloat(item.leftAnchor);- leftCount = anchorLeft[key] ? anchorLeft[key].length : 0;- }- let rightCount = 0;+ let rightCount = 0;+ if (item.rightAnchor) {+ const key = parseFloat(item.rightAnchor);+ rightCount = anchorRight[key] ? anchorRight[key].length : 0;+ }+ let centerCount = 0;+ if (item.centerAnchor) {+ const key = parseFloat(item.centerAnchor);+ centerCount = anchorCenter[key] ? anchorCenter[key].length : 0;+ }+ return [leftCount, rightCount, centerCount];+ }++ // find all left aligned blocks, all right aligned blocks, all centered blocks, in that order+ // we cannot check all 3 at once since we may end up double counting potential anchor matches+ // (i.e. we need to exclude block that we know are left/right aligned before counting possible+ // matching centered blocks)++ // find all lefts+ let hasChanged = true;+ while (hasChanged && duplicates.length > 0) {+ hasChanged = false;+ for (let i = duplicates.length - 1; i >= 0; --i) {+ const item = duplicates[i];+ const [leftCount, rightCount, centerCount] = anchorCounts(item);+ if (leftCount >= rightCount && leftCount >= centerCount) {+ item.snap = "left";if (item.rightAnchor) {- const key = parseFloat(item.rightAnchor);- rightCount = anchorRight[key] ? anchorRight[key].length : 0;+ const key = parseFloat(item.rightAnchor);+ if (anchorRight[key]) {+ anchorRight[key].splice(anchorRight[key].indexOf(item), 1);+ hasChanged = true;+ }}- let centerCount = 0;if (item.centerAnchor) {- const key = parseFloat(item.centerAnchor);- centerCount = anchorCenter[key] ? anchorCenter[key].length : 0;- }- return [leftCount, rightCount, centerCount];- }-- // find all left aligned blocks, all right aligned blocks, all centered blocks, in that order- // we cannot check all 3 at once since we may end up double counting potential anchor matches- // (i.e. we need to exclude block that we know are left/right aligned before counting possible- // matching centered blocks)-- // find all lefts- let hasChanged = true;- while (hasChanged && duplicates.length > 0) {- hasChanged = false;- for (let i = duplicates.length - 1; i >= 0; --i) {- const item = duplicates[i];- const [leftCount, rightCount, centerCount] = anchorCounts(item);- if (leftCount >= rightCount && leftCount >= centerCount) {- item.snap = "left";- if (item.rightAnchor) {- const key = parseFloat(item.rightAnchor);- if (anchorRight[key]) {- anchorRight[key].splice(anchorRight[key].indexOf(item), 1);- hasChanged = true;- }- }- if (item.centerAnchor) {- const key = parseFloat(item.centerAnchor);- if (anchorCenter[key]) {- anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);- hasChanged = true;- }- }- duplicates.splice(i, 1);- }- }- }-- // find all rights- hasChanged = true;- while (hasChanged && duplicates.length > 0) {- hasChanged = false;- for (let i = duplicates.length - 1; i >= 0; --i) {- const item = duplicates[i];- const [leftCount, rightCount, centerCount] = anchorCounts(item);- if (rightCount >= leftCount && rightCount >= centerCount) {- item.snap = "right";- if (item.leftAnchor) {- const key = parseFloat(item.leftAnchor);- if (anchorLeft[key]) {- anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);- hasChanged = true;- }- }- if (item.centerAnchor) {- const key = parseFloat(item.centerAnchor);- if (anchorCenter[key]) {- anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);- hasChanged = true;- }- }- duplicates.splice(i, 1);- }+ const key = parseFloat(item.centerAnchor);+ if (anchorCenter[key]) {+ anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);+ hasChanged = true;+ }}+ duplicates.splice(i, 1);+ }}-- // remaining duplicates are centered- for (const item of duplicates) {- item.snap = "center";+ }++ // find all rights+ hasChanged = true;+ while (hasChanged && duplicates.length > 0) {+ hasChanged = false;+ for (let i = duplicates.length - 1; i >= 0; --i) {+ const item = duplicates[i];+ const [leftCount, rightCount, centerCount] = anchorCounts(item);+ if (rightCount >= leftCount && rightCount >= centerCount) {+ item.snap = "right";if (item.leftAnchor) {- const key = parseFloat(item.leftAnchor);- if (anchorLeft[key]) {- anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);- }+ const key = parseFloat(item.leftAnchor);+ if (anchorLeft[key]) {+ anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);+ hasChanged = true;+ }}- if (item.rightAnchor) {- const key = parseFloat(item.rightAnchor);- if (anchorRight[key]) {- anchorRight[key].splice(anchorRight[key].indexOf(item), 1);- }+ if (item.centerAnchor) {+ const key = parseFloat(item.centerAnchor);+ if (anchorCenter[key]) {+ anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);+ hasChanged = true;+ }}+ duplicates.splice(i, 1);+ }}-- // filter anchors- // delete singleton- for (let anchor in anchorLeft) {- if (anchorLeft[anchor].length < 2) {- if (anchorLeft[anchor].length) {- delete anchorLeft[anchor][0].snap;- }- delete anchorLeft[anchor];- }+ }++ // remaining duplicates are centered+ for (const item of duplicates) {+ item.snap = "center";+ if (item.leftAnchor) {+ const key = parseFloat(item.leftAnchor);+ if (anchorLeft[key]) {+ anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);+ }}-- for (let anchor in anchorRight) {- if (anchorRight[anchor].length < 2) {- if (anchorRight[anchor].length) {- delete anchorRight[anchor][0].snap;- }- delete anchorRight[anchor];- }+ if (item.rightAnchor) {+ const key = parseFloat(item.rightAnchor);+ if (anchorRight[key]) {+ anchorRight[key].splice(anchorRight[key].indexOf(item), 1);+ }}-- for (let anchor in anchorCenter) {- if (anchorCenter[anchor].length < 2) {- if (anchorCenter[anchor].length) {- delete anchorCenter[anchor][0].snap;- }- delete anchorCenter[anchor];- }+ }++ // filter anchors+ // delete singleton+ for (const anchor in anchorLeft) {+ if (anchorLeft[anchor].length < 2) {+ if (anchorLeft[anchor].length) {+ delete anchorLeft[anchor][0].snap;+ }+ delete anchorLeft[anchor];}-- return {- anchorLeft,- anchorRight,- anchorCenter+ }++ for (const anchor in anchorRight) {+ if (anchorRight[anchor].length < 2) {+ if (anchorRight[anchor].length) {+ delete anchorRight[anchor][0].snap;+ }+ delete anchorRight[anchor];}-}---function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: number) {- // if no bbox is rotated (.r is set), return- if (!textBbox.find(b => b.r != 0)) {- return;+ }++ for (const anchor in anchorCenter) {+ if (anchorCenter[anchor].length < 2) {+ if (anchorCenter[anchor].length) {+ delete anchorCenter[anchor][0].snap;+ }+ delete anchorCenter[anchor];}+ }- textBbox.sort((a, b) => {- return a.x - b.x;- });+ return {+ anchorLeft,+ anchorRight,+ anchorCenter,+ };+}- // Horizontal Axis- let bboxGroup: ProjectionTextBox[][] = [];- let currentGroup: ProjectionTextBox[] = [];- let previousBbox: ProjectionTextBox | null = null;- for (let bbox of textBbox) {- if (previousBbox) {- if (bbox.r == previousBbox.r) {- currentGroup.push(bbox);- }- else {- // sort currentGroup by bbox.y- currentGroup.sort((a, b) => a.y - b.y);- bboxGroup.push(currentGroup);- currentGroup = [bbox];- }- }- else {- currentGroup.push(bbox);- }- previousBbox = bbox;- }- if (currentGroup.length) {+function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: number) {+ // if no bbox is rotated (.r is set), return+ if (!textBbox.find((b) => b.r != 0)) {+ return;+ }++ textBbox.sort((a, b) => {+ return a.x - b.x;+ });++ // Horizontal Axis+ let bboxGroup: ProjectionTextBox[][] = [];+ let currentGroup: ProjectionTextBox[] = [];+ let previousBbox: ProjectionTextBox | null = null;+ for (const bbox of textBbox) {+ if (previousBbox) {+ if (bbox.r == previousBbox.r) {+ currentGroup.push(bbox);+ } else {+ // sort currentGroup by bbox.ycurrentGroup.sort((a, b) => a.y - b.y);bboxGroup.push(currentGroup);+ currentGroup = [bbox];+ }+ } else {+ currentGroup.push(bbox);}-- // NOTE/ WARNING: height and width of bbox are NOT rotated beforehand!- for (let [index, group] of bboxGroup.entries()) {- if (group[0].r == 90 || group[0].r == 270) {- // check in all bbox if they are not group[0].r and have overlap in x axis- let globalOverlap = false;- for (let bbox of textBbox) {- if (bbox.r != group[0].r) {- let overlap = group.find(b => b.x >= bbox.x && b.x <= bbox.x + bbox.w && bbox.r != b.r);- if (overlap) {- globalOverlap = true;- }- }- }-- if (globalOverlap) {- // rotate bbox to be horizontal- for (let bbox of group) {- if (bbox.d) {- bbox.y += bbox.d;- bbox.d = 0;- }- bbox.r = 0;- bbox.rotated = true;- }- }- else if (!globalOverlap) {- // insert the bbox group in the Y axis after previous group and before next group.- // move Next group by current group height (width as not rotated yet).- let groupMaxX = Math.max(...group.map(v => v.x + v.w));-- let deltaY = 0;- if (index != 0) {- let previousGroup = bboxGroup[index - 1];- let previousGroupMaxY = Math.max(...previousGroup.map(v => v.y + v.h));- // pageHeight is radical but garantie no issue of allignement- deltaY = previousGroupMaxY + pageHeight;- }- // clockwize rotation- if (group[0].r == 90) {- for (let bbox of group) {- // w not h here as bbox w/h are not flipped- let x = Math.round(bbox.ry ?? bbox.y);- let y = (bbox.rx ?? bbox.x) + deltaY;- bbox.x = Math.round(x);- bbox.y = y;- bbox.r = 0;- bbox.rotated = true;- }- }- // counter clockwize rotation- if (group[0].r == 270) {- for (let bbox of group) {- let x = Math.round(bbox.ry ?? bbox.y);- let y = (bbox.rx ?? bbox.x) + deltaY;- bbox.x = Math.round(x);- bbox.y = y;- bbox.r = 0;- bbox.rotated = true;- }- }-- // pageHeight is radical but garantie no issue of allignement- let globalDelta = deltaY + groupMaxX + pageHeight;-- for (let [otherGroupIndex, other] of bboxGroup.entries()) {- if (otherGroupIndex <= index) {- continue;- }- for (let bbox of other) {- if (bbox.r == 90 || bbox.r == 270) {- bbox.d = (bbox.d ? bbox.d : 0) + globalDelta;- continue;- }- bbox.y += globalDelta;- }- }- }-+ previousBbox = bbox;+ }+ if (currentGroup.length) {+ currentGroup.sort((a, b) => a.y - b.y);+ bboxGroup.push(currentGroup);+ }++ // NOTE/ WARNING: height and width of bbox are NOT rotated beforehand!+ for (const [index, group] of bboxGroup.entries()) {+ if (group[0].r == 90 || group[0].r == 270) {+ // check in all bbox if they are not group[0].r and have overlap in x axis+ let globalOverlap = false;+ for (const bbox of textBbox) {+ if (bbox.r != group[0].r) {+ const overlap = group.find(+ (b) => b.x >= bbox.x && b.x <= bbox.x + bbox.w && bbox.r != b.r+ );+ if (overlap) {+ globalOverlap = true;+ }+ }+ }++ if (globalOverlap) {+ // rotate bbox to be horizontal+ for (const bbox of group) {+ if (bbox.d) {+ bbox.y += bbox.d;+ bbox.d = 0;+ }+ bbox.r = 0;+ bbox.rotated = true;+ }+ } else if (!globalOverlap) {+ // insert the bbox group in the Y axis after previous group and before next group.+ // move Next group by current group height (width as not rotated yet).+ const groupMaxX = Math.max(...group.map((v) => v.x + v.w));++ let deltaY = 0;+ if (index != 0) {+ const previousGroup = bboxGroup[index - 1];+ const previousGroupMaxY = Math.max(...previousGroup.map((v) => v.y + v.h));+ // pageHeight is radical but garantie no issue of allignement+ deltaY = previousGroupMaxY + pageHeight;+ }+ // clockwize rotation+ if (group[0].r == 90) {+ for (const bbox of group) {+ // w not h here as bbox w/h are not flipped+ const x = Math.round(bbox.ry ?? bbox.y);+ const y = (bbox.rx ?? bbox.x) + deltaY;+ bbox.x = Math.round(x);+ bbox.y = y;+ bbox.r = 0;+ bbox.rotated = true;+ }+ }+ // counter clockwize rotation+ if (group[0].r == 270) {+ for (const bbox of group) {+ const x = Math.round(bbox.ry ?? bbox.y);+ const y = (bbox.rx ?? bbox.x) + deltaY;+ bbox.x = Math.round(x);+ bbox.y = y;+ bbox.r = 0;+ bbox.rotated = true;+ }}- }-- textBbox.sort((a, b) => {- return a.y - b.y;- });+ // pageHeight is radical but garantie no issue of allignement+ const globalDelta = deltaY + groupMaxX + pageHeight;- // Vertical Axis- bboxGroup = [];- currentGroup = [];- previousBbox = null as ProjectionTextBox | null;- for (let bbox of textBbox) {- if (previousBbox) {- if (bbox.r == previousBbox.r) {- currentGroup.push(bbox);- }- else {- // sort currentGroup by bbox.y- currentGroup.sort((a, b) => a.x - b.x);- bboxGroup.push(currentGroup);- currentGroup = [bbox];+ for (const [otherGroupIndex, other] of bboxGroup.entries()) {+ if (otherGroupIndex <= index) {+ continue;+ }+ for (const bbox of other) {+ if (bbox.r == 90 || bbox.r == 270) {+ bbox.d = (bbox.d ? bbox.d : 0) + globalDelta;+ continue;}+ bbox.y += globalDelta;+ }}- else {- currentGroup.push(bbox);- }- previousBbox = bbox;+ }}- if (currentGroup.length) {+ }++ textBbox.sort((a, b) => {+ return a.y - b.y;+ });++ // Vertical Axis+ bboxGroup = [];+ currentGroup = [];+ previousBbox = null as ProjectionTextBox | null;+ for (const bbox of textBbox) {+ if (previousBbox) {+ if (bbox.r == previousBbox.r) {+ currentGroup.push(bbox);+ } else {+ // sort currentGroup by bbox.ycurrentGroup.sort((a, b) => a.x - b.x);bboxGroup.push(currentGroup);+ currentGroup = [bbox];+ }+ } else {+ currentGroup.push(bbox);}--- for (const group of bboxGroup) {- if (group[0].r == 180) {- // Switch upside down- for (let bbox of group) {- bbox.x = Math.round(bbox.ry ?? bbox.y);- bbox.y = bbox.rx ?? bbox.x;- bbox.r = 0;- bbox.rotated = true;- }- }+ previousBbox = bbox;+ }+ if (currentGroup.length) {+ currentGroup.sort((a, b) => a.x - b.x);+ bboxGroup.push(currentGroup);+ }++ for (const group of bboxGroup) {+ if (group[0].r == 180) {+ // Switch upside down+ for (const bbox of group) {+ bbox.x = Math.round(bbox.ry ?? bbox.y);+ bbox.y = bbox.rx ?? bbox.x;+ bbox.r = 0;+ bbox.rotated = true;+ }}+ }}-export function bboxToLine(textBbox: ProjectionTextBox[], medianWidth: number, medianHeight: number): ProjectionTextBox[][] {- // sort lines on first y axis then x axis (top - left)- textBbox.sort((a, b) => {- if (a.y == b.y) {- return a.x - b.x;- }- return a.y - b.y;- });-- function canMergeMarkup(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {- if (!previousBbox.markup && !bbox.markup) {- return true;- }- if (previousBbox.markup && bbox.markup- && previousBbox.markup.highlight === bbox.markup.highlight- && previousBbox.markup.underline === bbox.markup.underline- && previousBbox.markup.squiggly === bbox.markup.squiggly- && previousBbox.markup.strikeout === bbox.markup.strikeout- ) {- return true;- }- return false;+export function bboxToLine(+ textBbox: ProjectionTextBox[],+ medianWidth: number,+ medianHeight: number+): ProjectionTextBox[][] {+ // sort lines on first y axis then x axis (top - left)+ textBbox.sort((a, b) => {+ if (a.y == b.y) {+ return a.x - b.x;}+ return a.y - b.y;+ });- function canMerge(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {- if (bbox.y == previousBbox.y && bbox.h == previousBbox.h) {- const xDelta = bbox.x - previousBbox.x - previousBbox.w;- if (((xDelta < 0 && xDelta > -.5) || (xDelta >= 0 && xDelta < .1)) && canMergeMarkup(previousBbox, bbox)) {- return true;- }- }- return false;+ function canMergeMarkup(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {+ if (!previousBbox.markup && !bbox.markup) {+ return true;}-- function mergePageBbox(a: ProjectionTextBox, b: ProjectionTextBox): Coordinates {- const aBbox = a.pageBbox || {x: a.x, y: a.y, w: a.w, h: a.h};- const bBbox = b.pageBbox || {x: b.x, y: b.y, w: b.w, h: b.h};- const left = Math.min(aBbox.x, bBbox.x);- const top = Math.min(aBbox.y, bBbox.y);- const right = Math.max(aBbox.x + aBbox.w, bBbox.x + bBbox.w);- const bottom = Math.max(aBbox.y + aBbox.h, bBbox.y + bBbox.h);- return {x: left, y: top, w: right - left, h: bottom - top};+ if (+ previousBbox.markup &&+ bbox.markup &&+ previousBbox.markup.highlight === bbox.markup.highlight &&+ previousBbox.markup.underline === bbox.markup.underline &&+ previousBbox.markup.squiggly === bbox.markup.squiggly &&+ previousBbox.markup.strikeout === bbox.markup.strikeout+ ) {+ return true;}-- // merge Continuous bbox- for (let i = 1; i < textBbox.length; i++) {- let bbox = textBbox[i];- let previousBbox = textBbox[i - 1];- if (canMerge(previousBbox, bbox)) {- previousBbox.w = bbox.x + bbox.w - previousBbox.x;- previousBbox.str += bbox.str;- previousBbox.strLength += bbox.strLength;- previousBbox.pageBbox = mergePageBbox(previousBbox, bbox);- textBbox.splice(i, 1);- i--;- }+ return false;+ }++ function canMerge(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {+ if (bbox.y == previousBbox.y && bbox.h == previousBbox.h) {+ const xDelta = bbox.x - previousBbox.x - previousBbox.w;+ if (+ ((xDelta < 0 && xDelta > -0.5) || (xDelta >= 0 && xDelta < 0.1)) &&+ canMergeMarkup(previousBbox, bbox)+ ) {+ return true;+ }}-- // try to find the bounding box that align as line and group them by line- let lines: ProjectionTextBox[][] = [];- let currentLine: ProjectionTextBox[] = [];- let previousBbox = null;-- for (let bbox of textBbox) {- if (!previousBbox) {- currentLine.push(bbox);- }- // This is where we define how line are build. to be improved- else {- let lineMinY = Math.min(...currentLine.map(v => v.y));- let lineMaxY = Math.max(...currentLine.map(v => v.y + v.h));-- let lineCollide = false;- for (let currentLineItemBbox of currentLine) {-- let overlapLenght = Math.min(currentLineItemBbox.x + currentLineItemBbox.w, bbox.x + bbox.w) - Math.max(currentLineItemBbox.x, bbox.x);- if (overlapLenght > medianWidth / 3) {- lineCollide = true;- break;- }- }- if (!lineCollide- && (- (bbox.y + bbox.h * .5 >= lineMinY && bbox.y + bbox.h * .5 <= lineMaxY)- ||- (bbox.y >= lineMinY && bbox.y <= lineMaxY)- )- ) {- currentLine.push(bbox);- }- else {- if (currentLine.length) {- lines.push(currentLine);- }- currentLine = [bbox];- }- }- previousBbox = bbox;+ return false;+ }++ function mergePageBbox(a: ProjectionTextBox, b: ProjectionTextBox): Coordinates {+ const aBbox = a.pageBbox || { x: a.x, y: a.y, w: a.w, h: a.h };+ const bBbox = b.pageBbox || { x: b.x, y: b.y, w: b.w, h: b.h };+ const left = Math.min(aBbox.x, bBbox.x);+ const top = Math.min(aBbox.y, bBbox.y);+ const right = Math.max(aBbox.x + aBbox.w, bBbox.x + bBbox.w);+ const bottom = Math.max(aBbox.y + aBbox.h, bBbox.y + bBbox.h);+ return { x: left, y: top, w: right - left, h: bottom - top };+ }++ // merge Continuous bbox+ for (let i = 1; i < textBbox.length; i++) {+ const bbox = textBbox[i];+ const previousBbox = textBbox[i - 1];+ if (canMerge(previousBbox, bbox)) {+ previousBbox.w = bbox.x + bbox.w - previousBbox.x;+ previousBbox.str += bbox.str;+ previousBbox.strLength += bbox.strLength;+ previousBbox.pageBbox = mergePageBbox(previousBbox, bbox);+ textBbox.splice(i, 1);+ i--;}+ }- if (currentLine.length) {- lines.push(currentLine);- }+ // try to find the bounding box that align as line and group them by line+ const lines: ProjectionTextBox[][] = [];+ let currentLine: ProjectionTextBox[] = [];+ let previousBbox = null;- // sort each line by x- for (let line of lines) {- line.sort((a, b) => a.x - b.x);+ for (const bbox of textBbox) {+ if (!previousBbox) {+ currentLine.push(bbox);}-- // sort lines by y- lines.sort((a, b) => a[0].y - b[0].y);- // merge 'words'- let mergeThreshold = 1;- for (let line of lines) {- for (let i = 1; i < line.length; ++i) {- // merge box in word if:- // - same height- // - less than 2 in space- // if (line[i].h == line[i-1].h) {- let currentLine = line[i];- let previousLine = line[i - 1];- if (canMergeMarkup(previousLine, currentLine)) {- if (currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {- // if same word but less than .7 of prev line- if (currentLine.h != 0 && currentLine.h < previousLine.h * .7) {- // and not starting with space- if (currentLine.str[0] == ' ') {- break;- }- if (currentLine.y > previousLine.y + previousLine.h * .2) {- currentLine.str = strToSubscriptString(currentLine.str);- }- else {- currentLine.str = strToPostScript(currentLine.str);- }- }- previousLine.w = currentLine.x + currentLine.w - previousLine.x;- previousLine.str += currentLine.str;- previousLine.strLength += currentLine.strLength;- previousLine.pageBbox = mergePageBbox(previousLine, currentLine);- line.splice(i, 1);- i--;- }- else if (currentLine.x - previousLine.x - previousLine.w < previousLine.w / previousLine.strLength) {- // merge if space between this word and previous is less than average- // character width (using previous word font size)- previousLine.w = currentLine.x + currentLine.w - previousLine.x;- if (!previousLine.str.endsWith(" ")) {- previousLine.str += " ";- previousLine.strLength += 1;- }- previousLine.str += currentLine.str;- previousLine.strLength += currentLine.strLength;- previousLine.pageBbox = mergePageBbox(previousLine, currentLine);- line.splice(i, 1);- i--;-- }- }- // }+ // This is where we define how line are build. to be improved+ else {+ const lineMinY = Math.min(...currentLine.map((v) => v.y));+ const lineMaxY = Math.max(...currentLine.map((v) => v.y + v.h));++ let lineCollide = false;+ for (const currentLineItemBbox of currentLine) {+ const overlapLenght =+ Math.min(currentLineItemBbox.x + currentLineItemBbox.w, bbox.x + bbox.w) -+ Math.max(currentLineItemBbox.x, bbox.x);+ if (overlapLenght > medianWidth / 3) {+ lineCollide = true;+ break;}+ }+ if (+ !lineCollide &&+ ((bbox.y + bbox.h * 0.5 >= lineMinY && bbox.y + bbox.h * 0.5 <= lineMaxY) ||+ (bbox.y >= lineMinY && bbox.y <= lineMaxY))+ ) {+ currentLine.push(bbox);+ } else {+ if (currentLine.length) {+ lines.push(currentLine);+ }+ currentLine = [bbox];+ }}--- // check if we can merge the lines together- for (let i = 1; i < lines.length - 1; i++) {- let currentLine = lines[i];- let previousLine = lines[i - 1];-- let previousLineMinY = Math.min(...previousLine.map(v => v.y));- let previousLineMaxY = Math.max(...previousLine.map(v => v.y + v.h));- let currentLineMinY = Math.min(...currentLine.map(v => v.y));- let currentLineMaxY = Math.max(...currentLine.map(v => v.y + v.h));-- // does the 2 line overlap?- if (previousLineMaxY > currentLineMinY && previousLineMinY < currentLineMaxY) {- // check the bboxes of current line and prevline do not overlap- let bboxOverlap = false;- for (let bbox of currentLine) {- for (let prevBbox of previousLine) {- if (bbox.x >= prevBbox.x && bbox.x <= prevBbox.x + prevBbox.w) {- bboxOverlap = true;- break;- }- if (prevBbox.x >= bbox.x && prevBbox.x <= bbox.x + bbox.w) {- bboxOverlap = true;- break;- }- }+ previousBbox = bbox;+ }++ if (currentLine.length) {+ lines.push(currentLine);+ }++ // sort each line by x+ for (const line of lines) {+ line.sort((a, b) => a.x - b.x);+ }++ // sort lines by y+ lines.sort((a, b) => a[0].y - b[0].y);+ // merge 'words'+ const mergeThreshold = 1;+ for (const line of lines) {+ for (let i = 1; i < line.length; ++i) {+ // merge box in word if:+ // - same height+ // - less than 2 in space+ // if (line[i].h == line[i-1].h) {+ const currentLine = line[i];+ const previousLine = line[i - 1];+ if (canMergeMarkup(previousLine, currentLine)) {+ if (currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {+ // if same word but less than .7 of prev line+ if (currentLine.h != 0 && currentLine.h < previousLine.h * 0.7) {+ // and not starting with space+ if (currentLine.str[0] == " ") {+ break;}-- // merge if no overlap- if (!bboxOverlap) {- previousLine.push(...currentLine);- previousLine.sort((a, b) => a.x - b.x);- lines.splice(i--, 1);+ if (currentLine.y > previousLine.y + previousLine.h * 0.2) {+ currentLine.str = strToSubscriptString(currentLine.str);+ } else {+ currentLine.str = strToPostScript(currentLine.str);}+ }+ previousLine.w = currentLine.x + currentLine.w - previousLine.x;+ previousLine.str += currentLine.str;+ previousLine.strLength += currentLine.strLength;+ previousLine.pageBbox = mergePageBbox(previousLine, currentLine);+ line.splice(i, 1);+ i--;+ } else if (+ currentLine.x - previousLine.x - previousLine.w <+ previousLine.w / previousLine.strLength+ ) {+ // merge if space between this word and previous is less than average+ // character width (using previous word font size)+ previousLine.w = currentLine.x + currentLine.w - previousLine.x;+ if (!previousLine.str.endsWith(" ")) {+ previousLine.str += " ";+ previousLine.strLength += 1;+ }+ previousLine.str += currentLine.str;+ previousLine.strLength += currentLine.strLength;+ previousLine.pageBbox = mergePageBbox(previousLine, currentLine);+ line.splice(i, 1);+ i--;}+ }+ // }}-- for (let i = 1; i < lines.length; i++) {- const yDelta = lines[i][0].y - lines[i - 1][0].y - lines[i - 1][0].h;- // Calculate the number of blank lines to insert based on vertical spacing- // Use medianHeight as a reference for one line spacing- if (yDelta > medianHeight) {- // Calculate how many blank lines should be inserted- // Round to nearest integer to get approximate number of lines- const numBlankLines = Math.round(yDelta / medianHeight) - 1;- // Cap at a reasonable maximum (e.g., 10 blank lines) to avoid extreme cases- const linesToInsert = Math.min(Math.max(numBlankLines, 1), 10);-- // Insert the calculated number of blank lines- const blankLines = Array(linesToInsert).fill([]);- lines.splice(i, 0, ...blankLines);- i += linesToInsert;+ }++ // check if we can merge the lines together+ for (let i = 1; i < lines.length - 1; i++) {+ const currentLine = lines[i];+ const previousLine = lines[i - 1];++ const previousLineMinY = Math.min(...previousLine.map((v) => v.y));+ const previousLineMaxY = Math.max(...previousLine.map((v) => v.y + v.h));+ const currentLineMinY = Math.min(...currentLine.map((v) => v.y));+ const currentLineMaxY = Math.max(...currentLine.map((v) => v.y + v.h));++ // does the 2 line overlap?+ if (previousLineMaxY > currentLineMinY && previousLineMinY < currentLineMaxY) {+ // check the bboxes of current line and prevline do not overlap+ let bboxOverlap = false;+ for (const bbox of currentLine) {+ for (const prevBbox of previousLine) {+ if (bbox.x >= prevBbox.x && bbox.x <= prevBbox.x + prevBbox.w) {+ bboxOverlap = true;+ break;+ }+ if (prevBbox.x >= bbox.x && prevBbox.x <= bbox.x + bbox.w) {+ bboxOverlap = true;+ break;+ }}+ }++ // merge if no overlap+ if (!bboxOverlap) {+ previousLine.push(...currentLine);+ previousLine.sort((a, b) => a.x - b.x);+ lines.splice(i--, 1);+ }}- return lines;+ }++ for (let i = 1; i < lines.length; i++) {+ const yDelta = lines[i][0].y - lines[i - 1][0].y - lines[i - 1][0].h;+ // Calculate the number of blank lines to insert based on vertical spacing+ // Use medianHeight as a reference for one line spacing+ if (yDelta > medianHeight) {+ // Calculate how many blank lines should be inserted+ // Round to nearest integer to get approximate number of lines+ const numBlankLines = Math.round(yDelta / medianHeight) - 1;+ // Cap at a reasonable maximum (e.g., 10 blank lines) to avoid extreme cases+ const linesToInsert = Math.min(Math.max(numBlankLines, 1), 10);++ // Insert the calculated number of blank lines+ const blankLines = Array(linesToInsert).fill([]);+ lines.splice(i, 0, ...blankLines);+ i += linesToInsert;+ }+ }+ return lines;}function canRenderBbox(line: ProjectionTextBox[], bbox: ProjectionTextBox): boolean {- for (let item of line) {- if (item == bbox) {- return true;- }- if (!item.rendered) {- return false;- }+ for (const item of line) {+ if (item == bbox) {+ return true;}- return false;+ if (!item.rendered) {+ return false;+ }+ }+ return false;}-function updateForwardAnchorRightBound(snapMap: number[], forwardAnchor: ForwardAnchor, rightBound: number, anchorTarget: number): void {- // Anything snapped to the right of rightBound should be aligned to anchorTarget line length at minimum- for (let i = snapMap.length - 1; i >= 0; --i) {- const anchor = snapMap[i];- if (rightBound <= anchor) {- if (!forwardAnchor[anchor] || anchorTarget > forwardAnchor[anchor]) {- forwardAnchor[anchor] = anchorTarget;- }- }- else {- return;- }+function updateForwardAnchorRightBound(+ snapMap: number[],+ forwardAnchor: ForwardAnchor,+ rightBound: number,+ anchorTarget: number+): void {+ // Anything snapped to the right of rightBound should be aligned to anchorTarget line length at minimum+ for (let i = snapMap.length - 1; i >= 0; --i) {+ const anchor = snapMap[i];+ if (rightBound <= anchor) {+ if (!forwardAnchor[anchor] || anchorTarget > forwardAnchor[anchor]) {+ forwardAnchor[anchor] = anchorTarget;+ }+ } else {+ return;}+ }}-function updateForwardAnchors(bbox: ProjectionTextBox, nextBbox: ProjectionTextBox | null, snapMaps: SnapMaps, forwardAnchors: PageForwardAnchors, lineLength: number): void {- const rightBound = bbox.x + bbox.w;- let targetLength = lineLength;- if (nextBbox && (nextBbox.shouldSpace ?? 0) > 0) {- targetLength += (nextBbox.shouldSpace ?? 0);- }- updateForwardAnchorRightBound(snapMaps.left, forwardAnchors.left, rightBound, targetLength);- updateForwardAnchorRightBound(snapMaps.right, forwardAnchors.right, rightBound, targetLength);- // we do not update center anchors since centered text may span between snapped columns- updateForwardAnchorRightBound(snapMaps.floating, forwardAnchors.floating, rightBound, targetLength);+function updateForwardAnchors(+ bbox: ProjectionTextBox,+ nextBbox: ProjectionTextBox | null,+ snapMaps: SnapMaps,+ forwardAnchors: PageForwardAnchors,+ lineLength: number+): void {+ const rightBound = bbox.x + bbox.w;+ let targetLength = lineLength;+ if (nextBbox && (nextBbox.shouldSpace ?? 0) > 0) {+ targetLength += nextBbox.shouldSpace ?? 0;+ }+ updateForwardAnchorRightBound(snapMaps.left, forwardAnchors.left, rightBound, targetLength);+ updateForwardAnchorRightBound(snapMaps.right, forwardAnchors.right, rightBound, targetLength);+ // we do not update center anchors since centered text may span between snapped columns+ updateForwardAnchorRightBound(+ snapMaps.floating,+ forwardAnchors.floating,+ rightBound,+ targetLength+ );}function resetBlockForwardAnchors(forwardAnchors: PageForwardAnchors): void {- forwardAnchors.left = {};- forwardAnchors.right = {};- forwardAnchors.center = {};- forwardAnchors.floating = {};+ forwardAnchors.left = {};+ forwardAnchors.right = {};+ forwardAnchors.center = {};+ forwardAnchors.floating = {};}-function getMedianTextBoxSize (lines: ProjectionTextBox[]): TextBoxSize {- // calculate median textBox width- let widthList = [];- for (let bbox of lines) {- if (bbox.w > 0) {- widthList.push(bbox.w / bbox.strLength);- }+function getMedianTextBoxSize(lines: ProjectionTextBox[]): TextBoxSize {+ // calculate median textBox width+ const widthList = [];+ for (const bbox of lines) {+ if (bbox.w > 0) {+ widthList.push(bbox.w / bbox.strLength);}- const medianWidth = widthList.sort((a, b) => a - b)[Math.floor(widthList.length / 2)];-- // calculate median textBox height- let heightList = [];- for (let bbox of lines) {- if (bbox.h > 0) {- heightList.push(bbox.h);- }+ }+ const medianWidth = widthList.sort((a, b) => a - b)[Math.floor(widthList.length / 2)];++ // calculate median textBox height+ const heightList = [];+ for (const bbox of lines) {+ if (bbox.h > 0) {+ heightList.push(bbox.h);}- const medianHeight = heightList.sort((a, b) => a - b)[Math.floor(heightList.length / 2)];- return {width: medianWidth, height: medianHeight};+ }+ const medianHeight = heightList.sort((a, b) => a - b)[Math.floor(heightList.length / 2)];+ return { width: medianWidth, height: medianHeight };}-export function projectToGrid(config: LiteParseConfig, page: PageData, projectionBoxes: ProjectionTextBox[], prevAnchors: PrevAnchors, totalPages: number): { text: string, prevAnchors: PrevAnchors } {-- // detect '.' garbage in the lines- let dotCount = 0;- for (let bbox of projectionBoxes) {- // check if bbox.str contains only dots- if (bbox.str.match(/^\.+$/)) {- dotCount++;+export function projectToGrid(+ config: LiteParseConfig,+ page: PageData,+ projectionBoxes: ProjectionTextBox[],+ prevAnchors: PrevAnchors,+ totalPages: number+): { text: string; prevAnchors: PrevAnchors } {+ // detect '.' garbage in the lines+ let dotCount = 0;+ for (const bbox of projectionBoxes) {+ // check if bbox.str contains only dots+ if (bbox.str.match(/^\.+$/)) {+ dotCount++;+ }+ }++ if (dotCount > 100 && dotCount > projectionBoxes.length * 0.05) {+ // remove all dots and splice them from lines+ const newLines = [];+ for (const bbox of projectionBoxes) {+ if (bbox.str.match(/^\.+$/)) {+ continue;+ }+ if (bbox.str.match(/^·+$/)) {+ continue;+ }+ if (bbox.str.match(/^"+$/)) {+ continue;+ }+ newLines.push(bbox);+ }+ projectionBoxes = newLines;+ }++ // calculate median textBox width/height+ const pageMedianSizes = getMedianTextBoxSize(projectionBoxes);+ let medianWidth = pageMedianSizes.width;+ let medianHeight = pageMedianSizes.height;++ let { verticalLines, horizontalLines } = getVerticalAndHorizontalLinesFromPagePaths(+ config,+ page.paths+ );+ // filter vertical and horizontal lines are less than 80% of page.width / height+ verticalLines = verticalLines.filter((line) => line.height > page.height * 0.8);+ horizontalLines = horizontalLines.filter((line) => line.width > page.width * 0.8);++ // Save original bboxes (including OCR) for text attribution+ const attributionBboxes: ProjectionTextBox[] = [];+ for (const bbox of projectionBoxes as ProjectionTextBox[]) {+ if (!bbox || !bbox.str || bbox.vgap || bbox.isPlaceholder) {+ continue;+ }+ attributionBboxes.push({+ str: bbox.str,+ x: bbox.x,+ y: bbox.y,+ w: bbox.w,+ h: bbox.h,+ r: bbox.r,+ strLength: bbox.str.length,+ });+ }++ handleRotationReadingOrder(projectionBoxes, page.height);+ const lines = bboxToLine(projectionBoxes, medianWidth, medianHeight);++ // remove unprojectable text and apply markup to final lines+ for (let i = 0; i < lines.length; ++i) {+ const line = filterUnprojectableText(config, lines[i]);+ for (const bbox of line) {+ // With the way our grid projection currently works, we have to output+ // tags before raw line projection to avoid breaking the projection alignment.+ // The tags get replaced with MD as needed in output formatting, this does+ // result in output text containing the ~~ strikeout markup, but this is+ // mitigated since we skip markup entirely when we are not outputting markdown+ if (bbox.str.trim().length != 0 && bbox.markup) {+ bbox.str = applyMarkupTags(bbox.markup, bbox.str);+ }+ }+ lines[i] = line;+ }++ const forwardAnchors: PageForwardAnchors = {+ left: {},+ right: {},+ center: {},+ floating: {},+ };++ const rawLines: string[] = [];+ const rawLinesDelta = [];++ const blocks: LineRange[] = [];+ if (config.preserveLayoutAlignmentAcrossPages && totalPages > 1) {+ blocks.push({ start: 0, end: lines.length });+ } else {+ let emptyCount = 0;+ let start = -1;+ for (const [lineIndex, line] of lines.entries()) {+ if (line.length === 0) {+ emptyCount++;+ if (emptyCount > 1) {+ if (start >= 0) {+ // ignore completely empty blocks, include the double blank+ // line at the end of valid blocks+ blocks.push({ start: start, end: lineIndex + 1 });+ }+ start = -1;+ }+ } else {+ emptyCount = 0;+ if (start < 0) {+ start = lineIndex;}+ }}+ if (start > -1) {+ blocks.push({ start: start, end: lines.length });+ }+ }++ for (const block of blocks) {+ const { anchorLeft, anchorRight, anchorCenter } = extractAnchorsPointsFromLines(+ lines.slice(block.start, block.end),+ page,+ verticalLines+ );++ const snapMaps: SnapMaps = {+ left: [],+ right: [],+ center: [],+ floating: [],+ };- if (dotCount > 100 && dotCount > projectionBoxes.length * 0.05) {- // remove all dots and splice them from lines- let newLines = [];- for (let bbox of projectionBoxes) {- if (bbox.str.match(/^\.+$/)) {- continue;- }- if (bbox.str.match(/^·+$/)) {- continue;- }- if (bbox.str.match(/^"+$/)) {- continue;- }- newLines.push(bbox);- }- projectionBoxes = newLines;+ const uniqueSnaps = new Set<number>();+ for (const snap in anchorLeft) {+ uniqueSnaps.add(parseFloat(snap));}+ snapMaps.left.push(...uniqueSnaps);+ uniqueSnaps.clear();- // calculate median textBox width/height- const pageMedianSizes = getMedianTextBoxSize(projectionBoxes);- let medianWidth = pageMedianSizes.width;- let medianHeight = pageMedianSizes.height;-- let { verticalLines, horizontalLines } = getVerticalAndHorizontalLinesFromPagePaths(config, page.paths);- // filter vertical and horizontal lines are less than 80% of page.width / height- verticalLines = verticalLines.filter(line => line.height > page.height * 0.8);- horizontalLines = horizontalLines.filter(line => line.width > page.width * 0.8);-- // Save original bboxes (including OCR) for text attribution- const attributionBboxes: ProjectionTextBox[] = []- for (const bbox of (projectionBoxes as ProjectionTextBox[])) {- if (!bbox || !bbox.str || bbox.vgap || bbox.isPlaceholder) {- continue;- }- attributionBboxes.push({- str: bbox.str,- x: bbox.x,- y: bbox.y,- w: bbox.w,- h: bbox.h,- r: bbox.r,- strLength: bbox.str.length- });+ for (const snap in anchorRight) {+ uniqueSnaps.add(parseFloat(snap));}+ snapMaps.right.push(...uniqueSnaps);+ uniqueSnaps.clear();- handleRotationReadingOrder(projectionBoxes, page.height);- let lines = bboxToLine(projectionBoxes, medianWidth, medianHeight);-- // remove unprojectable text and apply markup to final lines- for (let i = 0; i < lines.length; ++i) {- const line = filterUnprojectableText(config, lines[i]);- for (const bbox of line) {- // With the way our grid projection currently works, we have to output- // tags before raw line projection to avoid breaking the projection alignment.- // The tags get replaced with MD as needed in output formatting, this does- // result in output text containing the ~~ strikeout markup, but this is- // mitigated since we skip markup entirely when we are not outputting markdown- if (bbox.str.trim().length != 0 && bbox.markup) {- bbox.str = applyMarkupTags(bbox.markup, bbox.str);- }- }- lines[i] = line;+ for (const snap in anchorCenter) {+ uniqueSnaps.add(parseFloat(snap));}+ snapMaps.center.push(...uniqueSnaps);+ uniqueSnaps.clear();- const forwardAnchors: PageForwardAnchors = {- left: {},- right: {},- center: {},- floating: {},- };+ let hasChanged = true;+ const leftSnap: Snap[] = [];+ const rightSnap: Snap[] = [];+ const centerSnap: Snap[] = [];- let rawLines: string[] = [];- let rawLinesDelta = [];+ if (!config.preserveLayoutAlignmentAcrossPages) {+ resetBlockForwardAnchors(forwardAnchors);- const blocks: LineRange[] = [];- if (config.preserveLayoutAlignmentAcrossPages && totalPages > 1) {- blocks.push({start: 0, end: lines.length});- }- else {- let emptyCount = 0;- let start = -1;- for (const [lineIndex, line] of lines.entries()) {- if (line.length === 0) {- emptyCount++;- if (emptyCount > 1) {- if (start >= 0) {- // ignore completely empty blocks, include the double blank- // line at the end of valid blocks- blocks.push({start: start, end: lineIndex + 1});- }- start = -1;- }- }- else {- emptyCount = 0;- if (start < 0) {- start = lineIndex;- }- }- }- if (start > -1) {- blocks.push({start: start, end: lines.length});- }+ const sizes = getMedianTextBoxSize(lines.slice(block.start, block.end).flat());+ medianWidth = sizes.width;+ medianHeight = sizes.height;}- for (const block of blocks) {- let {- anchorLeft,- anchorRight,- anchorCenter- } = extractAnchorsPointsFromLines(lines.slice(block.start, block.end), page, verticalLines);-- const snapMaps: SnapMaps = {- left: [],- right: [],- center: [],- floating: [],- }-- const uniqueSnaps = new Set<number>();- for (let snap in anchorLeft) {- uniqueSnaps.add(parseFloat(snap));+ // compute snaps+ for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {+ const line = lines[lineIndex];+ const forceUnsnapped = !canSnapLine(config, line);+ let prevBbox = null;++ for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {+ const bbox = line[boxIndex];+ bbox.forceUnsnapped = forceUnsnapped;++ const spaceThreshold = 2;+ // should we add a space between the two bbox?+ // TODO RTL+ if (prevBbox && bbox.x - (prevBbox.x + prevBbox.w) > spaceThreshold) {+ // add a space+ bbox.shouldSpace = 1;+ const xDelta = bbox.x - (prevBbox.x + prevBbox.w);+ const prevCharWidth = prevBbox.w / prevBbox.strLength;+ if (xDelta > prevCharWidth * 2) {+ // insert column spacing if any of:+ // - gap is more than an approximate tab (8x average char width)+ // - previous bbox is right snap+ // - this bbox is left snap+ // - both previous and this bbox are snaps+ // otherwise insert floating spacing+ if (+ (!bbox.forceUnsnapped && xDelta > prevCharWidth * 8) ||+ (bbox.snap && bbox.snap === "left") ||+ (prevBbox.snap && prevBbox.snap === "right") ||+ (bbox.snap && prevBbox.snap)+ ) {+ bbox.shouldSpace = COLUMN_SPACES;+ } else {+ bbox.shouldSpace = FLOATING_SPACES;+ }+ }+ } else {+ bbox.shouldSpace = 0;}- snapMaps.left.push(...uniqueSnaps);- uniqueSnaps.clear()- for (let snap in anchorRight) {- uniqueSnaps.add(parseFloat(snap));+ prevBbox = bbox;+ if (!bbox.snap) {+ uniqueSnaps.add(Math.round(bbox.x));+ } else if (bbox.snap == "left") {+ leftSnap.push({ bbox, lineIndex, boxIndex });+ } else if (bbox.snap == "right") {+ rightSnap.push({ bbox, lineIndex, boxIndex });+ } else if (bbox.snap == "center") {+ centerSnap.push({ bbox, lineIndex, boxIndex });}- snapMaps.right.push(...uniqueSnaps);- uniqueSnaps.clear()+ }+ }- for (let snap in anchorCenter) {- uniqueSnaps.add(parseFloat(snap));- }- snapMaps.center.push(...uniqueSnaps);- uniqueSnaps.clear()+ snapMaps.floating.push(...uniqueSnaps);+ uniqueSnaps.clear();- let hasChanged = true;- let leftSnap: Snap[] = [];- let rightSnap: Snap[] = [];- let centerSnap: Snap[] = [];+ snapMaps.floating.sort((a, b) => a - b);+ snapMaps.center.sort((a, b) => a - b);+ snapMaps.right.sort((a, b) => a - b);+ snapMaps.left.sort((a, b) => a - b);- if (!config.preserveLayoutAlignmentAcrossPages) {- resetBlockForwardAnchors(forwardAnchors);+ while (hasChanged || snapMaps.right.length || snapMaps.left.length || snapMaps.center.length) {+ hasChanged = false;- const sizes = getMedianTextBoxSize(lines.slice(block.start, block.end).flat());- medianWidth = sizes.width;- medianHeight = sizes.height;+ for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {+ const line = lines[lineIndex];+ if (!rawLines[lineIndex]) {+ rawLines[lineIndex] = "";+ rawLinesDelta[lineIndex] = 0;}- // compute snaps- for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {- const line = lines[lineIndex];- const forceUnsnapped = !canSnapLine(config, line);- let prevBbox = null;-- for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {- const bbox = line[boxIndex];- bbox.forceUnsnapped = forceUnsnapped;-- const spaceThreshold = 2;- // should we add a space between the two bbox?- // TODO RTL- if (prevBbox && bbox.x - (prevBbox.x + prevBbox.w) > spaceThreshold) {- // add a space- bbox.shouldSpace = 1;- const xDelta = bbox.x - (prevBbox.x + prevBbox.w);- const prevCharWidth = prevBbox.w / prevBbox.strLength;- if (xDelta > prevCharWidth * 2) {- // insert column spacing if any of:- // - gap is more than an approximate tab (8x average char width)- // - previous bbox is right snap- // - this bbox is left snap- // - both previous and this bbox are snaps- // otherwise insert floating spacing- if (!bbox.forceUnsnapped && (xDelta > prevCharWidth * 8) || (bbox.snap && bbox.snap === "left") || (prevBbox.snap && prevBbox.snap === "right") || (bbox.snap && prevBbox.snap)) {- bbox.shouldSpace = COLUMN_SPACES;- }- else {- bbox.shouldSpace = FLOATING_SPACES;- }- }- }- else {- bbox.shouldSpace = 0;- }+ for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {+ const bbox = line[boxIndex];+ if (bbox.rendered) {+ continue;+ }- prevBbox = bbox;- if (!bbox.snap) {- uniqueSnaps.add(Math.round(bbox.x));- }- else if (bbox.snap == 'left') {- leftSnap.push({ bbox, lineIndex, boxIndex });- }- else if (bbox.snap == 'right') {- rightSnap.push({ bbox, lineIndex, boxIndex });- }- else if (bbox.snap == 'center') {- centerSnap.push({ bbox, lineIndex, boxIndex });- }+ if (!bbox.forceUnsnapped) {+ if (bbox.snap) {+ continue;}- }- snapMaps.floating.push(...uniqueSnaps);- uniqueSnaps.clear()-- snapMaps.floating.sort((a, b) => a - b);- snapMaps.center.sort((a, b) => a - b);- snapMaps.right.sort((a, b) => a - b);- snapMaps.left.sort((a, b) => a - b);-- while (hasChanged- || snapMaps.right.length- || snapMaps.left.length- || snapMaps.center.length) {- hasChanged = false;-- for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {- const line = lines[lineIndex];- if (!rawLines[lineIndex]) {- rawLines[lineIndex] = "";- rawLinesDelta[lineIndex] = 0;- }-- for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {- const bbox = line[boxIndex];- if (bbox.rendered) {- continue;- }-- if (!bbox.forceUnsnapped) {- if (bbox.snap) {- continue;- }-- if ((snapMaps.left.length && snapMaps.left[0] < bbox.x)- || (snapMaps.right.length && snapMaps.right[0] < bbox.x)- || (snapMaps.center.length && snapMaps.center[0] < Math.round(bbox.x + bbox.w / 2))) {- continue;- }- }-- if (!canRenderBbox(line, bbox)) {- break;- }-- let targetX = Math.min(Math.round(bbox.x / medianWidth), COLUMN_SPACES);-- let lastSnapLeft = 0;- for (let key in forwardAnchors.left) {- if (parseInt(key) <= bbox.x) {- lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);- }- }- const lineMax = Math.max(lastSnapLeft, rawLines[lineIndex].trimEnd().length + (bbox.shouldSpace ?? 0));- if (targetX < lineMax) {- targetX = lineMax;- }-- if (!bbox.forceUnsnapped) {- const floatingAnchor = forwardAnchors.floating[Math.round(bbox.x)];- if (floatingAnchor && targetX < floatingAnchor) {- targetX = floatingAnchor;- }- }-- rawLines[lineIndex] = rawLines[lineIndex].trimEnd();- if (targetX > rawLines[lineIndex].length) {- rawLines[lineIndex] += ' '.repeat(targetX - rawLines[lineIndex].length);- }-- rawLines[lineIndex] += bbox.str;-- bbox.rendered = true;- hasChanged = true;-- let nextBbox: ProjectionTextBox | null = null;- if (line.length > (boxIndex + 1)) {- nextBbox = line[boxIndex + 1];- }- if (!bbox.forceUnsnapped) {- updateForwardAnchors(bbox, nextBbox, snapMaps, forwardAnchors, rawLines[lineIndex].length);- }- }+ if (+ (snapMaps.left.length && snapMaps.left[0] < bbox.x) ||+ (snapMaps.right.length && snapMaps.right[0] < bbox.x) ||+ (snapMaps.center.length && snapMaps.center[0] < Math.round(bbox.x + bbox.w / 2))+ ) {+ continue;}+ }- if (snapMaps.left.length- && (!snapMaps.right.length || snapMaps.left[0] <= snapMaps.right[0])- && (!snapMaps.center.length || snapMaps.left[0] <= snapMaps.center[0])) {- let thisTurnSnap: Snap[] = [];- for (let item of leftSnap) {- if (item.bbox.leftAnchor && parseFloat(item.bbox.leftAnchor) == snapMaps.left[0]) {- thisTurnSnap.push(item);- }- }- hasChanged = true;- if (!thisTurnSnap.length) {- snapMaps.left.shift();- continue;- }-- let targetX = Math.min(Math.round(snapMaps.left[0] / medianWidth), COLUMN_SPACES);- let lineMax = Math.max(...thisTurnSnap.map((v) => {- let spaceEnd = 0;- if (!rawLines[v.lineIndex].endsWith(' ')) {- spaceEnd = v.bbox.shouldSpace ?? 0;- }- if ((v.bbox.shouldSpace ?? 0) > 1) {- const trailingSpaces = rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;- if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {- spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;- }- }-- return rawLines[v.lineIndex].length + spaceEnd + 1;- }));-- if (targetX < lineMax) {- targetX = lineMax;- }+ if (!canRenderBbox(line, bbox)) {+ break;+ }- if (forwardAnchors.left[snapMaps.left[0]] && targetX < forwardAnchors.left[snapMaps.left[0]]) {- targetX = forwardAnchors.left[snapMaps.left[0]];- }- if (prevAnchors.forwardAnchorLeft[snapMaps.left[0]] && targetX < prevAnchors.forwardAnchorLeft[snapMaps.left[0]]) {- targetX = prevAnchors.forwardAnchorLeft[snapMaps.left[0]];- }+ let targetX = Math.min(Math.round(bbox.x / medianWidth), COLUMN_SPACES);- forwardAnchors.left[snapMaps.left[0]] = targetX;-- for (let currentLeftSnapBox of thisTurnSnap) {- const lineIndex = currentLeftSnapBox.lineIndex;- if (targetX > rawLines[lineIndex].length) {- rawLines[lineIndex] += ' '.repeat(targetX - rawLines[lineIndex].length);- }- rawLines[lineIndex] += currentLeftSnapBox.bbox.str;- currentLeftSnapBox.bbox.rendered = true;-- let nextBbox: ProjectionTextBox | null = null;- if (lines[lineIndex].length > (currentLeftSnapBox.boxIndex + 1)) {- nextBbox = lines[lineIndex][currentLeftSnapBox.boxIndex + 1];- }- updateForwardAnchors(currentLeftSnapBox.bbox, nextBbox, snapMaps, forwardAnchors, rawLines[lineIndex].length);- }-- for (let index = block.start; index < block.end; ++index) {- const line = rawLines[index];- if (line.length < targetX) {- rawLines[index] += ' '.repeat(targetX - line.length);- }- }- snapMaps.left.shift();+ let lastSnapLeft = 0;+ for (const key in forwardAnchors.left) {+ if (parseInt(key) <= bbox.x) {+ lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);}-- else if (snapMaps.right.length- && (!snapMaps.left.length || snapMaps.right[0] <= snapMaps.left[0])- && (!snapMaps.center.length || snapMaps.right[0] <= snapMaps.center[0])) {- let thisTurnSnap: Snap[] = [];- hasChanged = true;- for (let item of rightSnap) {- if (item.bbox.rightAnchor && parseFloat(item.bbox.rightAnchor) == snapMaps.right[0]) {- thisTurnSnap.push(item);- }- }-- if (!thisTurnSnap.length) {- snapMaps.right.shift();- continue;- }-- let targetX = Math.min(Math.round(snapMaps.right[0] / medianWidth), COLUMN_SPACES);- let lineMax = Math.max(...thisTurnSnap.map((v) => {- let lastSnapLeft = 0;- for (let key in forwardAnchors.left) {- if (parseInt(key) <= v.bbox.x) {- lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);- }- }- return Math.max(lastSnapLeft, rawLines[v.lineIndex].trimEnd().length + (v.bbox.shouldSpace ?? 0)) + v.bbox.strLength- }));-- if (targetX < lineMax) {- targetX = lineMax;- }- if (forwardAnchors.right[snapMaps.right[0]] && targetX < forwardAnchors.right[snapMaps.right[0]]) {- targetX = forwardAnchors.right[snapMaps.right[0]];- }- if (prevAnchors.forwardAnchorRight[snapMaps.right[0]] && targetX < prevAnchors.forwardAnchorRight[snapMaps.right[0]]) {- targetX = prevAnchors.forwardAnchorRight[snapMaps.right[0]];- }- forwardAnchors.right[snapMaps.right[0]] = targetX;-- for (let currentRightSnapBox of thisTurnSnap) {- const lineIndex = currentRightSnapBox.lineIndex;- rawLines[lineIndex] = rawLines[lineIndex].trimEnd();- if (targetX > rawLines[lineIndex].trimEnd().length + currentRightSnapBox.bbox.strLength) {- rawLines[lineIndex] += ' '.repeat(targetX - rawLines[lineIndex].length - currentRightSnapBox.bbox.strLength);- }- rawLines[lineIndex] += currentRightSnapBox.bbox.str;- currentRightSnapBox.bbox.rendered = true;-- let nextBbox: ProjectionTextBox | null = null;- if (lines[lineIndex].length > (currentRightSnapBox.boxIndex + 1)) {- nextBbox = lines[lineIndex][currentRightSnapBox.boxIndex + 1];- }- updateForwardAnchors(currentRightSnapBox.bbox, nextBbox, snapMaps, forwardAnchors, rawLines[lineIndex].length);- }- for (let index = block.start; index < block.end; ++index) {- const line = rawLines[index];- if (line.length < targetX) {- rawLines[index] += ' '.repeat(targetX - line.length);- }- }- snapMaps.right.shift();+ }+ const lineMax = Math.max(+ lastSnapLeft,+ rawLines[lineIndex].trimEnd().length + (bbox.shouldSpace ?? 0)+ );+ if (targetX < lineMax) {+ targetX = lineMax;+ }++ if (!bbox.forceUnsnapped) {+ const floatingAnchor = forwardAnchors.floating[Math.round(bbox.x)];+ if (floatingAnchor && targetX < floatingAnchor) {+ targetX = floatingAnchor;}+ }++ rawLines[lineIndex] = rawLines[lineIndex].trimEnd();+ if (targetX > rawLines[lineIndex].length) {+ rawLines[lineIndex] += " ".repeat(targetX - rawLines[lineIndex].length);+ }++ rawLines[lineIndex] += bbox.str;++ bbox.rendered = true;+ hasChanged = true;++ let nextBbox: ProjectionTextBox | null = null;+ if (line.length > boxIndex + 1) {+ nextBbox = line[boxIndex + 1];+ }+ if (!bbox.forceUnsnapped) {+ updateForwardAnchors(+ bbox,+ nextBbox,+ snapMaps,+ forwardAnchors,+ rawLines[lineIndex].length+ );+ }+ }+ }++ if (+ snapMaps.left.length &&+ (!snapMaps.right.length || snapMaps.left[0] <= snapMaps.right[0]) &&+ (!snapMaps.center.length || snapMaps.left[0] <= snapMaps.center[0])+ ) {+ const thisTurnSnap: Snap[] = [];+ for (const item of leftSnap) {+ if (item.bbox.leftAnchor && parseFloat(item.bbox.leftAnchor) == snapMaps.left[0]) {+ thisTurnSnap.push(item);+ }+ }+ hasChanged = true;+ if (!thisTurnSnap.length) {+ snapMaps.left.shift();+ continue;+ }- else if (snapMaps.center.length- && (!snapMaps.left.length || snapMaps.center[0] <= snapMaps.left[0])- && (!snapMaps.right.length || snapMaps.center[0] <= snapMaps.right[0])) {- let thisTurnSnap: Snap[] = [];- hasChanged = true;- for (let item of centerSnap) {- if (item.bbox.centerAnchor && parseFloat(item.bbox.centerAnchor) == snapMaps.center[0]) {- thisTurnSnap.push(item);- }- }- if (!thisTurnSnap.length) {- snapMaps.center.shift();- continue;- }- let targetX = Math.min(Math.round(snapMaps.center[0] / medianWidth), COLUMN_SPACES);- let lineMax = Math.max(- ...(thisTurnSnap.map((v) => {- let spaceEnd = 0;- if (!rawLines[v.lineIndex].endsWith(' ')) {- spaceEnd = v.bbox.shouldSpace ?? 0;- }- if ((v.bbox.shouldSpace ?? 0) > 1) {- const trailingSpaces = rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;- if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {- spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;- }- }- return rawLines[v.lineIndex].length- + Math.round(v.bbox.strLength / 2)- + spaceEnd- }- )));-- if (targetX < lineMax) {- targetX = lineMax;- }- if (forwardAnchors.center[snapMaps.center[0]] && targetX < forwardAnchors.center[snapMaps.center[0]]) {- targetX = forwardAnchors.center[snapMaps.center[0]];- }- if (prevAnchors.forwardAnchorCenter[snapMaps.center[0]] && targetX < prevAnchors.forwardAnchorCenter[snapMaps.center[0]]) {- targetX = prevAnchors.forwardAnchorCenter[snapMaps.center[0]];- }- forwardAnchors.center[snapMaps.center[0]] = targetX;- for (let currentCenterSnapBox of thisTurnSnap) {- if (targetX > rawLines[currentCenterSnapBox.lineIndex].length + Math.round(currentCenterSnapBox.bbox.strLength / 2)) {- rawLines[currentCenterSnapBox.lineIndex] += ' '.repeat(targetX - rawLines[currentCenterSnapBox.lineIndex].length - Math.round(currentCenterSnapBox.bbox.strLength / 2));- }- rawLines[currentCenterSnapBox.lineIndex] += currentCenterSnapBox.bbox.str;- currentCenterSnapBox.bbox.rendered = true;- }- snapMaps.center.shift();+ let targetX = Math.min(Math.round(snapMaps.left[0] / medianWidth), COLUMN_SPACES);+ const lineMax = Math.max(+ ...thisTurnSnap.map((v) => {+ let spaceEnd = 0;+ if (!rawLines[v.lineIndex].endsWith(" ")) {+ spaceEnd = v.bbox.shouldSpace ?? 0;+ }+ if ((v.bbox.shouldSpace ?? 0) > 1) {+ const trailingSpaces =+ rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;+ if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {+ spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;+ }}- }- }- fixSparseBlocks(blocks, rawLines);+ return rawLines[v.lineIndex].length + spaceEnd + 1;+ })+ );- let text = rawLines.join("\n");- // OSS: Return text instead of mutating page object- return {- text,- prevAnchors: {- forwardAnchorLeft: forwardAnchors.left,- forwardAnchorRight: forwardAnchors.right,- forwardAnchorCenter: forwardAnchors.center,+ if (targetX < lineMax) {+ targetX = lineMax;}- };-}-export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig): ParsedPage[] {- let prevAnchors: PrevAnchors = {- forwardAnchorLeft: {},- forwardAnchorRight: {},- forwardAnchorCenter: {}- };+ if (+ forwardAnchors.left[snapMaps.left[0]] &&+ targetX < forwardAnchors.left[snapMaps.left[0]]+ ) {+ targetX = forwardAnchors.left[snapMaps.left[0]];+ }+ if (+ prevAnchors.forwardAnchorLeft[snapMaps.left[0]] &&+ targetX < prevAnchors.forwardAnchorLeft[snapMaps.left[0]]+ ) {+ targetX = prevAnchors.forwardAnchorLeft[snapMaps.left[0]];+ }- const results: ParsedPage[] = [];+ forwardAnchors.left[snapMaps.left[0]] = targetX;++ for (const currentLeftSnapBox of thisTurnSnap) {+ const lineIndex = currentLeftSnapBox.lineIndex;+ if (targetX > rawLines[lineIndex].length) {+ rawLines[lineIndex] += " ".repeat(targetX - rawLines[lineIndex].length);+ }+ rawLines[lineIndex] += currentLeftSnapBox.bbox.str;+ currentLeftSnapBox.bbox.rendered = true;++ let nextBbox: ProjectionTextBox | null = null;+ if (lines[lineIndex].length > currentLeftSnapBox.boxIndex + 1) {+ nextBbox = lines[lineIndex][currentLeftSnapBox.boxIndex + 1];+ }+ updateForwardAnchors(+ currentLeftSnapBox.bbox,+ nextBbox,+ snapMaps,+ forwardAnchors,+ rawLines[lineIndex].length+ );+ }- for (const page of pages) {- // Build projection boxes from text items- const projectionBoxes = buildBbox(page, config);+ for (let index = block.start; index < block.end; ++index) {+ const line = rawLines[index];+ if (line.length < targetX) {+ rawLines[index] += " ".repeat(targetX - line.length);+ }+ }+ snapMaps.left.shift();+ } else if (+ snapMaps.right.length &&+ (!snapMaps.left.length || snapMaps.right[0] <= snapMaps.left[0]) &&+ (!snapMaps.center.length || snapMaps.right[0] <= snapMaps.center[0])+ ) {+ const thisTurnSnap: Snap[] = [];+ hasChanged = true;+ for (const item of rightSnap) {+ if (item.bbox.rightAnchor && parseFloat(item.bbox.rightAnchor) == snapMaps.right[0]) {+ thisTurnSnap.push(item);+ }+ }- // Project to grid- const { text, prevAnchors: newAnchors } = projectToGrid(config, page, projectionBoxes, prevAnchors, pages.length);+ if (!thisTurnSnap.length) {+ snapMaps.right.shift();+ continue;+ }- // Update forward anchors if preserving across pages- if (config.preserveLayoutAlignmentAcrossPages) {- for (let anchor in newAnchors.forwardAnchorLeft) {- prevAnchors.forwardAnchorLeft[anchor] = newAnchors.forwardAnchorLeft[anchor];+ let targetX = Math.min(Math.round(snapMaps.right[0] / medianWidth), COLUMN_SPACES);+ const lineMax = Math.max(+ ...thisTurnSnap.map((v) => {+ let lastSnapLeft = 0;+ for (const key in forwardAnchors.left) {+ if (parseInt(key) <= v.bbox.x) {+ lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);+ }}- for (let anchor in newAnchors.forwardAnchorRight) {- prevAnchors.forwardAnchorRight[anchor] = newAnchors.forwardAnchorRight[anchor];+ return (+ Math.max(+ lastSnapLeft,+ rawLines[v.lineIndex].trimEnd().length + (v.bbox.shouldSpace ?? 0)+ ) + v.bbox.strLength+ );+ })+ );++ if (targetX < lineMax) {+ targetX = lineMax;+ }+ if (+ forwardAnchors.right[snapMaps.right[0]] &&+ targetX < forwardAnchors.right[snapMaps.right[0]]+ ) {+ targetX = forwardAnchors.right[snapMaps.right[0]];+ }+ if (+ prevAnchors.forwardAnchorRight[snapMaps.right[0]] &&+ targetX < prevAnchors.forwardAnchorRight[snapMaps.right[0]]+ ) {+ targetX = prevAnchors.forwardAnchorRight[snapMaps.right[0]];+ }+ forwardAnchors.right[snapMaps.right[0]] = targetX;++ for (const currentRightSnapBox of thisTurnSnap) {+ const lineIndex = currentRightSnapBox.lineIndex;+ rawLines[lineIndex] = rawLines[lineIndex].trimEnd();+ if (targetX > rawLines[lineIndex].trimEnd().length + currentRightSnapBox.bbox.strLength) {+ rawLines[lineIndex] += " ".repeat(+ targetX - rawLines[lineIndex].length - currentRightSnapBox.bbox.strLength+ );+ }+ rawLines[lineIndex] += currentRightSnapBox.bbox.str;+ currentRightSnapBox.bbox.rendered = true;++ let nextBbox: ProjectionTextBox | null = null;+ if (lines[lineIndex].length > currentRightSnapBox.boxIndex + 1) {+ nextBbox = lines[lineIndex][currentRightSnapBox.boxIndex + 1];+ }+ updateForwardAnchors(+ currentRightSnapBox.bbox,+ nextBbox,+ snapMaps,+ forwardAnchors,+ rawLines[lineIndex].length+ );+ }+ for (let index = block.start; index < block.end; ++index) {+ const line = rawLines[index];+ if (line.length < targetX) {+ rawLines[index] += " ".repeat(targetX - line.length);+ }+ }+ snapMaps.right.shift();+ } else if (+ snapMaps.center.length &&+ (!snapMaps.left.length || snapMaps.center[0] <= snapMaps.left[0]) &&+ (!snapMaps.right.length || snapMaps.center[0] <= snapMaps.right[0])+ ) {+ const thisTurnSnap: Snap[] = [];+ hasChanged = true;+ for (const item of centerSnap) {+ if (item.bbox.centerAnchor && parseFloat(item.bbox.centerAnchor) == snapMaps.center[0]) {+ thisTurnSnap.push(item);+ }+ }+ if (!thisTurnSnap.length) {+ snapMaps.center.shift();+ continue;+ }+ let targetX = Math.min(Math.round(snapMaps.center[0] / medianWidth), COLUMN_SPACES);+ const lineMax = Math.max(+ ...thisTurnSnap.map((v) => {+ let spaceEnd = 0;+ if (!rawLines[v.lineIndex].endsWith(" ")) {+ spaceEnd = v.bbox.shouldSpace ?? 0;}- for (let anchor in newAnchors.forwardAnchorCenter) {- prevAnchors.forwardAnchorCenter[anchor] = newAnchors.forwardAnchorCenter[anchor];+ if ((v.bbox.shouldSpace ?? 0) > 1) {+ const trailingSpaces =+ rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;+ if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {+ spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;+ }}+ return rawLines[v.lineIndex].length + Math.round(v.bbox.strLength / 2) + spaceEnd;+ })+ );++ if (targetX < lineMax) {+ targetX = lineMax;}+ if (+ forwardAnchors.center[snapMaps.center[0]] &&+ targetX < forwardAnchors.center[snapMaps.center[0]]+ ) {+ targetX = forwardAnchors.center[snapMaps.center[0]];+ }+ if (+ prevAnchors.forwardAnchorCenter[snapMaps.center[0]] &&+ targetX < prevAnchors.forwardAnchorCenter[snapMaps.center[0]]+ ) {+ targetX = prevAnchors.forwardAnchorCenter[snapMaps.center[0]];+ }+ forwardAnchors.center[snapMaps.center[0]] = targetX;+ for (const currentCenterSnapBox of thisTurnSnap) {+ if (+ targetX >+ rawLines[currentCenterSnapBox.lineIndex].length ++ Math.round(currentCenterSnapBox.bbox.strLength / 2)+ ) {+ rawLines[currentCenterSnapBox.lineIndex] += " ".repeat(+ targetX -+ rawLines[currentCenterSnapBox.lineIndex].length -+ Math.round(currentCenterSnapBox.bbox.strLength / 2)+ );+ }+ rawLines[currentCenterSnapBox.lineIndex] += currentCenterSnapBox.bbox.str;+ currentCenterSnapBox.bbox.rendered = true;+ }+ snapMaps.center.shift();+ }+ }+ }++ fixSparseBlocks(blocks, rawLines);++ const text = rawLines.join("\n");+ // OSS: Return text instead of mutating page object+ return {+ text,+ prevAnchors: {+ forwardAnchorLeft: forwardAnchors.left,+ forwardAnchorRight: forwardAnchors.right,+ forwardAnchorCenter: forwardAnchors.center,+ },+ };+}- // Build result page- results.push({- pageNum: page.pageNum,- width: page.width,- height: page.height,- text,- textItems: page.textItems,- boundingBoxes: [],- tables: []- });+export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig): ParsedPage[] {+ const prevAnchors: PrevAnchors = {+ forwardAnchorLeft: {},+ forwardAnchorRight: {},+ forwardAnchorCenter: {},+ };++ const results: ParsedPage[] = [];++ for (const page of pages) {+ // Build projection boxes from text items+ const projectionBoxes = buildBbox(page, config);++ // Project to grid+ const { text, prevAnchors: newAnchors } = projectToGrid(+ config,+ page,+ projectionBoxes,+ prevAnchors,+ pages.length+ );++ // Update forward anchors if preserving across pages+ if (config.preserveLayoutAlignmentAcrossPages) {+ for (const anchor in newAnchors.forwardAnchorLeft) {+ prevAnchors.forwardAnchorLeft[anchor] = newAnchors.forwardAnchorLeft[anchor];+ }+ for (const anchor in newAnchors.forwardAnchorRight) {+ prevAnchors.forwardAnchorRight[anchor] = newAnchors.forwardAnchorRight[anchor];+ }+ for (const anchor in newAnchors.forwardAnchorCenter) {+ prevAnchors.forwardAnchorCenter[anchor] = newAnchors.forwardAnchorCenter[anchor];+ }}- // Clean raw text (margin detection, etc)- cleanRawText(results, config);+ // Build result page+ results.push({+ pageNum: page.pageNum,+ width: page.width,+ height: page.height,+ text,+ textItems: page.textItems,+ boundingBoxes: [],+ tables: [],+ });+ }++ // Clean raw text (margin detection, etc)+ cleanRawText(results, config);- return results;+ return results;}diff --git a/src/processing/markupUtils.ts b/src/processing/markupUtils.tsindex 63bf25a..94d91ab 100644--- a/src/processing/markupUtils.ts+++ b/src/processing/markupUtils.ts@@ -1,4 +1,4 @@-import { MarkupData } from '../core/types.js';+import { MarkupData } from "../core/types.js";/*** Apply markup tags to text based on markup datadiff --git a/src/processing/ocrUtils.ts b/src/processing/ocrUtils.tsindex 64a6c4d..9ae11f1 100644--- a/src/processing/ocrUtils.ts+++ b/src/processing/ocrUtils.ts@@ -1,4 +1,4 @@-import { Image, EasyOcrResultLine } from '../engines/pdf/interface.js';+import { Image, EasyOcrResultLine } from "../engines/pdf/interface.js";export interface OcrBlock {c: string; // Content/text@@ -60,14 +60,12 @@ export function parseImageOcrBlocks(image: Image): OcrBlock[] {/*** Parse EasyOCR stdout result into structured format*/-export function easyOcrResultLinesToList(- stdOutResult?: string-): EasyOcrResultLine[] {+export function easyOcrResultLinesToList(stdOutResult?: string): EasyOcrResultLine[] {if (!stdOutResult?.length) {return [];}- const lines = stdOutResult.split('\n');+ const lines = stdOutResult.split("\n");const blocks: EasyOcrResultLine[] = [];for (const line of lines) {diff --git a/src/processing/pathUtils.ts b/src/processing/pathUtils.tsindex 5ffb374..4686357 100644--- a/src/processing/pathUtils.ts+++ b/src/processing/pathUtils.ts@@ -1,5 +1,5 @@-import { Path } from '../engines/pdf/interface.js';-import { LiteParseConfig } from '../core/types.js';+import { Path } from "../engines/pdf/interface.js";+import { LiteParseConfig } from "../core/types.js";export interface Line {x: number;@@ -27,7 +27,7 @@ export function getVerticalAndHorizontalLinesFromPagePaths(// and extract line segments from rectangles and line pathsfor (const path of paths) {- if (path.type === 'line' && path.points.length >= 2) {+ if (path.type === "line" && path.points.length >= 2) {const [start, end] = path.points;const [x1, y1] = start;const [x2, y2] = end;@@ -54,7 +54,7 @@ export function getVerticalAndHorizontalLinesFromPagePaths(}}// Handle rectangles as potential table borders- else if (path.type === 'rectangle' && path.points.length >= 4) {+ else if (path.type === "rectangle" && path.points.length >= 4) {const xs = path.points.map((p) => p[0]);const ys = path.points.map((p) => p[1]);const minX = Math.min(...xs);diff --git a/src/processing/tables.ts b/src/processing/tables.tsindex 043d359..ebd5bd0 100644--- a/src/processing/tables.ts+++ b/src/processing/tables.ts@@ -1,5 +1,5 @@-import { Path } from '../engines/pdf/interface.js';-import { TextItem } from '../core/types.js';+import { Path } from "../engines/pdf/interface.js";+import { TextItem } from "../core/types.js";export interface DetectedTable {x1: number;@@ -43,7 +43,7 @@ function extractLines(paths: Path[]): { horizontal: Line[]; vertical: Line[] } {const tolerance = 2; // Tolerance for determining if a line is horizontal/verticalfor (const path of paths) {- if (path.type === 'line' && path.points.length >= 2) {+ if (path.type === "line" && path.points.length >= 2) {const [start, end] = path.points;const [x1, y1] = start;const [x2, y2] = end;@@ -73,7 +73,7 @@ function extractLines(paths: Path[]): { horizontal: Line[]; vertical: Line[] } {}}// Extract lines from rectangles- else if (path.type === 'rectangle' && path.points.length >= 4) {+ else if (path.type === "rectangle" && path.points.length >= 4) {const xs = path.points.map((p) => p[0]);const ys = path.points.map((p) => p[1]);const minX = Math.min(...xs);@@ -151,9 +151,7 @@ function clusterLines(lines: Line[], isHorizontal: boolean): Line[] {const threshold = 5;// Check if lines are close enough to merge- const distance = isHorizontal- ? Math.abs(line.y1 - current.y1)- : Math.abs(line.x1 - current.x1);+ const distance = isHorizontal ? Math.abs(line.y1 - current.y1) : Math.abs(line.x1 - current.x1);if (distance < threshold) {// Merge lines@@ -181,9 +179,7 @@ function clusterLines(lines: Line[], isHorizontal: boolean): Line[] {/*** Create cells from intersections*/-function createCellsFromIntersections(- intersections: Intersection[]-): TableCell[] {+function createCellsFromIntersections(intersections: Intersection[]): TableCell[] {if (intersections.length < 4) return [];// Sort intersections@@ -193,12 +189,8 @@ function createCellsFromIntersections(});// Find unique x and y coordinates- const xCoords = [...new Set(sorted.map((i) => Math.round(i.x)))].sort(- (a, b) => a - b- );- const yCoords = [...new Set(sorted.map((i) => Math.round(i.y)))].sort(- (a, b) => a - b- );+ const xCoords = [...new Set(sorted.map((i) => Math.round(i.x)))].sort((a, b) => a - b);+ const yCoords = [...new Set(sorted.map((i) => Math.round(i.y)))].sort((a, b) => a - b);const cells: TableCell[] = [];@@ -210,7 +202,7 @@ function createCellsFromIntersections(y1: yCoords[row],x2: xCoords[col + 1],y2: yCoords[row + 1],- text: '',+ text: "",row,col,});@@ -242,7 +234,7 @@ function assignTextToCells(cells: TableCell[], textItems: TextItem[]): void {}}- cell.text = cellText.join(' ').trim();+ cell.text = cellText.join(" ").trim();}}@@ -267,10 +259,7 @@ function isValidTable(* Detect tables using heuristic methods (outlined paths)* Uses line intersection detection to find table grid structures*/-export function detectTables(- paths: Path[],- textItems: TextItem[] = []-): DetectedTable[] {+export function detectTables(paths: Path[], textItems: TextItem[] = []): DetectedTable[] {const tables: DetectedTable[] = [];// Extract lines from paths@@ -324,10 +313,7 @@ export function detectTables(/*** Extract table structure from detected table region*/-export function extractTableStructure(- table: DetectedTable,- _paths: Path[]-): any {+export function extractTableStructure(table: DetectedTable, _paths: Path[]): any {return {rows: table.rows,cols: table.cols,diff --git a/src/processing/textUtils.ts b/src/processing/textUtils.tsindex 607033b..99c6b58 100644--- a/src/processing/textUtils.ts+++ b/src/processing/textUtils.ts@@ -3,35 +3,35 @@*/export function strToSubscriptString(str: string): string {const sub: { [key: string]: string } = {- '0': '₀',- '1': '₁',- '2': '₂',- '3': '₃',- '4': '₄',- '5': '₅',- '6': '₆',- '7': '₇',- '8': '₈',- '9': '₉',- '+': '₊',- '-': '₋',- 'a': 'ₐ',- 'e': 'ₑ',- 'o': 'ₒ',- 'x': 'ₓ',- 'ə': 'ₔ',- 'h': 'ₕ',- 'k': 'ₖ',- 'l': 'ₗ',- 'm': 'ₘ',- 'n': 'ₙ',- 'p': 'ₚ',- 'r': 'ᵣ',- 's': 'ₛ',- 't': 'ₜ',+ "0": "₀",+ "1": "₁",+ "2": "₂",+ "3": "₃",+ "4": "₄",+ "5": "₅",+ "6": "₆",+ "7": "₇",+ "8": "₈",+ "9": "₉",+ "+": "₊",+ "-": "₋",+ a: "ₐ",+ e: "ₑ",+ o: "ₒ",+ x: "ₓ",+ ə: "ₔ",+ h: "ₕ",+ k: "ₖ",+ l: "ₗ",+ m: "ₘ",+ n: "ₙ",+ p: "ₚ",+ r: "ᵣ",+ s: "ₛ",+ t: "ₜ",};- let subStr = '';+ let subStr = "";for (let i = 0; i < str.length; i++) {if (!sub[str[i]]) {return str;@@ -47,65 +47,65 @@ export function strToSubscriptString(str: string): string {*/export function strToPostScript(str: string): string {const post: { [key: string]: string } = {- '0': '⁰',- '1': '¹',- '2': '²',- '3': '³',- '4': '⁴',- '5': '⁵',- '6': '⁶',- '7': '⁷',- '8': '⁸',- '9': '⁹',- '+': '⁺',- '-': '⁻',- 'a': 'ᵃ',- 'b': 'ᵇ',- 'c': 'ᶜ',- 'd': 'ᵈ',- 'e': 'ᵉ',- 'f': 'ᶠ',- 'g': 'ᵍ',- 'h': 'ʰ',- 'i': 'ⁱ',- 'j': 'ʲ',- 'k': 'ᵏ',- 'l': 'ˡ',- 'm': 'ᵐ',- 'n': 'ⁿ',- 'o': 'ᵒ',- 'p': 'ᵖ',- 'r': 'ʳ',- 's': 'ˢ',- 't': 'ᵗ',- 'u': 'ᵘ',- 'v': 'ᵛ',- 'w': 'ʷ',- 'x': 'ˣ',- 'y': 'ʸ',- 'z': 'ᶻ',- 'A': 'ᴬ',- 'B': 'ᴮ',- 'D': 'ᴰ',- 'E': 'ᴱ',- 'G': 'ᴳ',- 'H': 'ᴴ',- 'I': 'ᴵ',- 'J': 'ᴶ',- 'K': 'ᴷ',- 'L': 'ᴸ',- 'M': 'ᴹ',- 'N': 'ᴺ',- 'O': 'ᴼ',- 'P': 'ᴾ',- 'R': 'ᴿ',- 'T': 'ᵀ',- 'U': 'ᵁ',- 'V': 'ⱽ',- 'W': 'ᵂ',+ "0": "⁰",+ "1": "¹",+ "2": "²",+ "3": "³",+ "4": "⁴",+ "5": "⁵",+ "6": "⁶",+ "7": "⁷",+ "8": "⁸",+ "9": "⁹",+ "+": "⁺",+ "-": "⁻",+ a: "ᵃ",+ b: "ᵇ",+ c: "ᶜ",+ d: "ᵈ",+ e: "ᵉ",+ f: "ᶠ",+ g: "ᵍ",+ h: "ʰ",+ i: "ⁱ",+ j: "ʲ",+ k: "ᵏ",+ l: "ˡ",+ m: "ᵐ",+ n: "ⁿ",+ o: "ᵒ",+ p: "ᵖ",+ r: "ʳ",+ s: "ˢ",+ t: "ᵗ",+ u: "ᵘ",+ v: "ᵛ",+ w: "ʷ",+ x: "ˣ",+ y: "ʸ",+ z: "ᶻ",+ A: "ᴬ",+ B: "ᴮ",+ D: "ᴰ",+ E: "ᴱ",+ G: "ᴳ",+ H: "ᴴ",+ I: "ᴵ",+ J: "ᴶ",+ K: "ᴷ",+ L: "ᴸ",+ M: "ᴹ",+ N: "ᴺ",+ O: "ᴼ",+ P: "ᴾ",+ R: "ᴿ",+ T: "ᵀ",+ U: "ᵁ",+ V: "ⱽ",+ W: "ᵂ",};- let postStr = '';+ let postStr = "";for (let i = 0; i < str.length; i++) {if (!post[str[i]]) {return str;