Skip to content

A fast, helpful, and open-source document parser

format

Logan Markewichdba4b4cf5c47
diff --git a/README.md b/README.md
index a03bebe..99a9a03 100644
--- a/README.md
+++ b/README.md
@@ -87,9 +87,14 @@ liteparse parse document.pdf --target-pages "1-5,10,15-20"
# Parse without OCR
liteparse parse document.pdf --no-ocr
+```
+
+### Batch Parsing
+
+You can also parse an entire directory of documents:
-# Use custom config file
-liteparse parse document.pdf --config config.json
+```bash
+liteparse batch-parse ./input-directory ./output-directory
```
### Generate Screenshots
@@ -133,7 +138,7 @@ console.log(result.text);
#### Parse Command
```
-liteparse parse <file> [options]
+$ liteparse parse <file> [options]
Options:
-o, --output <file> Output file path
@@ -152,10 +157,33 @@ Options:
--config <file> Config file (JSON)
```
+#### Batch Parse Command
+
+```
+$ liteparse batch-parse --help
+
+Parse multiple documents in batch mode (reuses PDF engine for efficiency)
+
+Options:
+ --format <format> Output format: json|text (default: "text")
+ --ocr-server-url <url> HTTP OCR server URL (uses Tesseract if not provided)
+ --no-ocr Disable OCR
+ --ocr-language <lang> OCR language(s) (default: "en")
+ --max-pages <n> Max pages to parse per file (default: "1000")
+ --dpi <dpi> DPI for rendering (default: "150")
+ --no-tables Disable table detection
+ --no-precise-bbox Disable precise bounding boxes
+ --recursive Recursively search input directory
+ --extension <ext> Only process files with this extension (e.g., ".pdf")
+ --config <file> Config file (JSON)
+ -q, --quiet Suppress progress output
+ -h, --help display help for command
+```
+
#### Screenshot Command
```
-liteparse screenshot <file> [options]
+$ liteparse screenshot <file> [options]
Options:
-o, --output-dir <dir> Output directory for screenshots (default: "./screenshots")
diff --git a/cli/parse.ts b/cli/parse.ts
index 627e425..3787212 100644
--- a/cli/parse.ts
+++ b/cli/parse.ts
@@ -1,35 +1,35 @@
-import { Command } from 'commander';
-import fs from 'fs/promises';
-import { existsSync, readdirSync, statSync } from 'fs';
-import path from 'path';
-import { LiteParse } from '../src/core/parser.js';
-import { LiteParseConfig, OutputFormat } from '../src/core/types.js';
-import { performance } from 'perf_hooks';
+import { Command } from "commander";
+import fs from "fs/promises";
+import { existsSync, readdirSync, statSync } from "fs";
+import path from "path";
+import { LiteParse } from "../src/core/parser.js";
+import { LiteParseConfig, OutputFormat } from "../src/core/types.js";
+import { performance } from "perf_hooks";
const program = new Command();
program
- .name('liteparse')
- .description('OSS document parsing tool (supports PDF, DOCX, XLSX, images, and more)')
- .version('1.0.0');
+ .name("liteparse")
+ .description("OSS document parsing tool (supports PDF, DOCX, XLSX, images, and more)")
+ .version("1.0.0");
program
- .command('parse <file>')
- .description('Parse a document file (PDF, DOCX, XLSX, PPTX, images, etc.)')
- .option('-o, --output <file>', 'Output file path')
- .option('--format <format>', 'Output format: json|text', 'text')
- .option('--ocr-server-url <url>', 'HTTP OCR server URL (uses Tesseract if not provided)')
- .option('--no-ocr', 'Disable OCR')
- .option('--ocr-language <lang>', 'OCR language(s)', 'en')
- .option('--max-pages <n>', 'Max pages to parse', '1000')
- .option('--target-pages <pages>', 'Target pages (e.g., "1-5,10,15-20")')
- .option('--dpi <dpi>', 'DPI for rendering', '150')
- .option('--no-tables', 'Disable table detection')
- .option('--no-precise-bbox', 'Disable precise bounding boxes')
- .option('--skip-diagonal-text', 'Skip diagonal text')
- .option('--preserve-small-text', 'Preserve very small text')
- .option('--config <file>', 'Config file (JSON)')
- .option('-q, --quiet', 'Suppress progress output')
+ .command("parse <file>")
+ .description("Parse a document file (PDF, DOCX, XLSX, PPTX, images, etc.)")
+ .option("-o, --output <file>", "Output file path")
+ .option("--format <format>", "Output format: json|text", "text")
+ .option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)")
+ .option("--no-ocr", "Disable OCR")
+ .option("--ocr-language <lang>", "OCR language(s)", "en")
+ .option("--max-pages <n>", "Max pages to parse", "1000")
+ .option("--target-pages <pages>", 'Target pages (e.g., "1-5,10,15-20")')
+ .option("--dpi <dpi>", "DPI for rendering", "150")
+ .option("--no-tables", "Disable table detection")
+ .option("--no-precise-bbox", "Disable precise bounding boxes")
+ .option("--skip-diagonal-text", "Skip diagonal text")
+ .option("--preserve-small-text", "Preserve very small text")
+ .option("--config <file>", "Config file (JSON)")
+ .option("-q, --quiet", "Suppress progress output")
.action(async (file: string, options: any) => {
try {
const quiet = options.quiet || false;
@@ -48,7 +48,7 @@ program
console.error(`Error: Config file not found: ${options.config}`);
process.exit(1);
}
- const configData = await fs.readFile(options.config, 'utf-8');
+ const configData = await fs.readFile(options.config, "utf-8");
config = JSON.parse(configData);
}
@@ -77,10 +77,10 @@ program
// Format output based on format
let output: string;
switch (config.outputFormat) {
- case 'json':
+ case "json":
output = JSON.stringify(result.json, null, 2);
break;
- case 'text':
+ case "text":
default:
output = result.text;
break;
@@ -106,14 +106,14 @@ program
});
program
- .command('screenshot <file>')
- .description('Generate screenshots of PDF pages')
- .option('-o, --output-dir <dir>', 'Output directory for screenshots', './screenshots')
- .option('--pages <pages>', 'Page numbers to screenshot (e.g., "1,3,5" or "1-5")')
- .option('--dpi <dpi>', 'DPI for rendering', '150')
- .option('--format <format>', 'Image format: png|jpg', 'png')
- .option('--config <file>', 'Config file (JSON)')
- .option('-q, --quiet', 'Suppress progress output')
+ .command("screenshot <file>")
+ .description("Generate screenshots of PDF pages")
+ .option("-o, --output-dir <dir>", "Output directory for screenshots", "./screenshots")
+ .option("--pages <pages>", 'Page numbers to screenshot (e.g., "1,3,5" or "1-5")')
+ .option("--dpi <dpi>", "DPI for rendering", "150")
+ .option("--format <format>", "Image format: png|jpg", "png")
+ .option("--config <file>", "Config file (JSON)")
+ .option("-q, --quiet", "Suppress progress output")
.action(async (file: string, options: any) => {
try {
const quiet = options.quiet || false;
@@ -132,7 +132,7 @@ program
console.error(`Error: Config file not found: ${options.config}`);
process.exit(1);
}
- const configData = await fs.readFile(options.config, 'utf-8');
+ const configData = await fs.readFile(options.config, "utf-8");
config = JSON.parse(configData);
}
@@ -183,29 +183,61 @@ program
// Supported file extensions for batch parsing
const SUPPORTED_EXTENSIONS = new Set([
- '.pdf',
- '.doc', '.docx', '.docm', '.dot', '.dotm', '.dotx', '.odt', '.ott',
- '.ppt', '.pptx', '.pptm', '.pot', '.potm', '.potx', '.odp', '.otp',
- '.xls', '.xlsx', '.xlsm', '.xlsb', '.ods', '.ots', '.csv', '.tsv',
- '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.tif', '.webp', '.svg',
- '.rtf', '.pages', '.key', '.numbers',
+ ".pdf",
+ ".doc",
+ ".docx",
+ ".docm",
+ ".dot",
+ ".dotm",
+ ".dotx",
+ ".odt",
+ ".ott",
+ ".ppt",
+ ".pptx",
+ ".pptm",
+ ".pot",
+ ".potm",
+ ".potx",
+ ".odp",
+ ".otp",
+ ".xls",
+ ".xlsx",
+ ".xlsm",
+ ".xlsb",
+ ".ods",
+ ".ots",
+ ".csv",
+ ".tsv",
+ ".jpg",
+ ".jpeg",
+ ".png",
+ ".gif",
+ ".bmp",
+ ".tiff",
+ ".tif",
+ ".webp",
+ ".svg",
+ ".rtf",
+ ".pages",
+ ".key",
+ ".numbers",
]);
program
- .command('batch-parse <input-dir> <output-dir>')
- .description('Parse multiple documents in batch mode')
- .option('--format <format>', 'Output format: json|text', 'text')
- .option('--ocr-server-url <url>', 'HTTP OCR server URL (uses Tesseract if not provided)')
- .option('--no-ocr', 'Disable OCR')
- .option('--ocr-language <lang>', 'OCR language(s)', 'en')
- .option('--max-pages <n>', 'Max pages to parse per file', '1000')
- .option('--dpi <dpi>', 'DPI for rendering', '150')
- .option('--no-tables', 'Disable table detection')
- .option('--no-precise-bbox', 'Disable precise bounding boxes')
- .option('--recursive', 'Recursively search input directory')
- .option('--extension <ext>', 'Only process files with this extension (e.g., ".pdf")')
- .option('--config <file>', 'Config file (JSON)')
- .option('-q, --quiet', 'Suppress progress output')
+ .command("batch-parse <input-dir> <output-dir>")
+ .description("Parse multiple documents in batch mode")
+ .option("--format <format>", "Output format: json|text", "text")
+ .option("--ocr-server-url <url>", "HTTP OCR server URL (uses Tesseract if not provided)")
+ .option("--no-ocr", "Disable OCR")
+ .option("--ocr-language <lang>", "OCR language(s)", "en")
+ .option("--max-pages <n>", "Max pages to parse per file", "1000")
+ .option("--dpi <dpi>", "DPI for rendering", "150")
+ .option("--no-tables", "Disable table detection")
+ .option("--no-precise-bbox", "Disable precise bounding boxes")
+ .option("--recursive", "Recursively search input directory")
+ .option("--extension <ext>", 'Only process files with this extension (e.g., ".pdf")')
+ .option("--config <file>", "Config file (JSON)")
+ .option("-q, --quiet", "Suppress progress output")
.action(async (inputDir: string, outputDir: string, options: any) => {
try {
const quiet = options.quiet || false;
@@ -232,7 +264,7 @@ program
const files = findFiles(inputDir, options.recursive, options.extension);
if (files.length === 0) {
- console.error('No supported files found in input directory');
+ console.error("No supported files found in input directory");
process.exit(1);
}
@@ -247,7 +279,7 @@ program
console.error(`Error: Config file not found: ${options.config}`);
process.exit(1);
}
- const configData = await fs.readFile(options.config, 'utf-8');
+ const configData = await fs.readFile(options.config, "utf-8");
config = JSON.parse(configData);
}
@@ -270,15 +302,12 @@ program
// Process files
let successCount = 0;
let errorCount = 0;
- const outputExt = options.format === 'json' ? '.json' : '.txt';
+ const outputExt = options.format === "json" ? ".json" : ".txt";
for (let i = 0; i < files.length; i++) {
const file = files[i];
const relativePath = path.relative(inputDir, file);
- const outputPath = path.join(
- outputDir,
- relativePath.replace(/\.[^.]+$/, outputExt)
- );
+ const outputPath = path.join(outputDir, relativePath.replace(/\.[^.]+$/, outputExt));
// Create output subdirectory if needed
const outputSubdir = path.dirname(outputPath);
@@ -292,7 +321,7 @@ program
// Format output
let output: string;
- if (options.format === 'json') {
+ if (options.format === "json") {
output = JSON.stringify(result.json, null, 2);
} else {
output = result.text;
@@ -303,7 +332,9 @@ program
if (!quiet) {
const fileTime = (performance.now() - fileStart).toFixed(0);
- console.error(`[${i + 1}/${files.length}] ✓ ${relativePath} (${result.pages.length} pages, ${fileTime}ms)`);
+ console.error(
+ `[${i + 1}/${files.length}] ✓ ${relativePath} (${result.pages.length} pages, ${fileTime}ms)`
+ );
}
} catch (error: any) {
errorCount++;
@@ -314,10 +345,11 @@ program
}
const totalTime = ((performance.now() - startTime) / 1000).toFixed(2);
- const avgTime = files.length > 0 ? ((performance.now() - startTime) / files.length).toFixed(0) : 0;
+ const avgTime =
+ files.length > 0 ? ((performance.now() - startTime) / files.length).toFixed(0) : 0;
if (!quiet) {
- console.error('');
+ console.error("");
console.error(`Batch complete: ${successCount} succeeded, ${errorCount} failed`);
console.error(`Total time: ${totalTime}s (avg ${avgTime}ms/file)`);
console.error(`Output: ${outputDir}`);
@@ -377,12 +409,12 @@ function findFiles(dir: string, recursive: boolean, filterExt?: string): string[
*/
function parsePageNumbers(pagesStr: string): number[] {
const pages: number[] = [];
- const parts = pagesStr.split(',');
+ const parts = pagesStr.split(",");
for (const part of parts) {
const trimmed = part.trim();
- if (trimmed.includes('-')) {
- const [start, end] = trimmed.split('-').map((n) => parseInt(n.trim()));
+ if (trimmed.includes("-")) {
+ const [start, end] = trimmed.split("-").map((n) => parseInt(n.trim()));
for (let i = start; i <= end; i++) {
pages.push(i);
}
diff --git a/package.json b/package.json
index a47d838..80ada6b 100644
--- a/package.json
+++ b/package.json
@@ -24,7 +24,13 @@
"scripts": {
"build": "tsc && npm run copy-vendor",
"copy-vendor": "mkdir -p dist/src/vendor && cp -r src/vendor/pdfjs dist/src/vendor/",
- "dev": "tsc --watch"
+ "dev": "tsc --watch",
+ "test": "vitest run",
+ "test:watch": "vitest",
+ "lint": "eslint src cli",
+ "lint:fix": "eslint src cli --fix",
+ "format": "prettier --write \"src/**/*.ts\" \"cli/**/*.ts\"",
+ "format:check": "prettier --check \"src/**/*.ts\" \"cli/**/*.ts\""
},
"dependencies": {
"@hyzyla/pdfium": "^2.1.9",
@@ -38,8 +44,15 @@
"zod": "^3.23.0"
},
"devDependencies": {
+ "@eslint/js": "^10.0.1",
"@types/node": "^22.0.0",
- "typescript": "~5.9.2"
+ "eslint": "^10.0.0",
+ "eslint-config-prettier": "^10.1.8",
+ "eslint-plugin-prettier": "^5.5.5",
+ "prettier": "^3.8.1",
+ "typescript": "~5.9.2",
+ "typescript-eslint": "^8.56.0",
+ "vitest": "^4.0.18"
},
"engines": {
"node": ">=18.0.0"
@@ -62,4 +75,4 @@
"homepage": "https://github.com/run-llama/liteparse#readme",
"author": "",
"license": "Apache-2.0"
-}
\ No newline at end of file
+}
diff --git a/src/conversion/convertToPdf.ts b/src/conversion/convertToPdf.ts
index 8d5ac4d..ea0eea3 100644
--- a/src/conversion/convertToPdf.ts
+++ b/src/conversion/convertToPdf.ts
@@ -1,7 +1,7 @@
-import { promises as fs } from 'fs';
-import { spawn } from 'child_process';
-import path from 'path';
-import os from 'os';
+import { promises as fs } from "fs";
+import { spawn } from "child_process";
+import path from "path";
+import os from "os";
export interface ConversionResult {
pdfPath: string;
@@ -14,57 +14,57 @@ export interface ConversionError {
}
export interface ConversionPassthrough {
- content: string
+ content: string;
}
// File extension categories
export const officeExtensions = [
- '.doc',
- '.docx',
- '.docm',
- '.dot',
- '.dotm',
- '.dotx',
- '.odt',
- '.ott',
- '.ppt',
- '.pptx',
- '.pptm',
- '.pot',
- '.potm',
- '.potx',
- '.odp',
- '.otp',
- '.rtf',
- '.pages',
- '.key',
+ ".doc",
+ ".docx",
+ ".docm",
+ ".dot",
+ ".dotm",
+ ".dotx",
+ ".odt",
+ ".ott",
+ ".ppt",
+ ".pptx",
+ ".pptm",
+ ".pot",
+ ".potm",
+ ".potx",
+ ".odp",
+ ".otp",
+ ".rtf",
+ ".pages",
+ ".key",
];
export const spreadsheetExtensions = [
- '.xls',
- '.xlsx',
- '.xlsm',
- '.xlsb',
- '.ods',
- '.ots',
- '.csv',
- '.tsv',
- '.numbers',
+ ".xls",
+ ".xlsx",
+ ".xlsm",
+ ".xlsb",
+ ".ods",
+ ".ots",
+ ".csv",
+ ".tsv",
+ ".numbers",
];
export const imageExtensions = [
- '.jpg',
- '.jpeg',
- '.png',
- '.gif',
- '.bmp',
- '.tiff',
- '.tif',
- '.webp',
- '.svg',
+ ".jpg",
+ ".jpeg",
+ ".png",
+ ".gif",
+ ".bmp",
+ ".tiff",
+ ".tif",
+ ".webp",
+ ".svg",
];
-export const htmlExtensions = ['.htm', '.html', '.xhtml'];
+export const htmlExtensions = [".htm", ".html", ".xhtml"];
/**
* Guess file extension from file content
@@ -77,60 +77,51 @@ export async function guessFileExtension(filePath: string): Promise<string> {
// Read first few bytes to detect file type
const buffer = Buffer.alloc(16);
- const fd = await fs.open(filePath, 'r');
+ const fd = await fs.open(filePath, "r");
await fd.read(buffer, 0, 16, 0);
await fd.close();
// PDF: %PDF
- if (buffer.toString('utf-8', 0, 4) === '%PDF') {
- return '.pdf';
+ if (buffer.toString("utf-8", 0, 4) === "%PDF") {
+ return ".pdf";
}
// PNG: 89 50 4E 47
- if (
- buffer[0] === 0x89 &&
- buffer[1] === 0x50 &&
- buffer[2] === 0x4e &&
- buffer[3] === 0x47
- ) {
- return '.png';
+ if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
+ return ".png";
}
// JPEG: FF D8 FF
if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
- return '.jpg';
+ return ".jpg";
}
// ZIP-based formats (docx, xlsx, etc): PK
if (buffer[0] === 0x50 && buffer[1] === 0x4b) {
// Could be docx, xlsx, pptx, odt, etc.
- return '.docx'; // Default to docx for now
+ return ".docx"; // Default to docx for now
}
- return ext || '.pdf';
+ return ext || ".pdf";
}
/**
* Execute command with timeout
*/
-async function executeCommand(
- command: string,
- args: string[],
- timeoutMs = 60000
-): Promise<string> {
+async function executeCommand(command: string, args: string[], timeoutMs = 60000): Promise<string> {
return new Promise((resolve, reject) => {
const proc = spawn(command, args, {
- stdio: ['ignore', 'pipe', 'pipe'],
+ stdio: ["ignore", "pipe", "pipe"],
});
- let stdout = '';
- let stderr = '';
+ let stdout = "";
+ let stderr = "";
- proc.stdout?.on('data', (data) => {
+ proc.stdout?.on("data", (data) => {
stdout += data.toString();
});
- proc.stderr?.on('data', (data) => {
+ proc.stderr?.on("data", (data) => {
stderr += data.toString();
});
@@ -139,7 +130,7 @@ async function executeCommand(
reject(new Error(`Command timeout after ${timeoutMs}ms`));
}, timeoutMs);
- proc.on('close', (code) => {
+ proc.on("close", (code) => {
clearTimeout(timeout);
if (code === 0) {
resolve(stdout);
@@ -148,7 +139,7 @@ async function executeCommand(
}
});
- proc.on('error', (err) => {
+ proc.on("error", (err) => {
clearTimeout(timeout);
reject(err);
});
@@ -160,7 +151,7 @@ async function executeCommand(
*/
async function isCommandAvailable(command: string): Promise<boolean> {
try {
- await executeCommand('which', [command], 5000);
+ await executeCommand("which", [command], 5000);
return true;
} catch {
return false;
@@ -170,28 +161,17 @@ async function isCommandAvailable(command: string): Promise<boolean> {
/**
* Convert office documents using LibreOffice
*/
-async function convertOfficeDocument(
- filePath: string,
- outputDir: string
-): Promise<string> {
- const hasLibreOffice = await isCommandAvailable('libreoffice');
+async function convertOfficeDocument(filePath: string, outputDir: string): Promise<string> {
+ const hasLibreOffice = await isCommandAvailable("libreoffice");
if (!hasLibreOffice) {
throw new Error(
- 'LibreOffice is not installed. Please install LibreOffice to convert office documents. On macOS: brew install --cask libreoffice, On Ubuntu: apt-get install libreoffice'
+ "LibreOffice is not installed. Please install LibreOffice to convert office documents. On macOS: brew install --cask libreoffice, On Ubuntu: apt-get install libreoffice"
);
}
await executeCommand(
- 'libreoffice',
- [
- '--headless',
- '--invisible',
- '--convert-to',
- 'pdf',
- '--outdir',
- outputDir,
- filePath,
- ],
+ "libreoffice",
+ ["--headless", "--invisible", "--convert-to", "pdf", "--outdir", outputDir, filePath],
120000 // 2 minutes timeout
);
@@ -204,21 +184,18 @@ async function convertOfficeDocument(
await fs.access(pdfPath);
return pdfPath;
} catch {
- throw new Error('LibreOffice conversion succeeded but output PDF not found');
+ throw new Error("LibreOffice conversion succeeded but output PDF not found");
}
}
/**
* Convert images to PDF using ImageMagick
*/
-async function convertImageToPdf(
- filePath: string,
- outputDir: string
-): Promise<string> {
- const hasConvert = await isCommandAvailable('convert');
+async function convertImageToPdf(filePath: string, outputDir: string): Promise<string> {
+ const hasConvert = await isCommandAvailable("convert");
if (!hasConvert) {
throw new Error(
- 'ImageMagick is not installed. Please install ImageMagick to convert images. On macOS: brew install imagemagick, On Ubuntu: apt-get install imagemagick'
+ "ImageMagick is not installed. Please install ImageMagick to convert images. On macOS: brew install imagemagick, On Ubuntu: apt-get install imagemagick"
);
}
@@ -226,15 +203,8 @@ async function convertImageToPdf(
const pdfPath = path.join(outputDir, `${baseName}.pdf`);
await executeCommand(
- 'convert',
- [
- filePath,
- '-density',
- '150',
- '-units',
- 'PixelsPerInch',
- pdfPath,
- ],
+ "convert",
+ [filePath, "-density", "150", "-units", "PixelsPerInch", pdfPath],
60000
);
@@ -254,7 +224,7 @@ export async function convertToPdf(
} catch {
return {
message: `File not found: ${filePath}`,
- code: 'FILE_NOT_FOUND',
+ code: "FILE_NOT_FOUND",
};
}
@@ -262,7 +232,7 @@ export async function convertToPdf(
const extension = await guessFileExtension(filePath);
// If already PDF, return as-is
- if (extension === '.pdf') {
+ if (extension === ".pdf") {
return {
pdfPath: filePath,
originalExtension: extension,
@@ -270,7 +240,7 @@ export async function convertToPdf(
}
// Create temp directory for output
- const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'liteparse-'));
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "liteparse-"));
// Convert based on file type
let pdfPath: string;
@@ -284,11 +254,10 @@ export async function convertToPdf(
} else if (imageExtensions.includes(extension)) {
console.error(`Converting image: ${path.basename(filePath)}`);
pdfPath = await convertImageToPdf(filePath, tmpDir);
- }
- else {
+ } else {
// Unsupported format
// Assume its a text-based format and pass through text directly
- const content = await fs.readFile(filePath, 'utf-8');
+ const content = await fs.readFile(filePath, "utf-8");
return {
content,
};
@@ -301,7 +270,7 @@ export async function convertToPdf(
} catch (error) {
return {
message: error instanceof Error ? error.message : String(error),
- code: 'CONVERSION_ERROR',
+ code: "CONVERSION_ERROR",
};
}
}
diff --git a/src/core/config.ts b/src/core/config.ts
index 1eef499..fdd9d1a 100644
--- a/src/core/config.ts
+++ b/src/core/config.ts
@@ -1,9 +1,9 @@
-import { LiteParseConfig } from './types.js';
+import { LiteParseConfig } from "./types.js";
export const DEFAULT_CONFIG: LiteParseConfig = {
// OCR - defaults to in-process Tesseract for zero-setup experience
// If ocrServerUrl is provided, uses HTTP OCR instead
- ocrLanguage: 'en',
+ ocrLanguage: "en",
ocrEnabled: true,
ocrServerUrl: undefined, // If set, uses HTTP OCR; otherwise uses Tesseract
@@ -13,7 +13,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {
dpi: 150,
// Output
- outputFormat: 'json',
+ outputFormat: "json",
includeImages: true,
includeCharts: true,
@@ -25,9 +25,7 @@ export const DEFAULT_CONFIG: LiteParseConfig = {
preserveLayoutAlignmentAcrossPages: false,
};
-export function mergeConfig(
- userConfig: Partial<LiteParseConfig>
-): LiteParseConfig {
+export function mergeConfig(userConfig: Partial<LiteParseConfig>): LiteParseConfig {
return {
...DEFAULT_CONFIG,
...userConfig,
diff --git a/src/core/parser.ts b/src/core/parser.ts
index 6a89d8e..3046db3 100644
--- a/src/core/parser.ts
+++ b/src/core/parser.ts
@@ -1,19 +1,16 @@
-import { LiteParseConfig, ParseResult, ScreenshotResult } from './types.js';
-import { mergeConfig } from './config.js';
-import { PdfEngine } from '../engines/pdf/interface.js';
-import { PdfJsEngine } from '../engines/pdf/pdfjs.js';
-import { PdfiumRenderer } from '../engines/pdf/pdfium-renderer.js';
-import { OcrEngine } from '../engines/ocr/interface.js';
-import { TesseractEngine } from '../engines/ocr/tesseract.js';
-import { HttpOcrEngine } from '../engines/ocr/http-simple.js';
-import { projectPagesToGrid } from '../processing/grid.js';
-import { buildBoundingBoxes } from '../processing/bbox.js';
-import { detectTables } from '../processing/tables.js';
-import { formatJSON } from '../output/json.js';
-import {
- convertToPdf,
- cleanupConversionFiles,
-} from '../conversion/convertToPdf.js';
+import { LiteParseConfig, ParseResult, ScreenshotResult } from "./types.js";
+import { mergeConfig } from "./config.js";
+import { PdfEngine } from "../engines/pdf/interface.js";
+import { PdfJsEngine } from "../engines/pdf/pdfjs.js";
+import { PdfiumRenderer } from "../engines/pdf/pdfium-renderer.js";
+import { OcrEngine } from "../engines/ocr/interface.js";
+import { TesseractEngine } from "../engines/ocr/tesseract.js";
+import { HttpOcrEngine } from "../engines/ocr/http-simple.js";
+import { projectPagesToGrid } from "../processing/grid.js";
+import { buildBoundingBoxes } from "../processing/bbox.js";
+import { detectTables } from "../processing/tables.js";
+import { formatJSON } from "../output/json.js";
+import { convertToPdf, cleanupConversionFiles } from "../conversion/convertToPdf.js";
export class LiteParse {
private config: LiteParseConfig;
@@ -25,7 +22,7 @@ export class LiteParse {
this.config = mergeConfig(userConfig);
// Initialize PDF engine
- this.pdfEngine = new PdfJsEngine()
+ this.pdfEngine = new PdfJsEngine();
// Initialize OCR engine
// Auto-detect: use HTTP OCR if URL provided, otherwise use Tesseract
@@ -53,13 +50,13 @@ export class LiteParse {
log(`Processing file: ${filePath}`);
const conversionResult = await convertToPdf(filePath);
- if ('code' in conversionResult) {
+ if ("code" in conversionResult) {
// Conversion error
throw new Error(`Conversion failed: ${conversionResult.message}`);
}
// Return early for text-based passthrough formats
- if ('content' in conversionResult) {
+ if ("content" in conversionResult) {
log(`File is a text-based format. Returning content directly.`);
return {
pages: [],
@@ -114,13 +111,13 @@ export class LiteParse {
}
// Build final text
- const fullText = processedPages.map((p) => p.text).join('\n\n');
+ const fullText = processedPages.map((p) => p.text).join("\n\n");
// Close PDF document
await this.pdfEngine.close(doc);
// Cleanup OCR engine if it's Tesseract (to free memory)
- if (this.ocrEngine && 'terminate' in this.ocrEngine) {
+ if (this.ocrEngine && "terminate" in this.ocrEngine) {
await (this.ocrEngine as TesseractEngine).terminate();
}
@@ -136,10 +133,10 @@ export class LiteParse {
// Format based on output format
switch (this.config.outputFormat) {
- case 'json':
+ case "json":
result.json = JSON.parse(formatJSON(result));
break;
- case 'text':
+ case "text":
// Already in text format
break;
}
@@ -179,11 +176,7 @@ export class LiteParse {
}
log(`Rendering page ${pageNum}...`);
- const imageBuffer = await renderer.renderPageToBuffer(
- filePath,
- pageNum,
- this.config.dpi
- );
+ const imageBuffer = await renderer.renderPageToBuffer(filePath, pageNum, this.config.dpi);
// Get page dimensions
const pageData = await this.pdfEngine.extractPage(doc, pageNum);
@@ -208,14 +201,10 @@ export class LiteParse {
/**
* Run OCR on pages that need it
*/
- private async runOCR(
- doc: any,
- pages: any[],
- log: (msg: string) => void
- ): Promise<void> {
+ private async runOCR(doc: any, pages: any[], log: (msg: string) => void): Promise<void> {
if (!this.ocrEngine) return;
- log('Running OCR on pages...');
+ log("Running OCR on pages...");
for (let i = 0; i < pages.length; i++) {
const page = pages[i];
@@ -237,14 +226,11 @@ export class LiteParse {
);
// Save temporary image file
- const fs = await import('fs/promises');
- const path = await import('path');
- const os = await import('os');
+ const fs = await import("fs/promises");
+ const path = await import("path");
+ const os = await import("os");
const tmpDir = os.tmpdir();
- const tmpImagePath = path.join(
- tmpDir,
- `page_${page.pageNum}_ocr.png`
- );
+ const tmpImagePath = path.join(tmpDir, `page_${page.pageNum}_ocr.png`);
await fs.writeFile(tmpImagePath, imageBuffer);
// Run OCR
@@ -264,16 +250,14 @@ export class LiteParse {
y: r.bbox[1],
width: r.bbox[2] - r.bbox[0],
height: r.bbox[3] - r.bbox[1],
- fontName: 'OCR',
+ fontName: "OCR",
fontSize: r.bbox[3] - r.bbox[1],
fromOcr: true,
}));
// Add OCR text items directly to page textItems
page.textItems.push(...ocrTextItems);
- log(
- ` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`
- );
+ log(` Found ${ocrTextItems.length} text items from OCR on page ${page.pageNum}`);
}
// Clean up temp file
diff --git a/src/core/types.ts b/src/core/types.ts
index 5b47aea..4b174a5 100644
--- a/src/core/types.ts
+++ b/src/core/types.ts
@@ -1,6 +1,6 @@
-import { DetectedTable } from '../processing/tables.js';
+import { DetectedTable } from "../processing/tables.js";
-export type OutputFormat = 'json' | 'text';
+export type OutputFormat = "json" | "text";
export interface LiteParseConfig {
// OCR
@@ -34,8 +34,8 @@ export interface TextItem {
y: number;
width: number;
height: number;
- w: number; // Alias for width
- h: number; // Alias for height
+ w: number; // Alias for width
+ h: number; // Alias for height
fontName?: string;
fontSize?: number;
r?: number; // Rotation angle in degrees (0, 90, 180, 270)
@@ -70,7 +70,7 @@ export interface ProjectionTextBox {
fromOCR?: boolean;
// Projection metadata
- snap?: 'left' | 'right' | 'center';
+ snap?: "left" | "right" | "center";
leftAnchor?: string;
rightAnchor?: string;
centerAnchor?: string;
diff --git a/src/engines/ocr/http-simple.ts b/src/engines/ocr/http-simple.ts
index c5c9350..ee87ee8 100644
--- a/src/engines/ocr/http-simple.ts
+++ b/src/engines/ocr/http-simple.ts
@@ -1,7 +1,7 @@
-import axios from 'axios';
-import FormData from 'form-data';
-import fs from 'fs';
-import { OcrEngine, OcrOptions, OcrResult } from './interface.js';
+import axios from "axios";
+import FormData from "form-data";
+import fs from "fs";
+import { OcrEngine, OcrOptions, OcrResult } from "./interface.js";
/**
* HTTP-based OCR engine that conforms to LiteParse OCR API specification.
@@ -14,26 +14,21 @@ import { OcrEngine, OcrOptions, OcrResult } from './interface.js';
* See ocr/easyocr/ and ocr/paddleocr/ for example server implementations.
*/
export class HttpOcrEngine implements OcrEngine {
- name = 'http-ocr';
+ name = "http-ocr";
private serverUrl: string;
constructor(serverUrl: string) {
this.serverUrl = serverUrl;
}
- async recognize(
- imagePath: string,
- options: OcrOptions
- ): Promise<OcrResult[]> {
+ async recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]> {
try {
// Prepare request
const formData = new FormData();
- formData.append('file', fs.createReadStream(imagePath));
+ formData.append("file", fs.createReadStream(imagePath));
- const language = Array.isArray(options.language)
- ? options.language[0]
- : options.language;
- formData.append('language', language);
+ const language = Array.isArray(options.language) ? options.language[0] : options.language;
+ formData.append("language", language);
// Make HTTP request
const response = await axios.post(this.serverUrl, formData, {
@@ -45,7 +40,7 @@ export class HttpOcrEngine implements OcrEngine {
const { results } = response.data;
if (!Array.isArray(results)) {
- console.warn('OCR server response missing results array:', response.data);
+ console.warn("OCR server response missing results array:", response.data);
return [];
}
@@ -70,10 +65,7 @@ export class HttpOcrEngine implements OcrEngine {
}
}
- async recognizeBatch(
- imagePaths: string[],
- options: OcrOptions
- ): Promise<OcrResult[][]> {
+ async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {
const results: OcrResult[][] = [];
for (const imagePath of imagePaths) {
const result = await this.recognize(imagePath, options);
diff --git a/src/engines/ocr/interface.ts b/src/engines/ocr/interface.ts
index 1d2c44c..5759084 100644
--- a/src/engines/ocr/interface.ts
+++ b/src/engines/ocr/interface.ts
@@ -1,10 +1,7 @@
export interface OcrEngine {
name: string;
recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]>;
- recognizeBatch(
- imagePaths: string[],
- options: OcrOptions
- ): Promise<OcrResult[][]>;
+ recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]>;
}
export interface OcrOptions {
diff --git a/src/engines/ocr/tesseract.ts b/src/engines/ocr/tesseract.ts
index 684822b..793d76a 100644
--- a/src/engines/ocr/tesseract.ts
+++ b/src/engines/ocr/tesseract.ts
@@ -1,13 +1,12 @@
-import { createWorker, Worker } from 'tesseract.js';
-import { OcrEngine, OcrOptions, OcrResult } from './interface.js';
-
+import { createWorker, Worker } from "tesseract.js";
+import { OcrEngine, OcrOptions, OcrResult } from "./interface.js";
export class TesseractEngine implements OcrEngine {
- name = 'tesseract';
+ name = "tesseract";
private worker?: Worker;
private currentLanguage?: string;
- async initialize(language: string = 'eng'): Promise<void> {
+ async initialize(language: string = "eng"): Promise<void> {
if (this.worker && this.currentLanguage === language) {
return; // Already initialized for this language
}
@@ -21,10 +20,7 @@ export class TesseractEngine implements OcrEngine {
this.currentLanguage = language;
}
- async recognize(
- imagePath: string,
- options: OcrOptions
- ): Promise<OcrResult[]> {
+ async recognize(imagePath: string, options: OcrOptions): Promise<OcrResult[]> {
// Handle language - tesseract.js uses language codes like 'eng', 'fra', 'deu'
const language = this.normalizeLanguage(
Array.isArray(options.language) ? options.language[0] : options.language
@@ -34,7 +30,7 @@ export class TesseractEngine implements OcrEngine {
await this.initialize(language);
if (!this.worker) {
- throw new Error('Tesseract worker not initialized');
+ throw new Error("Tesseract worker not initialized");
}
try {
@@ -46,12 +42,12 @@ export class TesseractEngine implements OcrEngine {
// Convert to our OcrResult format
const results: OcrResult[] = words.map((word) => ({
text: word.text,
- bbox: [
- word.bbox.x0,
- word.bbox.y0,
- word.bbox.x1,
- word.bbox.y1,
- ] as [number, number, number, number],
+ bbox: [word.bbox.x0, word.bbox.y0, word.bbox.x1, word.bbox.y1] as [
+ number,
+ number,
+ number,
+ number,
+ ],
confidence: word.confidence / 100, // Tesseract returns 0-100, we want 0-1
}));
@@ -63,10 +59,7 @@ export class TesseractEngine implements OcrEngine {
}
}
- async recognizeBatch(
- imagePaths: string[],
- options: OcrOptions
- ): Promise<OcrResult[][]> {
+ async recognizeBatch(imagePaths: string[], options: OcrOptions): Promise<OcrResult[][]> {
const results: OcrResult[][] = [];
for (let i = 0; i < imagePaths.length; i++) {
@@ -92,22 +85,22 @@ export class TesseractEngine implements OcrEngine {
*/
private normalizeLanguage(lang: string): string {
const languageMap: Record<string, string> = {
- en: 'eng',
- fr: 'fra',
- de: 'deu',
- es: 'spa',
- it: 'ita',
- pt: 'por',
- ru: 'rus',
- zh: 'chi_sim',
- 'zh-cn': 'chi_sim',
- 'zh-tw': 'chi_tra',
- ja: 'jpn',
- ko: 'kor',
- ar: 'ara',
- hi: 'hin',
- th: 'tha',
- vi: 'vie',
+ en: "eng",
+ fr: "fra",
+ de: "deu",
+ es: "spa",
+ it: "ita",
+ pt: "por",
+ ru: "rus",
+ zh: "chi_sim",
+ "zh-cn": "chi_sim",
+ "zh-tw": "chi_tra",
+ ja: "jpn",
+ ko: "kor",
+ ar: "ara",
+ hi: "hin",
+ th: "tha",
+ vi: "vie",
};
const normalized = lang.toLowerCase().trim();
diff --git a/src/engines/pdf/interface.ts b/src/engines/pdf/interface.ts
index 5c8c034..d7d86cd 100644
--- a/src/engines/pdf/interface.ts
+++ b/src/engines/pdf/interface.ts
@@ -1,19 +1,11 @@
-import { TextItem } from '../../core/types.js';
+import { TextItem } from "../../core/types.js";
export interface PdfEngine {
name: string;
loadDocument(filePath: string): Promise<PdfDocument>;
extractPage(doc: PdfDocument, pageNum: number): Promise<PageData>;
- extractAllPages(
- doc: PdfDocument,
- maxPages?: number,
- targetPages?: string
- ): Promise<PageData[]>;
- renderPageImage(
- doc: PdfDocument,
- pageNum: number,
- dpi: number
- ): Promise<Buffer>;
+ extractAllPages(doc: PdfDocument, maxPages?: number, targetPages?: string): Promise<PageData[]>;
+ renderPageImage(doc: PdfDocument, pageNum: number, dpi: number): Promise<Buffer>;
close(doc: PdfDocument): Promise<void>;
}
@@ -34,7 +26,7 @@ export interface PageData {
}
export interface Path {
- type: 'rectangle' | 'line' | 'curve';
+ type: "rectangle" | "line" | "curve";
points: number[][];
color?: string;
width?: number;
@@ -65,7 +57,7 @@ export interface Image {
export type EasyOcrResultLine = [
[[number, number], [number, number], [number, number], [number, number]],
string,
- string | number
+ string | number,
];
export interface Annotation {
diff --git a/src/engines/pdf/pdfium-renderer.ts b/src/engines/pdf/pdfium-renderer.ts
index 89d4719..e5c0705 100644
--- a/src/engines/pdf/pdfium-renderer.ts
+++ b/src/engines/pdf/pdfium-renderer.ts
@@ -1,6 +1,6 @@
-import { PDFiumLibrary, type PDFiumPageRenderOptions } from '@hyzyla/pdfium';
-import sharp from 'sharp';
-import { promises as fs } from 'fs';
+import { PDFiumLibrary, type PDFiumPageRenderOptions } from "@hyzyla/pdfium";
+import sharp from "sharp";
+import { promises as fs } from "fs";
/**
* PDFium-based PDF screenshot renderer
@@ -23,7 +23,7 @@ export class PdfiumRenderer {
await this.init();
if (!this.pdfium) {
- throw new Error('PDFium not initialized');
+ throw new Error("PDFium not initialized");
}
// Read PDF file
diff --git a/src/engines/pdf/pdfjs.ts b/src/engines/pdf/pdfjs.ts
index 83db1d2..786a756 100644
--- a/src/engines/pdf/pdfjs.ts
+++ b/src/engines/pdf/pdfjs.ts
@@ -1,22 +1,15 @@
-import fs from 'node:fs/promises';
-import { fileURLToPath } from 'node:url';
-import { dirname, join } from 'node:path';
-import {
- PdfEngine,
- PdfDocument,
- PageData,
- Path,
- Image,
- Annotation,
-} from './interface.js';
-import { TextItem } from '../../core/types.js';
-import { PdfiumRenderer } from './pdfium-renderer.js';
+import fs from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+import { PdfEngine, PdfDocument, PageData, Path, Image, Annotation } from "./interface.js";
+import { TextItem } from "../../core/types.js";
+import { PdfiumRenderer } from "./pdfium-renderer.js";
// Dynamic import of PDF.js
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// From dist/src/engines/pdf/ we need to go up to dist/src/vendor/pdfjs
-const PDFJS_DIR = join(__dirname, '../../vendor/pdfjs');
+const PDFJS_DIR = join(__dirname, "../../vendor/pdfjs");
// Import PDF.js dynamically
await import(`${PDFJS_DIR}/pdf.mjs`);
@@ -45,17 +38,20 @@ function multiplyMatrices(m1: number[], m2: number[]): number[] {
m1[0] * m2[2] + m1[2] * m2[3],
m1[1] * m2[2] + m1[3] * m2[3],
m1[0] * m2[4] + m1[2] * m2[5] + m1[4],
- m1[1] * m2[4] + m1[3] * m2[5] + m1[5]
+ m1[1] * m2[4] + m1[3] * m2[5] + m1[5],
];
}
/**
* Apply transformation matrix to a point
*/
-function applyTransformation(point: { x: number; y: number }, transform: number[]): { x: number; y: number } {
+function applyTransformation(
+ point: { x: number; y: number },
+ transform: number[]
+): { x: number; y: number } {
return {
x: point.x * transform[0] + point.y * transform[2] + transform[4],
- y: point.x * transform[1] + point.y * transform[3] + transform[5]
+ y: point.x * transform[1] + point.y * transform[3] + transform[5],
};
}
@@ -85,11 +81,11 @@ function singularValueDecompose2dScale(m: number[]): { x: number; y: number } {
// Pre-compiled regex patterns for string decoding (avoid recompilation per item)
const BUGGY_FONT_MARKER_REGEX = /:->|>_(\d+)_\d+_<|<-:/g;
-const BUGGY_FONT_MARKER_CHECK = ':->|>';
+const BUGGY_FONT_MARKER_CHECK = ":->|>";
const PIPE_PATTERN_REGEX = /\s*\|([^|])\|\s*/g;
export class PdfJsEngine implements PdfEngine {
- name = 'pdfjs';
+ name = "pdfjs";
private pdfiumRenderer: PdfiumRenderer | null = null;
private currentPdfPath: string | null = null;
@@ -146,10 +142,7 @@ export class PdfJsEngine implements PdfEngine {
const scale = singularValueDecompose2dScale(item.transform);
// Get upper-right corner
- const ur = applyTransformation(
- { x: item.width / scale.x, y: item.height / scale.y },
- cm
- );
+ const ur = applyTransformation({ x: item.width / scale.x, y: item.height / scale.y }, cm);
// Calculate final bounding box in viewport space
const left = Math.min(ll.x, ur.x);
@@ -175,19 +168,18 @@ export class PdfJsEngine implements PdfEngine {
let decodedStr = item.str;
if (decodedStr.includes(BUGGY_FONT_MARKER_CHECK)) {
BUGGY_FONT_MARKER_REGEX.lastIndex = 0; // Reset regex state
- decodedStr = decodedStr.replace(
- BUGGY_FONT_MARKER_REGEX,
- (_: string, charCode: string) => String.fromCharCode(parseInt(charCode))
+ decodedStr = decodedStr.replace(BUGGY_FONT_MARKER_REGEX, (_: string, charCode: string) =>
+ String.fromCharCode(parseInt(charCode))
);
}
// Handle pipe-separated characters: " |a| |r| |X| " -> "arX"
// Some PDFs encode text with characters separated by pipes and spaces
- if (decodedStr.includes('|')) {
+ if (decodedStr.includes("|")) {
PIPE_PATTERN_REGEX.lastIndex = 0; // Reset regex state
const matches = [...decodedStr.matchAll(PIPE_PATTERN_REGEX)];
if (matches.length > 0) {
- decodedStr = matches.map(m => m[1]).join('');
+ decodedStr = matches.map((m) => m[1]).join("");
}
}
@@ -233,10 +225,7 @@ export class PdfJsEngine implements PdfEngine {
maxPages?: number,
targetPages?: string
): Promise<PageData[]> {
- const numPages = Math.min(
- doc.numPages,
- maxPages || doc.numPages
- );
+ const numPages = Math.min(doc.numPages, maxPages || doc.numPages);
const pages: PageData[] = [];
@@ -259,25 +248,17 @@ export class PdfJsEngine implements PdfEngine {
return pages;
}
- async renderPageImage(
- _doc: PdfDocument,
- pageNum: number,
- dpi: number
- ): Promise<Buffer> {
+ async renderPageImage(_doc: PdfDocument, pageNum: number, dpi: number): Promise<Buffer> {
// Use PDFium for rendering (more robust with inline images)
if (!this.currentPdfPath) {
- throw new Error('PDF path not available for rendering');
+ throw new Error("PDF path not available for rendering");
}
if (!this.pdfiumRenderer) {
this.pdfiumRenderer = new PdfiumRenderer();
}
- return await this.pdfiumRenderer.renderPageToBuffer(
- this.currentPdfPath,
- pageNum,
- dpi
- );
+ return await this.pdfiumRenderer.renderPageToBuffer(this.currentPdfPath, pageNum, dpi);
}
async close(doc: PdfDocument): Promise<void> {
@@ -296,13 +277,13 @@ export class PdfJsEngine implements PdfEngine {
private parseTargetPages(targetPages: string, maxPages: number): number[] {
const pages: number[] = [];
- const parts = targetPages.split(',');
+ const parts = targetPages.split(",");
for (const part of parts) {
const trimmed = part.trim();
- if (trimmed.includes('-')) {
+ if (trimmed.includes("-")) {
// Range: "1-5"
- const [start, end] = trimmed.split('-').map((n) => parseInt(n.trim()));
+ const [start, end] = trimmed.split("-").map((n) => parseInt(n.trim()));
for (let i = start; i <= Math.min(end, maxPages); i++) {
if (i >= 1) {
pages.push(i);
diff --git a/src/index.ts b/src/index.ts
index 7d9b3e9..662ecab 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,6 +1,6 @@
#!/usr/bin/env node
-import { program } from '../cli/parse.js';
+import { program } from "../cli/parse.js";
// Run the CLI
program.parse(process.argv);
diff --git a/src/lib.ts b/src/lib.ts
index b324b42..36d0d6a 100644
--- a/src/lib.ts
+++ b/src/lib.ts
@@ -1,4 +1,4 @@
-export { LiteParse } from './core/parser.js';
+export { LiteParse } from "./core/parser.js";
export type {
LiteParseConfig,
OutputFormat,
@@ -8,4 +8,4 @@ export type {
TextItem,
// ImageItem,
// TableItem
-} from './core/types.js';
+} from "./core/types.js";
diff --git a/src/output/json.ts b/src/output/json.ts
index ad3973a..2ee156f 100644
--- a/src/output/json.ts
+++ b/src/output/json.ts
@@ -1,4 +1,4 @@
-import { ParseResult, ParsedPage } from '../core/types.js';
+import { ParseResult, ParsedPage } from "../core/types.js";
/**
* Build JSON output from parsed pages
diff --git a/src/output/text.ts b/src/output/text.ts
index 680ca22..60d9567 100644
--- a/src/output/text.ts
+++ b/src/output/text.ts
@@ -1,4 +1,4 @@
-import { ParseResult, ParsedPage } from '../core/types.js';
+import { ParseResult, ParsedPage } from "../core/types.js";
/**
* Format pages as plain text
@@ -9,7 +9,7 @@ export function formatText(result: ParseResult): string {
return header + page.text;
});
- return pageTexts.join('\n\n');
+ return pageTexts.join("\n\n");
}
/**
diff --git a/src/processing/bbox.ts b/src/processing/bbox.ts
index 26165ef..b11eaaa 100644
--- a/src/processing/bbox.ts
+++ b/src/processing/bbox.ts
@@ -4,9 +4,9 @@ import {
ProjectionTextBox,
OcrData,
LiteParseConfig,
-} from '../core/types.js';
-import { PageData, Image } from '../engines/pdf/interface.js';
-import { parseImageOcrBlocks, OcrBlock } from './ocrUtils.js';
+} from "../core/types.js";
+import { PageData, Image } from "../engines/pdf/interface.js";
+import { parseImageOcrBlocks, OcrBlock } from "./ocrUtils.js";
const OCR_CONFIDENCE_THRESHOLD = 0.1;
@@ -46,8 +46,7 @@ export function filterImagesForOCR(
): Image[] {
// Filter images that start with g_ or pattern_ (generated/pattern images)
let filtered = images.filter(
- (image) =>
- !image.type?.startsWith('g_') && !image.type?.startsWith('pattern_')
+ (image) => !image.type?.startsWith("g_") && !image.type?.startsWith("pattern_")
);
// Limit to max images per page, keeping the largest ones
@@ -59,7 +58,7 @@ export function filterImagesForOCR(
// Apply additional filtering criteria
filtered = filtered.filter((image) => {
// Ignore layout extracted images
- if (image.type?.includes('layout_')) {
+ if (image.type?.includes("layout_")) {
return false;
}
@@ -173,10 +172,7 @@ function filterOcrBlocksOverlappingWithText(
totalOverlapArea += overlapArea;
// Condition 2: Reject if OCR block covers more than 50% of any single text item
- if (
- textItemArea > 0 &&
- overlapArea / textItemArea >= OCR_OVERLAP_THRESHOLD
- ) {
+ if (textItemArea > 0 && overlapArea / textItemArea >= OCR_OVERLAP_THRESHOLD) {
return false;
}
}
@@ -196,10 +192,7 @@ function filterOcrBlocksOverlappingWithText(
* Build projection text boxes from page data, including OCR results
* This is the complete implementation from buildBbox.ts
*/
-export function buildBbox(
- pageData: PageData,
- config: LiteParseConfig
-): ProjectionTextBox[] {
+export function buildBbox(pageData: PageData, config: LiteParseConfig): ProjectionTextBox[] {
const lines: ProjectionTextBox[] = [];
// Process all extracted text items
@@ -244,7 +237,6 @@ export function buildBbox(
}));
for (const image of imagesToProcess) {
-
// Parse OCR blocks from image
let ocrData = parseImageOcrBlocks(image);
@@ -258,8 +250,7 @@ export function buildBbox(
const ocrParsed: OcrData[] = [];
for (const block of ocrData) {
- const confidenceRounded =
- Math.round(parseFloat(block.confidence.toString()) * 1000) / 1000;
+ const confidenceRounded = Math.round(parseFloat(block.confidence.toString()) * 1000) / 1000;
const line: ProjectionTextBox = {
fromOCR: true,
@@ -305,7 +296,7 @@ export function buildBoundingBoxes(textItems: TextItem[]): BoundingBox[] {
const bboxes: BoundingBox[] = [];
for (const item of textItems) {
- if (item.str.trim() === '') {
+ if (item.str.trim() === "") {
continue;
}
diff --git a/src/processing/cleanText.ts b/src/processing/cleanText.ts
index 551ec6e..d651909 100644
--- a/src/processing/cleanText.ts
+++ b/src/processing/cleanText.ts
@@ -1,4 +1,4 @@
-import { ParsedPage, LiteParseConfig } from '../core/types.js';
+import { ParsedPage, LiteParseConfig } from "../core/types.js";
/**
* Detect and remove margins from a single page.
@@ -9,7 +9,7 @@ import { ParsedPage, LiteParseConfig } from '../core/types.js';
* - Right margin (trailing whitespace on each line)
*/
function detectAndRemoveMarginOnPage(page: ParsedPage): void {
- const lines = page.text.split('\n');
+ const lines = page.text.split("\n");
let minX: number | undefined = undefined;
let minY: number | undefined = undefined;
@@ -39,7 +39,7 @@ function detectAndRemoveMarginOnPage(page: ParsedPage): void {
// If page is entirely empty, just return
if (minX === undefined || minY === undefined || maxY === undefined) {
- page.text = '';
+ page.text = "";
return;
}
@@ -60,7 +60,7 @@ function detectAndRemoveMarginOnPage(page: ParsedPage): void {
newLines.push(line.slice(minX));
}
- page.text = newLines.join('\n');
+ page.text = newLines.join("\n");
}
/**
@@ -76,15 +76,12 @@ function detectAndRemoveMargin(pages: ParsedPage[]): void {
/**
* Clean raw text output - removes margins, null characters
*/
-export function cleanRawText(
- pages: ParsedPage[],
- _config: LiteParseConfig
-): void {
+export function cleanRawText(pages: ParsedPage[], _config: LiteParseConfig): void {
// Remove margins (per-page)
detectAndRemoveMargin(pages);
// Remove null characters
for (const page of pages) {
- page.text = page.text.replace(/\u0000/g, ' ');
+ page.text = page.text.replace(/\u0000/g, " ");
}
}
diff --git a/src/processing/grid.ts b/src/processing/grid.ts
index 8bb52b2..abb2747 100644
--- a/src/processing/grid.ts
+++ b/src/processing/grid.ts
@@ -1,7 +1,7 @@
-import { PageData } from '../engines/pdf/interface.js';
-import { ParsedPage, LiteParseConfig } from '../core/types.js';
-import { projectPagesToGrid as projectPagesToGridComplete } from './gridProjection.js';
-import { DEFAULT_CONFIG } from '../core/config.js';
+import { PageData } from "../engines/pdf/interface.js";
+import { ParsedPage, LiteParseConfig } from "../core/types.js";
+import { projectPagesToGrid as projectPagesToGridComplete } from "./gridProjection.js";
+import { DEFAULT_CONFIG } from "../core/config.js";
/**
* Projects text items onto a grid for spatial text extraction
diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.ts
index 2d47b5a..4b05817 100644
--- a/src/processing/gridProjection.ts
+++ b/src/processing/gridProjection.ts
@@ -1,1519 +1,1626 @@
-import { strToSubscriptString, strToPostScript } from './textUtils.js';
-import { buildBbox } from './bbox.js';
-import { cleanRawText } from './cleanText.js';
-import { ProjectionTextBox, Coordinates, LiteParseConfig, ParsedPage } from '../core/types.js';
-import { PageData } from '../engines/pdf/interface.js';
+import { strToSubscriptString, strToPostScript } from "./textUtils.js";
+import { buildBbox } from "./bbox.js";
+import { cleanRawText } from "./cleanText.js";
+import { ProjectionTextBox, Coordinates, LiteParseConfig, ParsedPage } from "../core/types.js";
+import { PageData } from "../engines/pdf/interface.js";
-import { getVerticalAndHorizontalLinesFromPagePaths, VerticalLine } from './pathUtils.js';
-import { applyMarkupTags } from './markupUtils.js';
+import { getVerticalAndHorizontalLinesFromPagePaths, VerticalLine } from "./pathUtils.js";
+import { applyMarkupTags } from "./markupUtils.js";
// Minimum spaces between unsnapped bboxes (likely justified text
const FLOATING_SPACES = 2;
// Minimum spaces between snapped columns
const COLUMN_SPACES = 4;
-
type Snap = {
- bbox: ProjectionTextBox;
- lineIndex: number;
- boxIndex: number;
-}
+ bbox: ProjectionTextBox;
+ lineIndex: number;
+ boxIndex: number;
+};
type Anchor = {
- [key: number]: ProjectionTextBox[];
-}
+ [key: number]: ProjectionTextBox[];
+};
type Anchors = {
- anchorLeft: Anchor;
- anchorRight: Anchor;
- anchorCenter: Anchor;
+ anchorLeft: Anchor;
+ anchorRight: Anchor;
+ anchorCenter: Anchor;
};
type ForwardAnchor = { [key: string]: number };
type PrevAnchors = {
- forwardAnchorLeft: ForwardAnchor;
- forwardAnchorCenter: ForwardAnchor;
- forwardAnchorRight: ForwardAnchor;
-}
+ forwardAnchorLeft: ForwardAnchor;
+ forwardAnchorCenter: ForwardAnchor;
+ forwardAnchorRight: ForwardAnchor;
+};
type PageForwardAnchors = {
- left: ForwardAnchor,
- right: ForwardAnchor,
- center: ForwardAnchor,
- floating: ForwardAnchor,
-}
+ left: ForwardAnchor;
+ right: ForwardAnchor;
+ center: ForwardAnchor;
+ floating: ForwardAnchor;
+};
type SnapMaps = {
- left: number[],
- right: number[],
- center: number[],
- floating: number[],
-}
+ left: number[];
+ right: number[];
+ center: number[];
+ floating: number[];
+};
type LineRange = {
- start: number,
- end: number,
-}
+ start: number;
+ end: number;
+};
type TextBoxSize = {
- width: number,
- height: number,
-}
+ width: number;
+ height: number;
+};
function roundAnchor(anchor: number): number {
- // group anchor x-coord by nearest 1/4 unit
- return Math.round(anchor * 4) / 4;
+ // group anchor x-coord by nearest 1/4 unit
+ return Math.round(anchor * 4) / 4;
}
// 2pt @ PDF 72 DPI -> 8px @ 300DPI
const SMALL_FONT_SIZE_THRESHOLD = 2;
function isSmallTextLine(line: ProjectionTextBox[]): boolean {
- // check for line where >50% of the text is very small
- const smallText = line.filter(item => item.h < SMALL_FONT_SIZE_THRESHOLD);
- if (smallText.length / line.length > .5) {
- return true;
- }
- return false;
+ // check for line where >50% of the text is very small
+ const smallText = line.filter((item) => item.h < SMALL_FONT_SIZE_THRESHOLD);
+ if (smallText.length / line.length > 0.5) {
+ return true;
+ }
+ return false;
}
-function filterUnprojectableText(config: LiteParseConfig, line: ProjectionTextBox[]): ProjectionTextBox[] {
- // extract all text (OSS always uses fast mode)
- if (line.length === 0) {
- return line;
- }
-
- let filteredLine = line;
- if (!config.preserveVerySmallText && isSmallTextLine(line)) {
- // remove very small text lines
- filteredLine = filteredLine.filter(item => item.h >= SMALL_FONT_SIZE_THRESHOLD);
- }
- return filteredLine;
+function filterUnprojectableText(
+ config: LiteParseConfig,
+ line: ProjectionTextBox[]
+): ProjectionTextBox[] {
+ // extract all text (OSS always uses fast mode)
+ if (line.length === 0) {
+ return line;
+ }
+
+ let filteredLine = line;
+ if (!config.preserveVerySmallText && isSmallTextLine(line)) {
+ // remove very small text lines
+ filteredLine = filteredLine.filter((item) => item.h >= SMALL_FONT_SIZE_THRESHOLD);
+ }
+ return filteredLine;
}
function canSnapLine(config: LiteParseConfig, line: ProjectionTextBox[]): boolean {
- // force lines that will likely break projection to be unsnapped floating text
- // currently this includes:
- // - lines of entirely small text
- //
- // NOTE: this assumes undesirable text has already been filtered before projection
- // (i.e. parse mode based removal of text should be done before this in filterUnprojectableText())
- if (line.length === 0) {
- return true;
- }
-
- if (!config.preserveVerySmallText && isSmallTextLine(line)) {
- return false;
- }
+ // force lines that will likely break projection to be unsnapped floating text
+ // currently this includes:
+ // - lines of entirely small text
+ //
+ // NOTE: this assumes undesirable text has already been filtered before projection
+ // (i.e. parse mode based removal of text should be done before this in filterUnprojectableText())
+ if (line.length === 0) {
return true;
+ }
+
+ if (!config.preserveVerySmallText && isSmallTextLine(line)) {
+ return false;
+ }
+ return true;
}
function fixSparseBlocks(blocks: LineRange[], rawLines: string[]) {
- // compress whitespace in blocks containing very sparse lines (>70% whitespace)
- const regexp = new RegExp(`\\s{${COLUMN_SPACES},}`, "g");
- for (const block of blocks) {
- let total = 0;
- let whitespace = 0;
- for (let i = block.start; i < block.end; ++i) {
- if (!rawLines[i]) {
- continue;
- }
- rawLines[i] = rawLines[i].trimEnd();
- const line = rawLines[i];
- if (line.length === 0) {
- continue;
- }
- total += line.length;
- whitespace += line.match(/\s/g)?.length || 0
- }
- if (total >= 500 && whitespace / total > 0.8) {
- for (let i = block.start; i < block.end; ++i) {
- const line = rawLines[i];
- if (!line || line.length === 0) {
- continue;
- }
- rawLines[i] = line.replace(regexp, " ".repeat(FLOATING_SPACES))
- }
+ // compress whitespace in blocks containing very sparse lines (>70% whitespace)
+ const regexp = new RegExp(`\\s{${COLUMN_SPACES},}`, "g");
+ for (const block of blocks) {
+ let total = 0;
+ let whitespace = 0;
+ for (let i = block.start; i < block.end; ++i) {
+ if (!rawLines[i]) {
+ continue;
+ }
+ rawLines[i] = rawLines[i].trimEnd();
+ const line = rawLines[i];
+ if (line.length === 0) {
+ continue;
+ }
+ total += line.length;
+ whitespace += line.match(/\s/g)?.length || 0;
+ }
+ if (total >= 500 && whitespace / total > 0.8) {
+ for (let i = block.start; i < block.end; ++i) {
+ const line = rawLines[i];
+ if (!line || line.length === 0) {
+ continue;
}
+ rawLines[i] = line.replace(regexp, " ".repeat(FLOATING_SPACES));
+ }
}
+ }
}
-function extractAnchorsPointsFromLines(lines: ProjectionTextBox[][], page: PageData, verticalLines: VerticalLine[]): Anchors {
- const pageHeight = page.height;
-
- let anchorLeft: Anchor = {};
- let anchorRight: Anchor = {};
- let anchorCenter: Anchor = {};
-
- // Add anchors from vertical lines to ensure horizontal gaps
- // This creates left and right anchors at vertical line positions
- for (let vLine of verticalLines) {
- const leftAnchor = roundAnchor(vLine.x);
- const rightAnchor = roundAnchor(vLine.x + vLine.width);
-
- if (!anchorLeft[leftAnchor]) {
- anchorLeft[leftAnchor] = [];
- }
- if (!anchorRight[rightAnchor]) {
- anchorRight[rightAnchor] = [];
- }
-
- // Create dummy bboxes spanning the vertical line height to establish the anchor
- const dummyBboxLeft = {
- x: vLine.x,
- y: vLine.y1,
- w: 0,
- h: vLine.height,
- str: '',
- strLength: 0
- } as ProjectionTextBox;
-
- const dummyBboxRight = {
- x: vLine.x + vLine.width,
- y: vLine.y1,
- w: 0,
- h: vLine.height,
- str: '',
- strLength: 0
- } as ProjectionTextBox;
-
- anchorLeft[leftAnchor].push(dummyBboxLeft);
- anchorRight[rightAnchor].push(dummyBboxRight);
+function extractAnchorsPointsFromLines(
+ lines: ProjectionTextBox[][],
+ page: PageData,
+ verticalLines: VerticalLine[]
+): Anchors {
+ const pageHeight = page.height;
+
+ const anchorLeft: Anchor = {};
+ const anchorRight: Anchor = {};
+ const anchorCenter: Anchor = {};
+
+ // Add anchors from vertical lines to ensure horizontal gaps
+ // This creates left and right anchors at vertical line positions
+ for (const vLine of verticalLines) {
+ const leftAnchor = roundAnchor(vLine.x);
+ const rightAnchor = roundAnchor(vLine.x + vLine.width);
+
+ if (!anchorLeft[leftAnchor]) {
+ anchorLeft[leftAnchor] = [];
}
-
- for (let line of lines) {
- for (let bbox of line) {
- let anchor = roundAnchor(bbox.x);
- if (!anchorLeft[anchor]) {
- anchorLeft[anchor] = [];
- }
- anchorLeft[anchor].push(bbox);
-
- anchor = roundAnchor(bbox.x + bbox.w);
- if (!anchorRight[anchor]) {
- anchorRight[anchor] = [];
- }
- anchorRight[anchor].push(bbox);
-
- let center = Math.round(bbox.x + bbox.w / 2);
- if (!anchorCenter[center]) {
- anchorCenter[center] = [];
- }
- anchorCenter[center].push(bbox);
- }
+ if (!anchorRight[rightAnchor]) {
+ anchorRight[rightAnchor] = [];
}
- function deltaMin(collection: Anchor, delta: number) {
- for (let anchor in collection) {
- let maxDelta = pageHeight * delta;
- for (let i = 0; i < collection[anchor].length; i++) {
- let shouldKeep = false;
- if (i > 0) {
- if (collection[anchor][i].y - collection[anchor][i - 1].y < maxDelta) {
- shouldKeep = true;
- }
- }
- if (i < collection[anchor].length - 1) {
- if (collection[anchor][i + 1].y - collection[anchor][i].y < maxDelta) {
- shouldKeep = true;
- }
- }
+ // Create dummy bboxes spanning the vertical line height to establish the anchor
+ const dummyBboxLeft = {
+ x: vLine.x,
+ y: vLine.y1,
+ w: 0,
+ h: vLine.height,
+ str: "",
+ strLength: 0,
+ } as ProjectionTextBox;
+
+ const dummyBboxRight = {
+ x: vLine.x + vLine.width,
+ y: vLine.y1,
+ w: 0,
+ h: vLine.height,
+ str: "",
+ strLength: 0,
+ } as ProjectionTextBox;
+
+ anchorLeft[leftAnchor].push(dummyBboxLeft);
+ anchorRight[rightAnchor].push(dummyBboxRight);
+ }
+
+ for (const line of lines) {
+ for (const bbox of line) {
+ let anchor = roundAnchor(bbox.x);
+ if (!anchorLeft[anchor]) {
+ anchorLeft[anchor] = [];
+ }
+ anchorLeft[anchor].push(bbox);
+
+ anchor = roundAnchor(bbox.x + bbox.w);
+ if (!anchorRight[anchor]) {
+ anchorRight[anchor] = [];
+ }
+ anchorRight[anchor].push(bbox);
+
+ const center = Math.round(bbox.x + bbox.w / 2);
+ if (!anchorCenter[center]) {
+ anchorCenter[center] = [];
+ }
+ anchorCenter[center].push(bbox);
+ }
+ }
+
+ function deltaMin(collection: Anchor, delta: number) {
+ for (const anchor in collection) {
+ const maxDelta = pageHeight * delta;
+ for (let i = 0; i < collection[anchor].length; i++) {
+ let shouldKeep = false;
+ if (i > 0) {
+ if (collection[anchor][i].y - collection[anchor][i - 1].y < maxDelta) {
+ shouldKeep = true;
+ }
+ }
+ if (i < collection[anchor].length - 1) {
+ if (collection[anchor][i + 1].y - collection[anchor][i].y < maxDelta) {
+ shouldKeep = true;
+ }
+ }
- if (!shouldKeep) {
- collection[anchor].splice(i--, 1);
- }
- }
+ if (!shouldKeep) {
+ collection[anchor].splice(i--, 1);
}
+ }
}
-
- deltaMin(anchorRight, 0.1);
- deltaMin(anchorLeft, 0.2);
- deltaMin(anchorCenter, 0.05);
-
- function intercept(collection: Anchor) {
- for (let anchor in collection) {
- let shouldKeep = false;
- for (let i = 0; i < collection[anchor].length; i++) {
- if (i > 0) {
- let intercept = false;
- // check intercept
- let a1 = collection[anchor][i - 1];
- let a2 = collection[anchor][i];
- for (let line of lines) {
- if (line.length > 0 && line[0].y > a1.y && line[0].y < a2.y) {
- for (let item of line) {
- if (item.x < parseInt(anchor) && item.x + item.w > parseInt(anchor)) {
- intercept = true;
- break;
- }
- }
- if (intercept) {
- break;
- }
- }
- }
- if (!intercept) {
- shouldKeep = true;
- break;
- }
+ }
+
+ deltaMin(anchorRight, 0.1);
+ deltaMin(anchorLeft, 0.2);
+ deltaMin(anchorCenter, 0.05);
+
+ function intercept(collection: Anchor) {
+ for (const anchor in collection) {
+ let shouldKeep = false;
+ for (let i = 0; i < collection[anchor].length; i++) {
+ if (i > 0) {
+ let intercept = false;
+ // check intercept
+ const a1 = collection[anchor][i - 1];
+ const a2 = collection[anchor][i];
+ for (const line of lines) {
+ if (line.length > 0 && line[0].y > a1.y && line[0].y < a2.y) {
+ for (const item of line) {
+ if (item.x < parseInt(anchor) && item.x + item.w > parseInt(anchor)) {
+ intercept = true;
+ break;
}
+ }
+ if (intercept) {
+ break;
+ }
}
- if (!shouldKeep) {
- delete collection[anchor];
- }
+ }
+ if (!intercept) {
+ shouldKeep = true;
+ break;
+ }
}
+ }
+ if (!shouldKeep) {
+ delete collection[anchor];
+ }
}
+ }
+
+ intercept(anchorRight);
+ intercept(anchorLeft);
+ intercept(anchorCenter);
+
+ function group(collection: Anchor) {
+ for (const anchor in collection) {
+ const anchorNum = parseFloat(anchor);
+ // merge right
+ if (
+ collection[anchorNum + 1] &&
+ collection[anchorNum + 1].length > collection[anchorNum].length
+ ) {
+ collection[anchorNum + 1].push(...collection[anchorNum]);
+ delete collection[anchorNum];
+ }
+ // merge left
+ else if (
+ collection[anchorNum - 1] &&
+ collection[anchorNum - 1].length > collection[anchorNum].length
+ ) {
+ collection[anchorNum - 1].push(...collection[anchorNum]);
+ delete collection[anchorNum];
+ }
+ }
+ }
+
+ group(anchorLeft);
+ group(anchorRight);
+ group(anchorCenter);
+
+ function anyAnchorSurvived(bbox: ProjectionTextBox) {
+ return (
+ roundAnchor(bbox.x) in anchorLeft ||
+ roundAnchor(bbox.x + bbox.w) in anchorRight ||
+ Math.round(bbox.x + bbox.w / 2) in anchorCenter
+ );
+ }
+
+ // Try seeing if a floating bbox would align well with a surviving anchor on a line immediately above or below it
+ function tryAlignFloating(
+ collection: Anchor,
+ ANCHOR_MARGIN: number,
+ refXFromBbox: (bbox: ProjectionTextBox) => number,
+ anchorValFromBbox: (bbox: ProjectionTextBox) => number
+ ) {
+ for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
+ const line = lines[lineIndex];
+ for (const bbox of line) {
+ // Only consider floating bboxes
+ if (anyAnchorSurvived(bbox)) {
+ continue;
+ }
+ // Check the lines before and after
+ const candidateLines: ProjectionTextBox[][] = [];
+ if (lineIndex > 0) {
+ candidateLines.push(lines[lineIndex - 1]);
+ }
+ if (lineIndex < lines.length - 1) {
+ candidateLines.push(lines[lineIndex + 1]);
+ }
- intercept(anchorRight);
- intercept(anchorLeft);
- intercept(anchorCenter);
-
- function group(collection: Anchor) {
- for (let anchor in collection) {
- const anchorNum = parseFloat(anchor);
- // merge right
- if (collection[anchorNum + 1] && collection[anchorNum + 1].length > collection[anchorNum].length) {
- collection[anchorNum + 1].push(...collection[anchorNum]);
- delete collection[anchorNum];
+ // Check candidate lines for:
+ // Possible alignment
+ // Being within the margin
+ // Being the closest of the candidates
+ let candidateAnchor: string = "";
+ let prevDiff = ANCHOR_MARGIN + 1;
+ for (const candLine of candidateLines) {
+ for (const candBBox of candLine) {
+ const candAnchorVal = anchorValFromBbox(candBBox);
+ if (!(candAnchorVal in collection)) {
+ continue;
}
- // merge left
- else if (collection[anchorNum - 1] && collection[anchorNum - 1].length > collection[anchorNum].length) {
- collection[anchorNum - 1].push(...collection[anchorNum]);
- delete collection[anchorNum];
+ const xDiff = Math.abs(candAnchorVal - refXFromBbox(bbox));
+ if (xDiff <= ANCHOR_MARGIN && xDiff < prevDiff) {
+ candidateAnchor = candAnchorVal.toString();
+ prevDiff = xDiff;
}
+ }
}
- }
-
- group(anchorLeft);
- group(anchorRight);
- group(anchorCenter);
+ // No candidate found
+ if (candidateAnchor.length == 0) {
+ continue;
+ }
- function anyAnchorSurvived(bbox: ProjectionTextBox) {
- return roundAnchor(bbox.x) in anchorLeft ||
- roundAnchor(bbox.x + bbox.w) in anchorRight ||
- Math.round(bbox.x + bbox.w / 2) in anchorCenter;
+ // Candidate found - update the anchor's bbox list
+ collection[parseFloat(candidateAnchor)].push(bbox);
+ }
}
-
- // Try seeing if a floating bbox would align well with a surviving anchor on a line immediately above or below it
- function tryAlignFloating(collection: Anchor, ANCHOR_MARGIN: number, refXFromBbox: (bbox: ProjectionTextBox) => number, anchorValFromBbox: (bbox: ProjectionTextBox) => number) {
- for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
- const line = lines[lineIndex];
- for (let bbox of line) {
- // Only consider floating bboxes
- if (anyAnchorSurvived(bbox)) {
- continue;
- }
- // Check the lines before and after
- let candidateLines: ProjectionTextBox[][] = [];
- if (lineIndex > 0) {
- candidateLines.push(lines[lineIndex-1]);
- }
- if (lineIndex < lines.length-1) {
- candidateLines.push(lines[lineIndex+1]);
- }
-
- // Check candidate lines for:
- // Possible alignment
- // Being within the margin
- // Being the closest of the candidates
- let candidateAnchor: string = "";
- let prevDiff = ANCHOR_MARGIN + 1;
- for (let candLine of candidateLines) {
- for (let candBBox of candLine) {
- const candAnchorVal = anchorValFromBbox(candBBox);
- if (!(candAnchorVal in collection)) {
- continue;
- }
- const xDiff = Math.abs(candAnchorVal - refXFromBbox(bbox));
- if (xDiff <= ANCHOR_MARGIN && xDiff < prevDiff) {
- candidateAnchor = candAnchorVal.toString();
- prevDiff = xDiff;
- }
- }
- }
-
- // No candidate found
- if (candidateAnchor.length == 0) {
- continue;
- }
-
- // Candidate found - update the anchor's bbox list
- collection[parseFloat(candidateAnchor)].push(bbox);
- }
- }
+ }
+
+ // Try to left-align floating bboxes
+ tryAlignFloating(
+ anchorLeft,
+ 2,
+ (bbox) => bbox.x,
+ (bbox) => roundAnchor(bbox.x)
+ );
+
+ // Sort the anchors' lists of bboxes by y-value
+ function sortAnchor(collection: Anchor) {
+ for (const anchor in collection) {
+ collection[anchor].sort((a, b) => a.y - b.y);
}
-
- // Try to left-align floating bboxes
- tryAlignFloating(anchorLeft, 2, (bbox) => bbox.x, (bbox) => roundAnchor(bbox.x));
-
- // Sort the anchors' lists of bboxes by y-value
- function sortAnchor(collection: Anchor) {
- for (let anchor in collection) {
- collection[anchor].sort((a, b) => a.y - b.y)
- }
+ }
+
+ sortAnchor(anchorLeft);
+ sortAnchor(anchorRight);
+ sortAnchor(anchorCenter);
+
+ // deduplicate
+ const duplicates = [];
+ for (const anchor in anchorLeft) {
+ for (const item of anchorLeft[anchor]) {
+ item.snap = "left";
+ item.leftAnchor = anchor;
}
-
- sortAnchor(anchorLeft);
- sortAnchor(anchorRight);
- sortAnchor(anchorCenter);
-
-
- // deduplicate
- let duplicates = [];
- for (let anchor in anchorLeft) {
- for (let item of anchorLeft[anchor]) {
- item.snap = 'left';
- item.leftAnchor = anchor;
- }
+ }
+
+ for (const anchor in anchorRight) {
+ for (const item of anchorRight[anchor]) {
+ if (item.snap) {
+ item.isDup = true;
+ duplicates.push(item);
+ }
+ item.snap = "right";
+ item.rightAnchor = anchor;
}
-
- for (let anchor in anchorRight) {
- for (let item of anchorRight[anchor]) {
- if (item.snap) {
- item.isDup = true;
- duplicates.push(item);
- }
- item.snap = 'right';
- item.rightAnchor = anchor;
- }
+ }
+
+ for (const anchor in anchorCenter) {
+ for (const item of anchorCenter[anchor]) {
+ if (item.snap && !item.isDup) {
+ item.isDup = true;
+ duplicates.push(item);
+ }
+ item.snap = "center";
+ item.centerAnchor = anchor;
}
+ }
- for (let anchor in anchorCenter) {
- for (let item of anchorCenter[anchor]) {
- if (item.snap && !item.isDup) {
- item.isDup = true;
- duplicates.push(item);
- }
- item.snap = 'center';
- item.centerAnchor = anchor;
- }
+ function anchorCounts(item: ProjectionTextBox): number[] {
+ let leftCount = 0;
+ if (item.leftAnchor) {
+ const key = parseFloat(item.leftAnchor);
+ leftCount = anchorLeft[key] ? anchorLeft[key].length : 0;
}
-
- function anchorCounts(item: ProjectionTextBox): number[] {
- let leftCount = 0;
- if (item.leftAnchor) {
- const key = parseFloat(item.leftAnchor);
- leftCount = anchorLeft[key] ? anchorLeft[key].length : 0;
- }
- let rightCount = 0;
+ let rightCount = 0;
+ if (item.rightAnchor) {
+ const key = parseFloat(item.rightAnchor);
+ rightCount = anchorRight[key] ? anchorRight[key].length : 0;
+ }
+ let centerCount = 0;
+ if (item.centerAnchor) {
+ const key = parseFloat(item.centerAnchor);
+ centerCount = anchorCenter[key] ? anchorCenter[key].length : 0;
+ }
+ return [leftCount, rightCount, centerCount];
+ }
+
+ // find all left aligned blocks, all right aligned blocks, all centered blocks, in that order
+ // we cannot check all 3 at once since we may end up double counting potential anchor matches
+ // (i.e. we need to exclude block that we know are left/right aligned before counting possible
+ // matching centered blocks)
+
+ // find all lefts
+ let hasChanged = true;
+ while (hasChanged && duplicates.length > 0) {
+ hasChanged = false;
+ for (let i = duplicates.length - 1; i >= 0; --i) {
+ const item = duplicates[i];
+ const [leftCount, rightCount, centerCount] = anchorCounts(item);
+ if (leftCount >= rightCount && leftCount >= centerCount) {
+ item.snap = "left";
if (item.rightAnchor) {
- const key = parseFloat(item.rightAnchor);
- rightCount = anchorRight[key] ? anchorRight[key].length : 0;
+ const key = parseFloat(item.rightAnchor);
+ if (anchorRight[key]) {
+ anchorRight[key].splice(anchorRight[key].indexOf(item), 1);
+ hasChanged = true;
+ }
}
- let centerCount = 0;
if (item.centerAnchor) {
- const key = parseFloat(item.centerAnchor);
- centerCount = anchorCenter[key] ? anchorCenter[key].length : 0;
- }
- return [leftCount, rightCount, centerCount];
- }
-
- // find all left aligned blocks, all right aligned blocks, all centered blocks, in that order
- // we cannot check all 3 at once since we may end up double counting potential anchor matches
- // (i.e. we need to exclude block that we know are left/right aligned before counting possible
- // matching centered blocks)
-
- // find all lefts
- let hasChanged = true;
- while (hasChanged && duplicates.length > 0) {
- hasChanged = false;
- for (let i = duplicates.length - 1; i >= 0; --i) {
- const item = duplicates[i];
- const [leftCount, rightCount, centerCount] = anchorCounts(item);
- if (leftCount >= rightCount && leftCount >= centerCount) {
- item.snap = "left";
- if (item.rightAnchor) {
- const key = parseFloat(item.rightAnchor);
- if (anchorRight[key]) {
- anchorRight[key].splice(anchorRight[key].indexOf(item), 1);
- hasChanged = true;
- }
- }
- if (item.centerAnchor) {
- const key = parseFloat(item.centerAnchor);
- if (anchorCenter[key]) {
- anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);
- hasChanged = true;
- }
- }
- duplicates.splice(i, 1);
- }
- }
- }
-
- // find all rights
- hasChanged = true;
- while (hasChanged && duplicates.length > 0) {
- hasChanged = false;
- for (let i = duplicates.length - 1; i >= 0; --i) {
- const item = duplicates[i];
- const [leftCount, rightCount, centerCount] = anchorCounts(item);
- if (rightCount >= leftCount && rightCount >= centerCount) {
- item.snap = "right";
- if (item.leftAnchor) {
- const key = parseFloat(item.leftAnchor);
- if (anchorLeft[key]) {
- anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);
- hasChanged = true;
- }
- }
- if (item.centerAnchor) {
- const key = parseFloat(item.centerAnchor);
- if (anchorCenter[key]) {
- anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);
- hasChanged = true;
- }
- }
- duplicates.splice(i, 1);
- }
+ const key = parseFloat(item.centerAnchor);
+ if (anchorCenter[key]) {
+ anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);
+ hasChanged = true;
+ }
}
+ duplicates.splice(i, 1);
+ }
}
-
- // remaining duplicates are centered
- for (const item of duplicates) {
- item.snap = "center";
+ }
+
+ // find all rights
+ hasChanged = true;
+ while (hasChanged && duplicates.length > 0) {
+ hasChanged = false;
+ for (let i = duplicates.length - 1; i >= 0; --i) {
+ const item = duplicates[i];
+ const [leftCount, rightCount, centerCount] = anchorCounts(item);
+ if (rightCount >= leftCount && rightCount >= centerCount) {
+ item.snap = "right";
if (item.leftAnchor) {
- const key = parseFloat(item.leftAnchor);
- if (anchorLeft[key]) {
- anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);
- }
+ const key = parseFloat(item.leftAnchor);
+ if (anchorLeft[key]) {
+ anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);
+ hasChanged = true;
+ }
}
- if (item.rightAnchor) {
- const key = parseFloat(item.rightAnchor);
- if (anchorRight[key]) {
- anchorRight[key].splice(anchorRight[key].indexOf(item), 1);
- }
+ if (item.centerAnchor) {
+ const key = parseFloat(item.centerAnchor);
+ if (anchorCenter[key]) {
+ anchorCenter[key].splice(anchorCenter[key].indexOf(item), 1);
+ hasChanged = true;
+ }
}
+ duplicates.splice(i, 1);
+ }
}
-
- // filter anchors
- // delete singleton
- for (let anchor in anchorLeft) {
- if (anchorLeft[anchor].length < 2) {
- if (anchorLeft[anchor].length) {
- delete anchorLeft[anchor][0].snap;
- }
- delete anchorLeft[anchor];
- }
+ }
+
+ // remaining duplicates are centered
+ for (const item of duplicates) {
+ item.snap = "center";
+ if (item.leftAnchor) {
+ const key = parseFloat(item.leftAnchor);
+ if (anchorLeft[key]) {
+ anchorLeft[key].splice(anchorLeft[key].indexOf(item), 1);
+ }
}
-
- for (let anchor in anchorRight) {
- if (anchorRight[anchor].length < 2) {
- if (anchorRight[anchor].length) {
- delete anchorRight[anchor][0].snap;
- }
- delete anchorRight[anchor];
- }
+ if (item.rightAnchor) {
+ const key = parseFloat(item.rightAnchor);
+ if (anchorRight[key]) {
+ anchorRight[key].splice(anchorRight[key].indexOf(item), 1);
+ }
}
-
- for (let anchor in anchorCenter) {
- if (anchorCenter[anchor].length < 2) {
- if (anchorCenter[anchor].length) {
- delete anchorCenter[anchor][0].snap;
- }
- delete anchorCenter[anchor];
- }
+ }
+
+ // filter anchors
+ // delete singleton
+ for (const anchor in anchorLeft) {
+ if (anchorLeft[anchor].length < 2) {
+ if (anchorLeft[anchor].length) {
+ delete anchorLeft[anchor][0].snap;
+ }
+ delete anchorLeft[anchor];
}
-
- return {
- anchorLeft,
- anchorRight,
- anchorCenter
+ }
+
+ for (const anchor in anchorRight) {
+ if (anchorRight[anchor].length < 2) {
+ if (anchorRight[anchor].length) {
+ delete anchorRight[anchor][0].snap;
+ }
+ delete anchorRight[anchor];
}
-}
-
-
-function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: number) {
- // if no bbox is rotated (.r is set), return
- if (!textBbox.find(b => b.r != 0)) {
- return;
+ }
+
+ for (const anchor in anchorCenter) {
+ if (anchorCenter[anchor].length < 2) {
+ if (anchorCenter[anchor].length) {
+ delete anchorCenter[anchor][0].snap;
+ }
+ delete anchorCenter[anchor];
}
+ }
- textBbox.sort((a, b) => {
- return a.x - b.x;
- });
+ return {
+ anchorLeft,
+ anchorRight,
+ anchorCenter,
+ };
+}
- // Horizontal Axis
- let bboxGroup: ProjectionTextBox[][] = [];
- let currentGroup: ProjectionTextBox[] = [];
- let previousBbox: ProjectionTextBox | null = null;
- for (let bbox of textBbox) {
- if (previousBbox) {
- if (bbox.r == previousBbox.r) {
- currentGroup.push(bbox);
- }
- else {
- // sort currentGroup by bbox.y
- currentGroup.sort((a, b) => a.y - b.y);
- bboxGroup.push(currentGroup);
- currentGroup = [bbox];
- }
- }
- else {
- currentGroup.push(bbox);
- }
- previousBbox = bbox;
- }
- if (currentGroup.length) {
+function handleRotationReadingOrder(textBbox: ProjectionTextBox[], pageHeight: number) {
+ // if no bbox is rotated (.r is set), return
+ if (!textBbox.find((b) => b.r != 0)) {
+ return;
+ }
+
+ textBbox.sort((a, b) => {
+ return a.x - b.x;
+ });
+
+ // Horizontal Axis
+ let bboxGroup: ProjectionTextBox[][] = [];
+ let currentGroup: ProjectionTextBox[] = [];
+ let previousBbox: ProjectionTextBox | null = null;
+ for (const bbox of textBbox) {
+ if (previousBbox) {
+ if (bbox.r == previousBbox.r) {
+ currentGroup.push(bbox);
+ } else {
+ // sort currentGroup by bbox.y
currentGroup.sort((a, b) => a.y - b.y);
bboxGroup.push(currentGroup);
+ currentGroup = [bbox];
+ }
+ } else {
+ currentGroup.push(bbox);
}
-
- // NOTE/ WARNING: height and width of bbox are NOT rotated beforehand!
- for (let [index, group] of bboxGroup.entries()) {
- if (group[0].r == 90 || group[0].r == 270) {
- // check in all bbox if they are not group[0].r and have overlap in x axis
- let globalOverlap = false;
- for (let bbox of textBbox) {
- if (bbox.r != group[0].r) {
- let overlap = group.find(b => b.x >= bbox.x && b.x <= bbox.x + bbox.w && bbox.r != b.r);
- if (overlap) {
- globalOverlap = true;
- }
- }
- }
-
- if (globalOverlap) {
- // rotate bbox to be horizontal
- for (let bbox of group) {
- if (bbox.d) {
- bbox.y += bbox.d;
- bbox.d = 0;
- }
- bbox.r = 0;
- bbox.rotated = true;
- }
- }
- else if (!globalOverlap) {
- // insert the bbox group in the Y axis after previous group and before next group.
- // move Next group by current group height (width as not rotated yet).
- let groupMaxX = Math.max(...group.map(v => v.x + v.w));
-
- let deltaY = 0;
- if (index != 0) {
- let previousGroup = bboxGroup[index - 1];
- let previousGroupMaxY = Math.max(...previousGroup.map(v => v.y + v.h));
- // pageHeight is radical but garantie no issue of allignement
- deltaY = previousGroupMaxY + pageHeight;
- }
- // clockwize rotation
- if (group[0].r == 90) {
- for (let bbox of group) {
- // w not h here as bbox w/h are not flipped
- let x = Math.round(bbox.ry ?? bbox.y);
- let y = (bbox.rx ?? bbox.x) + deltaY;
- bbox.x = Math.round(x);
- bbox.y = y;
- bbox.r = 0;
- bbox.rotated = true;
- }
- }
- // counter clockwize rotation
- if (group[0].r == 270) {
- for (let bbox of group) {
- let x = Math.round(bbox.ry ?? bbox.y);
- let y = (bbox.rx ?? bbox.x) + deltaY;
- bbox.x = Math.round(x);
- bbox.y = y;
- bbox.r = 0;
- bbox.rotated = true;
- }
- }
-
- // pageHeight is radical but garantie no issue of allignement
- let globalDelta = deltaY + groupMaxX + pageHeight;
-
- for (let [otherGroupIndex, other] of bboxGroup.entries()) {
- if (otherGroupIndex <= index) {
- continue;
- }
- for (let bbox of other) {
- if (bbox.r == 90 || bbox.r == 270) {
- bbox.d = (bbox.d ? bbox.d : 0) + globalDelta;
- continue;
- }
- bbox.y += globalDelta;
- }
- }
- }
-
+ previousBbox = bbox;
+ }
+ if (currentGroup.length) {
+ currentGroup.sort((a, b) => a.y - b.y);
+ bboxGroup.push(currentGroup);
+ }
+
+ // NOTE/ WARNING: height and width of bbox are NOT rotated beforehand!
+ for (const [index, group] of bboxGroup.entries()) {
+ if (group[0].r == 90 || group[0].r == 270) {
+ // check in all bbox if they are not group[0].r and have overlap in x axis
+ let globalOverlap = false;
+ for (const bbox of textBbox) {
+ if (bbox.r != group[0].r) {
+ const overlap = group.find(
+ (b) => b.x >= bbox.x && b.x <= bbox.x + bbox.w && bbox.r != b.r
+ );
+ if (overlap) {
+ globalOverlap = true;
+ }
+ }
+ }
+
+ if (globalOverlap) {
+ // rotate bbox to be horizontal
+ for (const bbox of group) {
+ if (bbox.d) {
+ bbox.y += bbox.d;
+ bbox.d = 0;
+ }
+ bbox.r = 0;
+ bbox.rotated = true;
+ }
+ } else if (!globalOverlap) {
+ // insert the bbox group in the Y axis after previous group and before next group.
+ // move Next group by current group height (width as not rotated yet).
+ const groupMaxX = Math.max(...group.map((v) => v.x + v.w));
+
+ let deltaY = 0;
+ if (index != 0) {
+ const previousGroup = bboxGroup[index - 1];
+ const previousGroupMaxY = Math.max(...previousGroup.map((v) => v.y + v.h));
+ // pageHeight is radical but garantie no issue of allignement
+ deltaY = previousGroupMaxY + pageHeight;
+ }
+ // clockwize rotation
+ if (group[0].r == 90) {
+ for (const bbox of group) {
+ // w not h here as bbox w/h are not flipped
+ const x = Math.round(bbox.ry ?? bbox.y);
+ const y = (bbox.rx ?? bbox.x) + deltaY;
+ bbox.x = Math.round(x);
+ bbox.y = y;
+ bbox.r = 0;
+ bbox.rotated = true;
+ }
+ }
+ // counter clockwize rotation
+ if (group[0].r == 270) {
+ for (const bbox of group) {
+ const x = Math.round(bbox.ry ?? bbox.y);
+ const y = (bbox.rx ?? bbox.x) + deltaY;
+ bbox.x = Math.round(x);
+ bbox.y = y;
+ bbox.r = 0;
+ bbox.rotated = true;
+ }
}
- }
-
- textBbox.sort((a, b) => {
- return a.y - b.y;
- });
+ // pageHeight is radical but garantie no issue of allignement
+ const globalDelta = deltaY + groupMaxX + pageHeight;
- // Vertical Axis
- bboxGroup = [];
- currentGroup = [];
- previousBbox = null as ProjectionTextBox | null;
- for (let bbox of textBbox) {
- if (previousBbox) {
- if (bbox.r == previousBbox.r) {
- currentGroup.push(bbox);
- }
- else {
- // sort currentGroup by bbox.y
- currentGroup.sort((a, b) => a.x - b.x);
- bboxGroup.push(currentGroup);
- currentGroup = [bbox];
+ for (const [otherGroupIndex, other] of bboxGroup.entries()) {
+ if (otherGroupIndex <= index) {
+ continue;
+ }
+ for (const bbox of other) {
+ if (bbox.r == 90 || bbox.r == 270) {
+ bbox.d = (bbox.d ? bbox.d : 0) + globalDelta;
+ continue;
}
+ bbox.y += globalDelta;
+ }
}
- else {
- currentGroup.push(bbox);
- }
- previousBbox = bbox;
+ }
}
- if (currentGroup.length) {
+ }
+
+ textBbox.sort((a, b) => {
+ return a.y - b.y;
+ });
+
+ // Vertical Axis
+ bboxGroup = [];
+ currentGroup = [];
+ previousBbox = null as ProjectionTextBox | null;
+ for (const bbox of textBbox) {
+ if (previousBbox) {
+ if (bbox.r == previousBbox.r) {
+ currentGroup.push(bbox);
+ } else {
+ // sort currentGroup by bbox.y
currentGroup.sort((a, b) => a.x - b.x);
bboxGroup.push(currentGroup);
+ currentGroup = [bbox];
+ }
+ } else {
+ currentGroup.push(bbox);
}
-
-
- for (const group of bboxGroup) {
- if (group[0].r == 180) {
- // Switch upside down
- for (let bbox of group) {
- bbox.x = Math.round(bbox.ry ?? bbox.y);
- bbox.y = bbox.rx ?? bbox.x;
- bbox.r = 0;
- bbox.rotated = true;
- }
- }
+ previousBbox = bbox;
+ }
+ if (currentGroup.length) {
+ currentGroup.sort((a, b) => a.x - b.x);
+ bboxGroup.push(currentGroup);
+ }
+
+ for (const group of bboxGroup) {
+ if (group[0].r == 180) {
+ // Switch upside down
+ for (const bbox of group) {
+ bbox.x = Math.round(bbox.ry ?? bbox.y);
+ bbox.y = bbox.rx ?? bbox.x;
+ bbox.r = 0;
+ bbox.rotated = true;
+ }
}
+ }
}
-export function bboxToLine(textBbox: ProjectionTextBox[], medianWidth: number, medianHeight: number): ProjectionTextBox[][] {
- // sort lines on first y axis then x axis (top - left)
- textBbox.sort((a, b) => {
- if (a.y == b.y) {
- return a.x - b.x;
- }
- return a.y - b.y;
- });
-
- function canMergeMarkup(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {
- if (!previousBbox.markup && !bbox.markup) {
- return true;
- }
- if (previousBbox.markup && bbox.markup
- && previousBbox.markup.highlight === bbox.markup.highlight
- && previousBbox.markup.underline === bbox.markup.underline
- && previousBbox.markup.squiggly === bbox.markup.squiggly
- && previousBbox.markup.strikeout === bbox.markup.strikeout
- ) {
- return true;
- }
- return false;
+export function bboxToLine(
+ textBbox: ProjectionTextBox[],
+ medianWidth: number,
+ medianHeight: number
+): ProjectionTextBox[][] {
+ // sort lines on first y axis then x axis (top - left)
+ textBbox.sort((a, b) => {
+ if (a.y == b.y) {
+ return a.x - b.x;
}
+ return a.y - b.y;
+ });
- function canMerge(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {
- if (bbox.y == previousBbox.y && bbox.h == previousBbox.h) {
- const xDelta = bbox.x - previousBbox.x - previousBbox.w;
- if (((xDelta < 0 && xDelta > -.5) || (xDelta >= 0 && xDelta < .1)) && canMergeMarkup(previousBbox, bbox)) {
- return true;
- }
- }
- return false;
+ function canMergeMarkup(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {
+ if (!previousBbox.markup && !bbox.markup) {
+ return true;
}
-
- function mergePageBbox(a: ProjectionTextBox, b: ProjectionTextBox): Coordinates {
- const aBbox = a.pageBbox || {x: a.x, y: a.y, w: a.w, h: a.h};
- const bBbox = b.pageBbox || {x: b.x, y: b.y, w: b.w, h: b.h};
- const left = Math.min(aBbox.x, bBbox.x);
- const top = Math.min(aBbox.y, bBbox.y);
- const right = Math.max(aBbox.x + aBbox.w, bBbox.x + bBbox.w);
- const bottom = Math.max(aBbox.y + aBbox.h, bBbox.y + bBbox.h);
- return {x: left, y: top, w: right - left, h: bottom - top};
+ if (
+ previousBbox.markup &&
+ bbox.markup &&
+ previousBbox.markup.highlight === bbox.markup.highlight &&
+ previousBbox.markup.underline === bbox.markup.underline &&
+ previousBbox.markup.squiggly === bbox.markup.squiggly &&
+ previousBbox.markup.strikeout === bbox.markup.strikeout
+ ) {
+ return true;
}
-
- // merge Continuous bbox
- for (let i = 1; i < textBbox.length; i++) {
- let bbox = textBbox[i];
- let previousBbox = textBbox[i - 1];
- if (canMerge(previousBbox, bbox)) {
- previousBbox.w = bbox.x + bbox.w - previousBbox.x;
- previousBbox.str += bbox.str;
- previousBbox.strLength += bbox.strLength;
- previousBbox.pageBbox = mergePageBbox(previousBbox, bbox);
- textBbox.splice(i, 1);
- i--;
- }
+ return false;
+ }
+
+ function canMerge(previousBbox: ProjectionTextBox, bbox: ProjectionTextBox): boolean {
+ if (bbox.y == previousBbox.y && bbox.h == previousBbox.h) {
+ const xDelta = bbox.x - previousBbox.x - previousBbox.w;
+ if (
+ ((xDelta < 0 && xDelta > -0.5) || (xDelta >= 0 && xDelta < 0.1)) &&
+ canMergeMarkup(previousBbox, bbox)
+ ) {
+ return true;
+ }
}
-
- // try to find the bounding box that align as line and group them by line
- let lines: ProjectionTextBox[][] = [];
- let currentLine: ProjectionTextBox[] = [];
- let previousBbox = null;
-
- for (let bbox of textBbox) {
- if (!previousBbox) {
- currentLine.push(bbox);
- }
- // This is where we define how line are build. to be improved
- else {
- let lineMinY = Math.min(...currentLine.map(v => v.y));
- let lineMaxY = Math.max(...currentLine.map(v => v.y + v.h));
-
- let lineCollide = false;
- for (let currentLineItemBbox of currentLine) {
-
- let overlapLenght = Math.min(currentLineItemBbox.x + currentLineItemBbox.w, bbox.x + bbox.w) - Math.max(currentLineItemBbox.x, bbox.x);
- if (overlapLenght > medianWidth / 3) {
- lineCollide = true;
- break;
- }
- }
- if (!lineCollide
- && (
- (bbox.y + bbox.h * .5 >= lineMinY && bbox.y + bbox.h * .5 <= lineMaxY)
- ||
- (bbox.y >= lineMinY && bbox.y <= lineMaxY)
- )
- ) {
- currentLine.push(bbox);
- }
- else {
- if (currentLine.length) {
- lines.push(currentLine);
- }
- currentLine = [bbox];
- }
- }
- previousBbox = bbox;
+ return false;
+ }
+
+ function mergePageBbox(a: ProjectionTextBox, b: ProjectionTextBox): Coordinates {
+ const aBbox = a.pageBbox || { x: a.x, y: a.y, w: a.w, h: a.h };
+ const bBbox = b.pageBbox || { x: b.x, y: b.y, w: b.w, h: b.h };
+ const left = Math.min(aBbox.x, bBbox.x);
+ const top = Math.min(aBbox.y, bBbox.y);
+ const right = Math.max(aBbox.x + aBbox.w, bBbox.x + bBbox.w);
+ const bottom = Math.max(aBbox.y + aBbox.h, bBbox.y + bBbox.h);
+ return { x: left, y: top, w: right - left, h: bottom - top };
+ }
+
+ // merge Continuous bbox
+ for (let i = 1; i < textBbox.length; i++) {
+ const bbox = textBbox[i];
+ const previousBbox = textBbox[i - 1];
+ if (canMerge(previousBbox, bbox)) {
+ previousBbox.w = bbox.x + bbox.w - previousBbox.x;
+ previousBbox.str += bbox.str;
+ previousBbox.strLength += bbox.strLength;
+ previousBbox.pageBbox = mergePageBbox(previousBbox, bbox);
+ textBbox.splice(i, 1);
+ i--;
}
+ }
- if (currentLine.length) {
- lines.push(currentLine);
- }
+ // try to find the bounding box that align as line and group them by line
+ const lines: ProjectionTextBox[][] = [];
+ let currentLine: ProjectionTextBox[] = [];
+ let previousBbox = null;
- // sort each line by x
- for (let line of lines) {
- line.sort((a, b) => a.x - b.x);
+ for (const bbox of textBbox) {
+ if (!previousBbox) {
+ currentLine.push(bbox);
}
-
- // sort lines by y
- lines.sort((a, b) => a[0].y - b[0].y);
- // merge 'words'
- let mergeThreshold = 1;
- for (let line of lines) {
- for (let i = 1; i < line.length; ++i) {
- // merge box in word if:
- // - same height
- // - less than 2 in space
- // if (line[i].h == line[i-1].h) {
- let currentLine = line[i];
- let previousLine = line[i - 1];
- if (canMergeMarkup(previousLine, currentLine)) {
- if (currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {
- // if same word but less than .7 of prev line
- if (currentLine.h != 0 && currentLine.h < previousLine.h * .7) {
- // and not starting with space
- if (currentLine.str[0] == ' ') {
- break;
- }
- if (currentLine.y > previousLine.y + previousLine.h * .2) {
- currentLine.str = strToSubscriptString(currentLine.str);
- }
- else {
- currentLine.str = strToPostScript(currentLine.str);
- }
- }
- previousLine.w = currentLine.x + currentLine.w - previousLine.x;
- previousLine.str += currentLine.str;
- previousLine.strLength += currentLine.strLength;
- previousLine.pageBbox = mergePageBbox(previousLine, currentLine);
- line.splice(i, 1);
- i--;
- }
- else if (currentLine.x - previousLine.x - previousLine.w < previousLine.w / previousLine.strLength) {
- // merge if space between this word and previous is less than average
- // character width (using previous word font size)
- previousLine.w = currentLine.x + currentLine.w - previousLine.x;
- if (!previousLine.str.endsWith(" ")) {
- previousLine.str += " ";
- previousLine.strLength += 1;
- }
- previousLine.str += currentLine.str;
- previousLine.strLength += currentLine.strLength;
- previousLine.pageBbox = mergePageBbox(previousLine, currentLine);
- line.splice(i, 1);
- i--;
-
- }
- }
- // }
+ // This is where we define how line are build. to be improved
+ else {
+ const lineMinY = Math.min(...currentLine.map((v) => v.y));
+ const lineMaxY = Math.max(...currentLine.map((v) => v.y + v.h));
+
+ let lineCollide = false;
+ for (const currentLineItemBbox of currentLine) {
+ const overlapLenght =
+ Math.min(currentLineItemBbox.x + currentLineItemBbox.w, bbox.x + bbox.w) -
+ Math.max(currentLineItemBbox.x, bbox.x);
+ if (overlapLenght > medianWidth / 3) {
+ lineCollide = true;
+ break;
}
+ }
+ if (
+ !lineCollide &&
+ ((bbox.y + bbox.h * 0.5 >= lineMinY && bbox.y + bbox.h * 0.5 <= lineMaxY) ||
+ (bbox.y >= lineMinY && bbox.y <= lineMaxY))
+ ) {
+ currentLine.push(bbox);
+ } else {
+ if (currentLine.length) {
+ lines.push(currentLine);
+ }
+ currentLine = [bbox];
+ }
}
-
-
- // check if we can merge the lines together
- for (let i = 1; i < lines.length - 1; i++) {
- let currentLine = lines[i];
- let previousLine = lines[i - 1];
-
- let previousLineMinY = Math.min(...previousLine.map(v => v.y));
- let previousLineMaxY = Math.max(...previousLine.map(v => v.y + v.h));
- let currentLineMinY = Math.min(...currentLine.map(v => v.y));
- let currentLineMaxY = Math.max(...currentLine.map(v => v.y + v.h));
-
- // does the 2 line overlap?
- if (previousLineMaxY > currentLineMinY && previousLineMinY < currentLineMaxY) {
- // check the bboxes of current line and prevline do not overlap
- let bboxOverlap = false;
- for (let bbox of currentLine) {
- for (let prevBbox of previousLine) {
- if (bbox.x >= prevBbox.x && bbox.x <= prevBbox.x + prevBbox.w) {
- bboxOverlap = true;
- break;
- }
- if (prevBbox.x >= bbox.x && prevBbox.x <= bbox.x + bbox.w) {
- bboxOverlap = true;
- break;
- }
- }
+ previousBbox = bbox;
+ }
+
+ if (currentLine.length) {
+ lines.push(currentLine);
+ }
+
+ // sort each line by x
+ for (const line of lines) {
+ line.sort((a, b) => a.x - b.x);
+ }
+
+ // sort lines by y
+ lines.sort((a, b) => a[0].y - b[0].y);
+ // merge 'words'
+ const mergeThreshold = 1;
+ for (const line of lines) {
+ for (let i = 1; i < line.length; ++i) {
+ // merge box in word if:
+ // - same height
+ // - less than 2 in space
+ // if (line[i].h == line[i-1].h) {
+ const currentLine = line[i];
+ const previousLine = line[i - 1];
+ if (canMergeMarkup(previousLine, currentLine)) {
+ if (currentLine.x - previousLine.x - previousLine.w <= mergeThreshold) {
+ // if same word but less than .7 of prev line
+ if (currentLine.h != 0 && currentLine.h < previousLine.h * 0.7) {
+ // and not starting with space
+ if (currentLine.str[0] == " ") {
+ break;
}
-
- // merge if no overlap
- if (!bboxOverlap) {
- previousLine.push(...currentLine);
- previousLine.sort((a, b) => a.x - b.x);
- lines.splice(i--, 1);
+ if (currentLine.y > previousLine.y + previousLine.h * 0.2) {
+ currentLine.str = strToSubscriptString(currentLine.str);
+ } else {
+ currentLine.str = strToPostScript(currentLine.str);
}
+ }
+ previousLine.w = currentLine.x + currentLine.w - previousLine.x;
+ previousLine.str += currentLine.str;
+ previousLine.strLength += currentLine.strLength;
+ previousLine.pageBbox = mergePageBbox(previousLine, currentLine);
+ line.splice(i, 1);
+ i--;
+ } else if (
+ currentLine.x - previousLine.x - previousLine.w <
+ previousLine.w / previousLine.strLength
+ ) {
+ // merge if space between this word and previous is less than average
+ // character width (using previous word font size)
+ previousLine.w = currentLine.x + currentLine.w - previousLine.x;
+ if (!previousLine.str.endsWith(" ")) {
+ previousLine.str += " ";
+ previousLine.strLength += 1;
+ }
+ previousLine.str += currentLine.str;
+ previousLine.strLength += currentLine.strLength;
+ previousLine.pageBbox = mergePageBbox(previousLine, currentLine);
+ line.splice(i, 1);
+ i--;
}
+ }
+ // }
}
-
- for (let i = 1; i < lines.length; i++) {
- const yDelta = lines[i][0].y - lines[i - 1][0].y - lines[i - 1][0].h;
- // Calculate the number of blank lines to insert based on vertical spacing
- // Use medianHeight as a reference for one line spacing
- if (yDelta > medianHeight) {
- // Calculate how many blank lines should be inserted
- // Round to nearest integer to get approximate number of lines
- const numBlankLines = Math.round(yDelta / medianHeight) - 1;
- // Cap at a reasonable maximum (e.g., 10 blank lines) to avoid extreme cases
- const linesToInsert = Math.min(Math.max(numBlankLines, 1), 10);
-
- // Insert the calculated number of blank lines
- const blankLines = Array(linesToInsert).fill([]);
- lines.splice(i, 0, ...blankLines);
- i += linesToInsert;
+ }
+
+ // check if we can merge the lines together
+ for (let i = 1; i < lines.length - 1; i++) {
+ const currentLine = lines[i];
+ const previousLine = lines[i - 1];
+
+ const previousLineMinY = Math.min(...previousLine.map((v) => v.y));
+ const previousLineMaxY = Math.max(...previousLine.map((v) => v.y + v.h));
+ const currentLineMinY = Math.min(...currentLine.map((v) => v.y));
+ const currentLineMaxY = Math.max(...currentLine.map((v) => v.y + v.h));
+
+ // does the 2 line overlap?
+ if (previousLineMaxY > currentLineMinY && previousLineMinY < currentLineMaxY) {
+ // check the bboxes of current line and prevline do not overlap
+ let bboxOverlap = false;
+ for (const bbox of currentLine) {
+ for (const prevBbox of previousLine) {
+ if (bbox.x >= prevBbox.x && bbox.x <= prevBbox.x + prevBbox.w) {
+ bboxOverlap = true;
+ break;
+ }
+ if (prevBbox.x >= bbox.x && prevBbox.x <= bbox.x + bbox.w) {
+ bboxOverlap = true;
+ break;
+ }
}
+ }
+
+ // merge if no overlap
+ if (!bboxOverlap) {
+ previousLine.push(...currentLine);
+ previousLine.sort((a, b) => a.x - b.x);
+ lines.splice(i--, 1);
+ }
}
- return lines;
+ }
+
+ for (let i = 1; i < lines.length; i++) {
+ const yDelta = lines[i][0].y - lines[i - 1][0].y - lines[i - 1][0].h;
+ // Calculate the number of blank lines to insert based on vertical spacing
+ // Use medianHeight as a reference for one line spacing
+ if (yDelta > medianHeight) {
+ // Calculate how many blank lines should be inserted
+ // Round to nearest integer to get approximate number of lines
+ const numBlankLines = Math.round(yDelta / medianHeight) - 1;
+ // Cap at a reasonable maximum (e.g., 10 blank lines) to avoid extreme cases
+ const linesToInsert = Math.min(Math.max(numBlankLines, 1), 10);
+
+ // Insert the calculated number of blank lines
+ const blankLines = Array(linesToInsert).fill([]);
+ lines.splice(i, 0, ...blankLines);
+ i += linesToInsert;
+ }
+ }
+ return lines;
}
function canRenderBbox(line: ProjectionTextBox[], bbox: ProjectionTextBox): boolean {
- for (let item of line) {
- if (item == bbox) {
- return true;
- }
- if (!item.rendered) {
- return false;
- }
+ for (const item of line) {
+ if (item == bbox) {
+ return true;
}
- return false;
+ if (!item.rendered) {
+ return false;
+ }
+ }
+ return false;
}
-function updateForwardAnchorRightBound(snapMap: number[], forwardAnchor: ForwardAnchor, rightBound: number, anchorTarget: number): void {
- // Anything snapped to the right of rightBound should be aligned to anchorTarget line length at minimum
- for (let i = snapMap.length - 1; i >= 0; --i) {
- const anchor = snapMap[i];
- if (rightBound <= anchor) {
- if (!forwardAnchor[anchor] || anchorTarget > forwardAnchor[anchor]) {
- forwardAnchor[anchor] = anchorTarget;
- }
- }
- else {
- return;
- }
+function updateForwardAnchorRightBound(
+ snapMap: number[],
+ forwardAnchor: ForwardAnchor,
+ rightBound: number,
+ anchorTarget: number
+): void {
+ // Anything snapped to the right of rightBound should be aligned to anchorTarget line length at minimum
+ for (let i = snapMap.length - 1; i >= 0; --i) {
+ const anchor = snapMap[i];
+ if (rightBound <= anchor) {
+ if (!forwardAnchor[anchor] || anchorTarget > forwardAnchor[anchor]) {
+ forwardAnchor[anchor] = anchorTarget;
+ }
+ } else {
+ return;
}
+ }
}
-function updateForwardAnchors(bbox: ProjectionTextBox, nextBbox: ProjectionTextBox | null, snapMaps: SnapMaps, forwardAnchors: PageForwardAnchors, lineLength: number): void {
- const rightBound = bbox.x + bbox.w;
- let targetLength = lineLength;
- if (nextBbox && (nextBbox.shouldSpace ?? 0) > 0) {
- targetLength += (nextBbox.shouldSpace ?? 0);
- }
- updateForwardAnchorRightBound(snapMaps.left, forwardAnchors.left, rightBound, targetLength);
- updateForwardAnchorRightBound(snapMaps.right, forwardAnchors.right, rightBound, targetLength);
- // we do not update center anchors since centered text may span between snapped columns
- updateForwardAnchorRightBound(snapMaps.floating, forwardAnchors.floating, rightBound, targetLength);
+function updateForwardAnchors(
+ bbox: ProjectionTextBox,
+ nextBbox: ProjectionTextBox | null,
+ snapMaps: SnapMaps,
+ forwardAnchors: PageForwardAnchors,
+ lineLength: number
+): void {
+ const rightBound = bbox.x + bbox.w;
+ let targetLength = lineLength;
+ if (nextBbox && (nextBbox.shouldSpace ?? 0) > 0) {
+ targetLength += nextBbox.shouldSpace ?? 0;
+ }
+ updateForwardAnchorRightBound(snapMaps.left, forwardAnchors.left, rightBound, targetLength);
+ updateForwardAnchorRightBound(snapMaps.right, forwardAnchors.right, rightBound, targetLength);
+ // we do not update center anchors since centered text may span between snapped columns
+ updateForwardAnchorRightBound(
+ snapMaps.floating,
+ forwardAnchors.floating,
+ rightBound,
+ targetLength
+ );
}
function resetBlockForwardAnchors(forwardAnchors: PageForwardAnchors): void {
- forwardAnchors.left = {};
- forwardAnchors.right = {};
- forwardAnchors.center = {};
- forwardAnchors.floating = {};
+ forwardAnchors.left = {};
+ forwardAnchors.right = {};
+ forwardAnchors.center = {};
+ forwardAnchors.floating = {};
}
-function getMedianTextBoxSize (lines: ProjectionTextBox[]): TextBoxSize {
- // calculate median textBox width
- let widthList = [];
- for (let bbox of lines) {
- if (bbox.w > 0) {
- widthList.push(bbox.w / bbox.strLength);
- }
+function getMedianTextBoxSize(lines: ProjectionTextBox[]): TextBoxSize {
+ // calculate median textBox width
+ const widthList = [];
+ for (const bbox of lines) {
+ if (bbox.w > 0) {
+ widthList.push(bbox.w / bbox.strLength);
}
- const medianWidth = widthList.sort((a, b) => a - b)[Math.floor(widthList.length / 2)];
-
- // calculate median textBox height
- let heightList = [];
- for (let bbox of lines) {
- if (bbox.h > 0) {
- heightList.push(bbox.h);
- }
+ }
+ const medianWidth = widthList.sort((a, b) => a - b)[Math.floor(widthList.length / 2)];
+
+ // calculate median textBox height
+ const heightList = [];
+ for (const bbox of lines) {
+ if (bbox.h > 0) {
+ heightList.push(bbox.h);
}
- const medianHeight = heightList.sort((a, b) => a - b)[Math.floor(heightList.length / 2)];
- return {width: medianWidth, height: medianHeight};
+ }
+ const medianHeight = heightList.sort((a, b) => a - b)[Math.floor(heightList.length / 2)];
+ return { width: medianWidth, height: medianHeight };
}
-export function projectToGrid(config: LiteParseConfig, page: PageData, projectionBoxes: ProjectionTextBox[], prevAnchors: PrevAnchors, totalPages: number): { text: string, prevAnchors: PrevAnchors } {
-
- // detect '.' garbage in the lines
- let dotCount = 0;
- for (let bbox of projectionBoxes) {
- // check if bbox.str contains only dots
- if (bbox.str.match(/^\.+$/)) {
- dotCount++;
+export function projectToGrid(
+ config: LiteParseConfig,
+ page: PageData,
+ projectionBoxes: ProjectionTextBox[],
+ prevAnchors: PrevAnchors,
+ totalPages: number
+): { text: string; prevAnchors: PrevAnchors } {
+ // detect '.' garbage in the lines
+ let dotCount = 0;
+ for (const bbox of projectionBoxes) {
+ // check if bbox.str contains only dots
+ if (bbox.str.match(/^\.+$/)) {
+ dotCount++;
+ }
+ }
+
+ if (dotCount > 100 && dotCount > projectionBoxes.length * 0.05) {
+ // remove all dots and splice them from lines
+ const newLines = [];
+ for (const bbox of projectionBoxes) {
+ if (bbox.str.match(/^\.+$/)) {
+ continue;
+ }
+ if (bbox.str.match(/^·+$/)) {
+ continue;
+ }
+ if (bbox.str.match(/^"+$/)) {
+ continue;
+ }
+ newLines.push(bbox);
+ }
+ projectionBoxes = newLines;
+ }
+
+ // calculate median textBox width/height
+ const pageMedianSizes = getMedianTextBoxSize(projectionBoxes);
+ let medianWidth = pageMedianSizes.width;
+ let medianHeight = pageMedianSizes.height;
+
+ let { verticalLines, horizontalLines } = getVerticalAndHorizontalLinesFromPagePaths(
+ config,
+ page.paths
+ );
+ // filter vertical and horizontal lines are less than 80% of page.width / height
+ verticalLines = verticalLines.filter((line) => line.height > page.height * 0.8);
+ horizontalLines = horizontalLines.filter((line) => line.width > page.width * 0.8);
+
+ // Save original bboxes (including OCR) for text attribution
+ const attributionBboxes: ProjectionTextBox[] = [];
+ for (const bbox of projectionBoxes as ProjectionTextBox[]) {
+ if (!bbox || !bbox.str || bbox.vgap || bbox.isPlaceholder) {
+ continue;
+ }
+ attributionBboxes.push({
+ str: bbox.str,
+ x: bbox.x,
+ y: bbox.y,
+ w: bbox.w,
+ h: bbox.h,
+ r: bbox.r,
+ strLength: bbox.str.length,
+ });
+ }
+
+ handleRotationReadingOrder(projectionBoxes, page.height);
+ const lines = bboxToLine(projectionBoxes, medianWidth, medianHeight);
+
+ // remove unprojectable text and apply markup to final lines
+ for (let i = 0; i < lines.length; ++i) {
+ const line = filterUnprojectableText(config, lines[i]);
+ for (const bbox of line) {
+ // With the way our grid projection currently works, we have to output
+ // tags before raw line projection to avoid breaking the projection alignment.
+ // The tags get replaced with MD as needed in output formatting, this does
+ // result in output text containing the ~~ strikeout markup, but this is
+ // mitigated since we skip markup entirely when we are not outputting markdown
+ if (bbox.str.trim().length != 0 && bbox.markup) {
+ bbox.str = applyMarkupTags(bbox.markup, bbox.str);
+ }
+ }
+ lines[i] = line;
+ }
+
+ const forwardAnchors: PageForwardAnchors = {
+ left: {},
+ right: {},
+ center: {},
+ floating: {},
+ };
+
+ const rawLines: string[] = [];
+ const rawLinesDelta = [];
+
+ const blocks: LineRange[] = [];
+ if (config.preserveLayoutAlignmentAcrossPages && totalPages > 1) {
+ blocks.push({ start: 0, end: lines.length });
+ } else {
+ let emptyCount = 0;
+ let start = -1;
+ for (const [lineIndex, line] of lines.entries()) {
+ if (line.length === 0) {
+ emptyCount++;
+ if (emptyCount > 1) {
+ if (start >= 0) {
+ // ignore completely empty blocks, include the double blank
+ // line at the end of valid blocks
+ blocks.push({ start: start, end: lineIndex + 1 });
+ }
+ start = -1;
+ }
+ } else {
+ emptyCount = 0;
+ if (start < 0) {
+ start = lineIndex;
}
+ }
}
+ if (start > -1) {
+ blocks.push({ start: start, end: lines.length });
+ }
+ }
+
+ for (const block of blocks) {
+ const { anchorLeft, anchorRight, anchorCenter } = extractAnchorsPointsFromLines(
+ lines.slice(block.start, block.end),
+ page,
+ verticalLines
+ );
+
+ const snapMaps: SnapMaps = {
+ left: [],
+ right: [],
+ center: [],
+ floating: [],
+ };
- if (dotCount > 100 && dotCount > projectionBoxes.length * 0.05) {
- // remove all dots and splice them from lines
- let newLines = [];
- for (let bbox of projectionBoxes) {
- if (bbox.str.match(/^\.+$/)) {
- continue;
- }
- if (bbox.str.match(/^·+$/)) {
- continue;
- }
- if (bbox.str.match(/^"+$/)) {
- continue;
- }
- newLines.push(bbox);
- }
- projectionBoxes = newLines;
+ const uniqueSnaps = new Set<number>();
+ for (const snap in anchorLeft) {
+ uniqueSnaps.add(parseFloat(snap));
}
+ snapMaps.left.push(...uniqueSnaps);
+ uniqueSnaps.clear();
- // calculate median textBox width/height
- const pageMedianSizes = getMedianTextBoxSize(projectionBoxes);
- let medianWidth = pageMedianSizes.width;
- let medianHeight = pageMedianSizes.height;
-
- let { verticalLines, horizontalLines } = getVerticalAndHorizontalLinesFromPagePaths(config, page.paths);
- // filter vertical and horizontal lines are less than 80% of page.width / height
- verticalLines = verticalLines.filter(line => line.height > page.height * 0.8);
- horizontalLines = horizontalLines.filter(line => line.width > page.width * 0.8);
-
- // Save original bboxes (including OCR) for text attribution
- const attributionBboxes: ProjectionTextBox[] = []
- for (const bbox of (projectionBoxes as ProjectionTextBox[])) {
- if (!bbox || !bbox.str || bbox.vgap || bbox.isPlaceholder) {
- continue;
- }
- attributionBboxes.push({
- str: bbox.str,
- x: bbox.x,
- y: bbox.y,
- w: bbox.w,
- h: bbox.h,
- r: bbox.r,
- strLength: bbox.str.length
- });
+ for (const snap in anchorRight) {
+ uniqueSnaps.add(parseFloat(snap));
}
+ snapMaps.right.push(...uniqueSnaps);
+ uniqueSnaps.clear();
- handleRotationReadingOrder(projectionBoxes, page.height);
- let lines = bboxToLine(projectionBoxes, medianWidth, medianHeight);
-
- // remove unprojectable text and apply markup to final lines
- for (let i = 0; i < lines.length; ++i) {
- const line = filterUnprojectableText(config, lines[i]);
- for (const bbox of line) {
- // With the way our grid projection currently works, we have to output
- // tags before raw line projection to avoid breaking the projection alignment.
- // The tags get replaced with MD as needed in output formatting, this does
- // result in output text containing the ~~ strikeout markup, but this is
- // mitigated since we skip markup entirely when we are not outputting markdown
- if (bbox.str.trim().length != 0 && bbox.markup) {
- bbox.str = applyMarkupTags(bbox.markup, bbox.str);
- }
- }
- lines[i] = line;
+ for (const snap in anchorCenter) {
+ uniqueSnaps.add(parseFloat(snap));
}
+ snapMaps.center.push(...uniqueSnaps);
+ uniqueSnaps.clear();
- const forwardAnchors: PageForwardAnchors = {
- left: {},
- right: {},
- center: {},
- floating: {},
- };
+ let hasChanged = true;
+ const leftSnap: Snap[] = [];
+ const rightSnap: Snap[] = [];
+ const centerSnap: Snap[] = [];
- let rawLines: string[] = [];
- let rawLinesDelta = [];
+ if (!config.preserveLayoutAlignmentAcrossPages) {
+ resetBlockForwardAnchors(forwardAnchors);
- const blocks: LineRange[] = [];
- if (config.preserveLayoutAlignmentAcrossPages && totalPages > 1) {
- blocks.push({start: 0, end: lines.length});
- }
- else {
- let emptyCount = 0;
- let start = -1;
- for (const [lineIndex, line] of lines.entries()) {
- if (line.length === 0) {
- emptyCount++;
- if (emptyCount > 1) {
- if (start >= 0) {
- // ignore completely empty blocks, include the double blank
- // line at the end of valid blocks
- blocks.push({start: start, end: lineIndex + 1});
- }
- start = -1;
- }
- }
- else {
- emptyCount = 0;
- if (start < 0) {
- start = lineIndex;
- }
- }
- }
- if (start > -1) {
- blocks.push({start: start, end: lines.length});
- }
+ const sizes = getMedianTextBoxSize(lines.slice(block.start, block.end).flat());
+ medianWidth = sizes.width;
+ medianHeight = sizes.height;
}
- for (const block of blocks) {
- let {
- anchorLeft,
- anchorRight,
- anchorCenter
- } = extractAnchorsPointsFromLines(lines.slice(block.start, block.end), page, verticalLines);
-
- const snapMaps: SnapMaps = {
- left: [],
- right: [],
- center: [],
- floating: [],
- }
-
- const uniqueSnaps = new Set<number>();
- for (let snap in anchorLeft) {
- uniqueSnaps.add(parseFloat(snap));
+ // compute snaps
+ for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {
+ const line = lines[lineIndex];
+ const forceUnsnapped = !canSnapLine(config, line);
+ let prevBbox = null;
+
+ for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {
+ const bbox = line[boxIndex];
+ bbox.forceUnsnapped = forceUnsnapped;
+
+ const spaceThreshold = 2;
+ // should we add a space between the two bbox?
+ // TODO RTL
+ if (prevBbox && bbox.x - (prevBbox.x + prevBbox.w) > spaceThreshold) {
+ // add a space
+ bbox.shouldSpace = 1;
+ const xDelta = bbox.x - (prevBbox.x + prevBbox.w);
+ const prevCharWidth = prevBbox.w / prevBbox.strLength;
+ if (xDelta > prevCharWidth * 2) {
+ // insert column spacing if any of:
+ // - gap is more than an approximate tab (8x average char width)
+ // - previous bbox is right snap
+ // - this bbox is left snap
+ // - both previous and this bbox are snaps
+ // otherwise insert floating spacing
+ if (
+ (!bbox.forceUnsnapped && xDelta > prevCharWidth * 8) ||
+ (bbox.snap && bbox.snap === "left") ||
+ (prevBbox.snap && prevBbox.snap === "right") ||
+ (bbox.snap && prevBbox.snap)
+ ) {
+ bbox.shouldSpace = COLUMN_SPACES;
+ } else {
+ bbox.shouldSpace = FLOATING_SPACES;
+ }
+ }
+ } else {
+ bbox.shouldSpace = 0;
}
- snapMaps.left.push(...uniqueSnaps);
- uniqueSnaps.clear()
- for (let snap in anchorRight) {
- uniqueSnaps.add(parseFloat(snap));
+ prevBbox = bbox;
+ if (!bbox.snap) {
+ uniqueSnaps.add(Math.round(bbox.x));
+ } else if (bbox.snap == "left") {
+ leftSnap.push({ bbox, lineIndex, boxIndex });
+ } else if (bbox.snap == "right") {
+ rightSnap.push({ bbox, lineIndex, boxIndex });
+ } else if (bbox.snap == "center") {
+ centerSnap.push({ bbox, lineIndex, boxIndex });
}
- snapMaps.right.push(...uniqueSnaps);
- uniqueSnaps.clear()
+ }
+ }
- for (let snap in anchorCenter) {
- uniqueSnaps.add(parseFloat(snap));
- }
- snapMaps.center.push(...uniqueSnaps);
- uniqueSnaps.clear()
+ snapMaps.floating.push(...uniqueSnaps);
+ uniqueSnaps.clear();
- let hasChanged = true;
- let leftSnap: Snap[] = [];
- let rightSnap: Snap[] = [];
- let centerSnap: Snap[] = [];
+ snapMaps.floating.sort((a, b) => a - b);
+ snapMaps.center.sort((a, b) => a - b);
+ snapMaps.right.sort((a, b) => a - b);
+ snapMaps.left.sort((a, b) => a - b);
- if (!config.preserveLayoutAlignmentAcrossPages) {
- resetBlockForwardAnchors(forwardAnchors);
+ while (hasChanged || snapMaps.right.length || snapMaps.left.length || snapMaps.center.length) {
+ hasChanged = false;
- const sizes = getMedianTextBoxSize(lines.slice(block.start, block.end).flat());
- medianWidth = sizes.width;
- medianHeight = sizes.height;
+ for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {
+ const line = lines[lineIndex];
+ if (!rawLines[lineIndex]) {
+ rawLines[lineIndex] = "";
+ rawLinesDelta[lineIndex] = 0;
}
- // compute snaps
- for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {
- const line = lines[lineIndex];
- const forceUnsnapped = !canSnapLine(config, line);
- let prevBbox = null;
-
- for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {
- const bbox = line[boxIndex];
- bbox.forceUnsnapped = forceUnsnapped;
-
- const spaceThreshold = 2;
- // should we add a space between the two bbox?
- // TODO RTL
- if (prevBbox && bbox.x - (prevBbox.x + prevBbox.w) > spaceThreshold) {
- // add a space
- bbox.shouldSpace = 1;
- const xDelta = bbox.x - (prevBbox.x + prevBbox.w);
- const prevCharWidth = prevBbox.w / prevBbox.strLength;
- if (xDelta > prevCharWidth * 2) {
- // insert column spacing if any of:
- // - gap is more than an approximate tab (8x average char width)
- // - previous bbox is right snap
- // - this bbox is left snap
- // - both previous and this bbox are snaps
- // otherwise insert floating spacing
- if (!bbox.forceUnsnapped && (xDelta > prevCharWidth * 8) || (bbox.snap && bbox.snap === "left") || (prevBbox.snap && prevBbox.snap === "right") || (bbox.snap && prevBbox.snap)) {
- bbox.shouldSpace = COLUMN_SPACES;
- }
- else {
- bbox.shouldSpace = FLOATING_SPACES;
- }
- }
- }
- else {
- bbox.shouldSpace = 0;
- }
+ for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {
+ const bbox = line[boxIndex];
+ if (bbox.rendered) {
+ continue;
+ }
- prevBbox = bbox;
- if (!bbox.snap) {
- uniqueSnaps.add(Math.round(bbox.x));
- }
- else if (bbox.snap == 'left') {
- leftSnap.push({ bbox, lineIndex, boxIndex });
- }
- else if (bbox.snap == 'right') {
- rightSnap.push({ bbox, lineIndex, boxIndex });
- }
- else if (bbox.snap == 'center') {
- centerSnap.push({ bbox, lineIndex, boxIndex });
- }
+ if (!bbox.forceUnsnapped) {
+ if (bbox.snap) {
+ continue;
}
- }
- snapMaps.floating.push(...uniqueSnaps);
- uniqueSnaps.clear()
-
- snapMaps.floating.sort((a, b) => a - b);
- snapMaps.center.sort((a, b) => a - b);
- snapMaps.right.sort((a, b) => a - b);
- snapMaps.left.sort((a, b) => a - b);
-
- while (hasChanged
- || snapMaps.right.length
- || snapMaps.left.length
- || snapMaps.center.length) {
- hasChanged = false;
-
- for (let lineIndex = block.start; lineIndex < block.end; ++lineIndex) {
- const line = lines[lineIndex];
- if (!rawLines[lineIndex]) {
- rawLines[lineIndex] = "";
- rawLinesDelta[lineIndex] = 0;
- }
-
- for (let boxIndex = 0; boxIndex < line.length; ++boxIndex) {
- const bbox = line[boxIndex];
- if (bbox.rendered) {
- continue;
- }
-
- if (!bbox.forceUnsnapped) {
- if (bbox.snap) {
- continue;
- }
-
- if ((snapMaps.left.length && snapMaps.left[0] < bbox.x)
- || (snapMaps.right.length && snapMaps.right[0] < bbox.x)
- || (snapMaps.center.length && snapMaps.center[0] < Math.round(bbox.x + bbox.w / 2))) {
- continue;
- }
- }
-
- if (!canRenderBbox(line, bbox)) {
- break;
- }
-
- let targetX = Math.min(Math.round(bbox.x / medianWidth), COLUMN_SPACES);
-
- let lastSnapLeft = 0;
- for (let key in forwardAnchors.left) {
- if (parseInt(key) <= bbox.x) {
- lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);
- }
- }
- const lineMax = Math.max(lastSnapLeft, rawLines[lineIndex].trimEnd().length + (bbox.shouldSpace ?? 0));
- if (targetX < lineMax) {
- targetX = lineMax;
- }
-
- if (!bbox.forceUnsnapped) {
- const floatingAnchor = forwardAnchors.floating[Math.round(bbox.x)];
- if (floatingAnchor && targetX < floatingAnchor) {
- targetX = floatingAnchor;
- }
- }
-
- rawLines[lineIndex] = rawLines[lineIndex].trimEnd();
- if (targetX > rawLines[lineIndex].length) {
- rawLines[lineIndex] += ' '.repeat(targetX - rawLines[lineIndex].length);
- }
-
- rawLines[lineIndex] += bbox.str;
-
- bbox.rendered = true;
- hasChanged = true;
-
- let nextBbox: ProjectionTextBox | null = null;
- if (line.length > (boxIndex + 1)) {
- nextBbox = line[boxIndex + 1];
- }
- if (!bbox.forceUnsnapped) {
- updateForwardAnchors(bbox, nextBbox, snapMaps, forwardAnchors, rawLines[lineIndex].length);
- }
- }
+ if (
+ (snapMaps.left.length && snapMaps.left[0] < bbox.x) ||
+ (snapMaps.right.length && snapMaps.right[0] < bbox.x) ||
+ (snapMaps.center.length && snapMaps.center[0] < Math.round(bbox.x + bbox.w / 2))
+ ) {
+ continue;
}
+ }
- if (snapMaps.left.length
- && (!snapMaps.right.length || snapMaps.left[0] <= snapMaps.right[0])
- && (!snapMaps.center.length || snapMaps.left[0] <= snapMaps.center[0])) {
- let thisTurnSnap: Snap[] = [];
- for (let item of leftSnap) {
- if (item.bbox.leftAnchor && parseFloat(item.bbox.leftAnchor) == snapMaps.left[0]) {
- thisTurnSnap.push(item);
- }
- }
- hasChanged = true;
- if (!thisTurnSnap.length) {
- snapMaps.left.shift();
- continue;
- }
-
- let targetX = Math.min(Math.round(snapMaps.left[0] / medianWidth), COLUMN_SPACES);
- let lineMax = Math.max(...thisTurnSnap.map((v) => {
- let spaceEnd = 0;
- if (!rawLines[v.lineIndex].endsWith(' ')) {
- spaceEnd = v.bbox.shouldSpace ?? 0;
- }
- if ((v.bbox.shouldSpace ?? 0) > 1) {
- const trailingSpaces = rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;
- if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {
- spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;
- }
- }
-
- return rawLines[v.lineIndex].length + spaceEnd + 1;
- }));
-
- if (targetX < lineMax) {
- targetX = lineMax;
- }
+ if (!canRenderBbox(line, bbox)) {
+ break;
+ }
- if (forwardAnchors.left[snapMaps.left[0]] && targetX < forwardAnchors.left[snapMaps.left[0]]) {
- targetX = forwardAnchors.left[snapMaps.left[0]];
- }
- if (prevAnchors.forwardAnchorLeft[snapMaps.left[0]] && targetX < prevAnchors.forwardAnchorLeft[snapMaps.left[0]]) {
- targetX = prevAnchors.forwardAnchorLeft[snapMaps.left[0]];
- }
+ let targetX = Math.min(Math.round(bbox.x / medianWidth), COLUMN_SPACES);
- forwardAnchors.left[snapMaps.left[0]] = targetX;
-
- for (let currentLeftSnapBox of thisTurnSnap) {
- const lineIndex = currentLeftSnapBox.lineIndex;
- if (targetX > rawLines[lineIndex].length) {
- rawLines[lineIndex] += ' '.repeat(targetX - rawLines[lineIndex].length);
- }
- rawLines[lineIndex] += currentLeftSnapBox.bbox.str;
- currentLeftSnapBox.bbox.rendered = true;
-
- let nextBbox: ProjectionTextBox | null = null;
- if (lines[lineIndex].length > (currentLeftSnapBox.boxIndex + 1)) {
- nextBbox = lines[lineIndex][currentLeftSnapBox.boxIndex + 1];
- }
- updateForwardAnchors(currentLeftSnapBox.bbox, nextBbox, snapMaps, forwardAnchors, rawLines[lineIndex].length);
- }
-
- for (let index = block.start; index < block.end; ++index) {
- const line = rawLines[index];
- if (line.length < targetX) {
- rawLines[index] += ' '.repeat(targetX - line.length);
- }
- }
- snapMaps.left.shift();
+ let lastSnapLeft = 0;
+ for (const key in forwardAnchors.left) {
+ if (parseInt(key) <= bbox.x) {
+ lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);
}
-
- else if (snapMaps.right.length
- && (!snapMaps.left.length || snapMaps.right[0] <= snapMaps.left[0])
- && (!snapMaps.center.length || snapMaps.right[0] <= snapMaps.center[0])) {
- let thisTurnSnap: Snap[] = [];
- hasChanged = true;
- for (let item of rightSnap) {
- if (item.bbox.rightAnchor && parseFloat(item.bbox.rightAnchor) == snapMaps.right[0]) {
- thisTurnSnap.push(item);
- }
- }
-
- if (!thisTurnSnap.length) {
- snapMaps.right.shift();
- continue;
- }
-
- let targetX = Math.min(Math.round(snapMaps.right[0] / medianWidth), COLUMN_SPACES);
- let lineMax = Math.max(...thisTurnSnap.map((v) => {
- let lastSnapLeft = 0;
- for (let key in forwardAnchors.left) {
- if (parseInt(key) <= v.bbox.x) {
- lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);
- }
- }
- return Math.max(lastSnapLeft, rawLines[v.lineIndex].trimEnd().length + (v.bbox.shouldSpace ?? 0)) + v.bbox.strLength
- }));
-
- if (targetX < lineMax) {
- targetX = lineMax;
- }
- if (forwardAnchors.right[snapMaps.right[0]] && targetX < forwardAnchors.right[snapMaps.right[0]]) {
- targetX = forwardAnchors.right[snapMaps.right[0]];
- }
- if (prevAnchors.forwardAnchorRight[snapMaps.right[0]] && targetX < prevAnchors.forwardAnchorRight[snapMaps.right[0]]) {
- targetX = prevAnchors.forwardAnchorRight[snapMaps.right[0]];
- }
- forwardAnchors.right[snapMaps.right[0]] = targetX;
-
- for (let currentRightSnapBox of thisTurnSnap) {
- const lineIndex = currentRightSnapBox.lineIndex;
- rawLines[lineIndex] = rawLines[lineIndex].trimEnd();
- if (targetX > rawLines[lineIndex].trimEnd().length + currentRightSnapBox.bbox.strLength) {
- rawLines[lineIndex] += ' '.repeat(targetX - rawLines[lineIndex].length - currentRightSnapBox.bbox.strLength);
- }
- rawLines[lineIndex] += currentRightSnapBox.bbox.str;
- currentRightSnapBox.bbox.rendered = true;
-
- let nextBbox: ProjectionTextBox | null = null;
- if (lines[lineIndex].length > (currentRightSnapBox.boxIndex + 1)) {
- nextBbox = lines[lineIndex][currentRightSnapBox.boxIndex + 1];
- }
- updateForwardAnchors(currentRightSnapBox.bbox, nextBbox, snapMaps, forwardAnchors, rawLines[lineIndex].length);
- }
- for (let index = block.start; index < block.end; ++index) {
- const line = rawLines[index];
- if (line.length < targetX) {
- rawLines[index] += ' '.repeat(targetX - line.length);
- }
- }
- snapMaps.right.shift();
+ }
+ const lineMax = Math.max(
+ lastSnapLeft,
+ rawLines[lineIndex].trimEnd().length + (bbox.shouldSpace ?? 0)
+ );
+ if (targetX < lineMax) {
+ targetX = lineMax;
+ }
+
+ if (!bbox.forceUnsnapped) {
+ const floatingAnchor = forwardAnchors.floating[Math.round(bbox.x)];
+ if (floatingAnchor && targetX < floatingAnchor) {
+ targetX = floatingAnchor;
}
+ }
+
+ rawLines[lineIndex] = rawLines[lineIndex].trimEnd();
+ if (targetX > rawLines[lineIndex].length) {
+ rawLines[lineIndex] += " ".repeat(targetX - rawLines[lineIndex].length);
+ }
+
+ rawLines[lineIndex] += bbox.str;
+
+ bbox.rendered = true;
+ hasChanged = true;
+
+ let nextBbox: ProjectionTextBox | null = null;
+ if (line.length > boxIndex + 1) {
+ nextBbox = line[boxIndex + 1];
+ }
+ if (!bbox.forceUnsnapped) {
+ updateForwardAnchors(
+ bbox,
+ nextBbox,
+ snapMaps,
+ forwardAnchors,
+ rawLines[lineIndex].length
+ );
+ }
+ }
+ }
+
+ if (
+ snapMaps.left.length &&
+ (!snapMaps.right.length || snapMaps.left[0] <= snapMaps.right[0]) &&
+ (!snapMaps.center.length || snapMaps.left[0] <= snapMaps.center[0])
+ ) {
+ const thisTurnSnap: Snap[] = [];
+ for (const item of leftSnap) {
+ if (item.bbox.leftAnchor && parseFloat(item.bbox.leftAnchor) == snapMaps.left[0]) {
+ thisTurnSnap.push(item);
+ }
+ }
+ hasChanged = true;
+ if (!thisTurnSnap.length) {
+ snapMaps.left.shift();
+ continue;
+ }
- else if (snapMaps.center.length
- && (!snapMaps.left.length || snapMaps.center[0] <= snapMaps.left[0])
- && (!snapMaps.right.length || snapMaps.center[0] <= snapMaps.right[0])) {
- let thisTurnSnap: Snap[] = [];
- hasChanged = true;
- for (let item of centerSnap) {
- if (item.bbox.centerAnchor && parseFloat(item.bbox.centerAnchor) == snapMaps.center[0]) {
- thisTurnSnap.push(item);
- }
- }
- if (!thisTurnSnap.length) {
- snapMaps.center.shift();
- continue;
- }
- let targetX = Math.min(Math.round(snapMaps.center[0] / medianWidth), COLUMN_SPACES);
- let lineMax = Math.max(
- ...(thisTurnSnap.map((v) => {
- let spaceEnd = 0;
- if (!rawLines[v.lineIndex].endsWith(' ')) {
- spaceEnd = v.bbox.shouldSpace ?? 0;
- }
- if ((v.bbox.shouldSpace ?? 0) > 1) {
- const trailingSpaces = rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;
- if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {
- spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;
- }
- }
- return rawLines[v.lineIndex].length
- + Math.round(v.bbox.strLength / 2)
- + spaceEnd
- }
- )));
-
- if (targetX < lineMax) {
- targetX = lineMax;
- }
- if (forwardAnchors.center[snapMaps.center[0]] && targetX < forwardAnchors.center[snapMaps.center[0]]) {
- targetX = forwardAnchors.center[snapMaps.center[0]];
- }
- if (prevAnchors.forwardAnchorCenter[snapMaps.center[0]] && targetX < prevAnchors.forwardAnchorCenter[snapMaps.center[0]]) {
- targetX = prevAnchors.forwardAnchorCenter[snapMaps.center[0]];
- }
- forwardAnchors.center[snapMaps.center[0]] = targetX;
- for (let currentCenterSnapBox of thisTurnSnap) {
- if (targetX > rawLines[currentCenterSnapBox.lineIndex].length + Math.round(currentCenterSnapBox.bbox.strLength / 2)) {
- rawLines[currentCenterSnapBox.lineIndex] += ' '.repeat(targetX - rawLines[currentCenterSnapBox.lineIndex].length - Math.round(currentCenterSnapBox.bbox.strLength / 2));
- }
- rawLines[currentCenterSnapBox.lineIndex] += currentCenterSnapBox.bbox.str;
- currentCenterSnapBox.bbox.rendered = true;
- }
- snapMaps.center.shift();
+ let targetX = Math.min(Math.round(snapMaps.left[0] / medianWidth), COLUMN_SPACES);
+ const lineMax = Math.max(
+ ...thisTurnSnap.map((v) => {
+ let spaceEnd = 0;
+ if (!rawLines[v.lineIndex].endsWith(" ")) {
+ spaceEnd = v.bbox.shouldSpace ?? 0;
+ }
+ if ((v.bbox.shouldSpace ?? 0) > 1) {
+ const trailingSpaces =
+ rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;
+ if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {
+ spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;
+ }
}
- }
- }
- fixSparseBlocks(blocks, rawLines);
+ return rawLines[v.lineIndex].length + spaceEnd + 1;
+ })
+ );
- let text = rawLines.join("\n");
- // OSS: Return text instead of mutating page object
- return {
- text,
- prevAnchors: {
- forwardAnchorLeft: forwardAnchors.left,
- forwardAnchorRight: forwardAnchors.right,
- forwardAnchorCenter: forwardAnchors.center,
+ if (targetX < lineMax) {
+ targetX = lineMax;
}
- };
-}
-export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig): ParsedPage[] {
- let prevAnchors: PrevAnchors = {
- forwardAnchorLeft: {},
- forwardAnchorRight: {},
- forwardAnchorCenter: {}
- };
+ if (
+ forwardAnchors.left[snapMaps.left[0]] &&
+ targetX < forwardAnchors.left[snapMaps.left[0]]
+ ) {
+ targetX = forwardAnchors.left[snapMaps.left[0]];
+ }
+ if (
+ prevAnchors.forwardAnchorLeft[snapMaps.left[0]] &&
+ targetX < prevAnchors.forwardAnchorLeft[snapMaps.left[0]]
+ ) {
+ targetX = prevAnchors.forwardAnchorLeft[snapMaps.left[0]];
+ }
- const results: ParsedPage[] = [];
+ forwardAnchors.left[snapMaps.left[0]] = targetX;
+
+ for (const currentLeftSnapBox of thisTurnSnap) {
+ const lineIndex = currentLeftSnapBox.lineIndex;
+ if (targetX > rawLines[lineIndex].length) {
+ rawLines[lineIndex] += " ".repeat(targetX - rawLines[lineIndex].length);
+ }
+ rawLines[lineIndex] += currentLeftSnapBox.bbox.str;
+ currentLeftSnapBox.bbox.rendered = true;
+
+ let nextBbox: ProjectionTextBox | null = null;
+ if (lines[lineIndex].length > currentLeftSnapBox.boxIndex + 1) {
+ nextBbox = lines[lineIndex][currentLeftSnapBox.boxIndex + 1];
+ }
+ updateForwardAnchors(
+ currentLeftSnapBox.bbox,
+ nextBbox,
+ snapMaps,
+ forwardAnchors,
+ rawLines[lineIndex].length
+ );
+ }
- for (const page of pages) {
- // Build projection boxes from text items
- const projectionBoxes = buildBbox(page, config);
+ for (let index = block.start; index < block.end; ++index) {
+ const line = rawLines[index];
+ if (line.length < targetX) {
+ rawLines[index] += " ".repeat(targetX - line.length);
+ }
+ }
+ snapMaps.left.shift();
+ } else if (
+ snapMaps.right.length &&
+ (!snapMaps.left.length || snapMaps.right[0] <= snapMaps.left[0]) &&
+ (!snapMaps.center.length || snapMaps.right[0] <= snapMaps.center[0])
+ ) {
+ const thisTurnSnap: Snap[] = [];
+ hasChanged = true;
+ for (const item of rightSnap) {
+ if (item.bbox.rightAnchor && parseFloat(item.bbox.rightAnchor) == snapMaps.right[0]) {
+ thisTurnSnap.push(item);
+ }
+ }
- // Project to grid
- const { text, prevAnchors: newAnchors } = projectToGrid(config, page, projectionBoxes, prevAnchors, pages.length);
+ if (!thisTurnSnap.length) {
+ snapMaps.right.shift();
+ continue;
+ }
- // Update forward anchors if preserving across pages
- if (config.preserveLayoutAlignmentAcrossPages) {
- for (let anchor in newAnchors.forwardAnchorLeft) {
- prevAnchors.forwardAnchorLeft[anchor] = newAnchors.forwardAnchorLeft[anchor];
+ let targetX = Math.min(Math.round(snapMaps.right[0] / medianWidth), COLUMN_SPACES);
+ const lineMax = Math.max(
+ ...thisTurnSnap.map((v) => {
+ let lastSnapLeft = 0;
+ for (const key in forwardAnchors.left) {
+ if (parseInt(key) <= v.bbox.x) {
+ lastSnapLeft = Math.max(lastSnapLeft, forwardAnchors.left[key]);
+ }
}
- for (let anchor in newAnchors.forwardAnchorRight) {
- prevAnchors.forwardAnchorRight[anchor] = newAnchors.forwardAnchorRight[anchor];
+ return (
+ Math.max(
+ lastSnapLeft,
+ rawLines[v.lineIndex].trimEnd().length + (v.bbox.shouldSpace ?? 0)
+ ) + v.bbox.strLength
+ );
+ })
+ );
+
+ if (targetX < lineMax) {
+ targetX = lineMax;
+ }
+ if (
+ forwardAnchors.right[snapMaps.right[0]] &&
+ targetX < forwardAnchors.right[snapMaps.right[0]]
+ ) {
+ targetX = forwardAnchors.right[snapMaps.right[0]];
+ }
+ if (
+ prevAnchors.forwardAnchorRight[snapMaps.right[0]] &&
+ targetX < prevAnchors.forwardAnchorRight[snapMaps.right[0]]
+ ) {
+ targetX = prevAnchors.forwardAnchorRight[snapMaps.right[0]];
+ }
+ forwardAnchors.right[snapMaps.right[0]] = targetX;
+
+ for (const currentRightSnapBox of thisTurnSnap) {
+ const lineIndex = currentRightSnapBox.lineIndex;
+ rawLines[lineIndex] = rawLines[lineIndex].trimEnd();
+ if (targetX > rawLines[lineIndex].trimEnd().length + currentRightSnapBox.bbox.strLength) {
+ rawLines[lineIndex] += " ".repeat(
+ targetX - rawLines[lineIndex].length - currentRightSnapBox.bbox.strLength
+ );
+ }
+ rawLines[lineIndex] += currentRightSnapBox.bbox.str;
+ currentRightSnapBox.bbox.rendered = true;
+
+ let nextBbox: ProjectionTextBox | null = null;
+ if (lines[lineIndex].length > currentRightSnapBox.boxIndex + 1) {
+ nextBbox = lines[lineIndex][currentRightSnapBox.boxIndex + 1];
+ }
+ updateForwardAnchors(
+ currentRightSnapBox.bbox,
+ nextBbox,
+ snapMaps,
+ forwardAnchors,
+ rawLines[lineIndex].length
+ );
+ }
+ for (let index = block.start; index < block.end; ++index) {
+ const line = rawLines[index];
+ if (line.length < targetX) {
+ rawLines[index] += " ".repeat(targetX - line.length);
+ }
+ }
+ snapMaps.right.shift();
+ } else if (
+ snapMaps.center.length &&
+ (!snapMaps.left.length || snapMaps.center[0] <= snapMaps.left[0]) &&
+ (!snapMaps.right.length || snapMaps.center[0] <= snapMaps.right[0])
+ ) {
+ const thisTurnSnap: Snap[] = [];
+ hasChanged = true;
+ for (const item of centerSnap) {
+ if (item.bbox.centerAnchor && parseFloat(item.bbox.centerAnchor) == snapMaps.center[0]) {
+ thisTurnSnap.push(item);
+ }
+ }
+ if (!thisTurnSnap.length) {
+ snapMaps.center.shift();
+ continue;
+ }
+ let targetX = Math.min(Math.round(snapMaps.center[0] / medianWidth), COLUMN_SPACES);
+ const lineMax = Math.max(
+ ...thisTurnSnap.map((v) => {
+ let spaceEnd = 0;
+ if (!rawLines[v.lineIndex].endsWith(" ")) {
+ spaceEnd = v.bbox.shouldSpace ?? 0;
}
- for (let anchor in newAnchors.forwardAnchorCenter) {
- prevAnchors.forwardAnchorCenter[anchor] = newAnchors.forwardAnchorCenter[anchor];
+ if ((v.bbox.shouldSpace ?? 0) > 1) {
+ const trailingSpaces =
+ rawLines[v.lineIndex].length - rawLines[v.lineIndex].trimEnd().length;
+ if (trailingSpaces < (v.bbox.shouldSpace ?? 0)) {
+ spaceEnd = (v.bbox.shouldSpace ?? 0) - trailingSpaces;
+ }
}
+ return rawLines[v.lineIndex].length + Math.round(v.bbox.strLength / 2) + spaceEnd;
+ })
+ );
+
+ if (targetX < lineMax) {
+ targetX = lineMax;
}
+ if (
+ forwardAnchors.center[snapMaps.center[0]] &&
+ targetX < forwardAnchors.center[snapMaps.center[0]]
+ ) {
+ targetX = forwardAnchors.center[snapMaps.center[0]];
+ }
+ if (
+ prevAnchors.forwardAnchorCenter[snapMaps.center[0]] &&
+ targetX < prevAnchors.forwardAnchorCenter[snapMaps.center[0]]
+ ) {
+ targetX = prevAnchors.forwardAnchorCenter[snapMaps.center[0]];
+ }
+ forwardAnchors.center[snapMaps.center[0]] = targetX;
+ for (const currentCenterSnapBox of thisTurnSnap) {
+ if (
+ targetX >
+ rawLines[currentCenterSnapBox.lineIndex].length +
+ Math.round(currentCenterSnapBox.bbox.strLength / 2)
+ ) {
+ rawLines[currentCenterSnapBox.lineIndex] += " ".repeat(
+ targetX -
+ rawLines[currentCenterSnapBox.lineIndex].length -
+ Math.round(currentCenterSnapBox.bbox.strLength / 2)
+ );
+ }
+ rawLines[currentCenterSnapBox.lineIndex] += currentCenterSnapBox.bbox.str;
+ currentCenterSnapBox.bbox.rendered = true;
+ }
+ snapMaps.center.shift();
+ }
+ }
+ }
+
+ fixSparseBlocks(blocks, rawLines);
+
+ const text = rawLines.join("\n");
+ // OSS: Return text instead of mutating page object
+ return {
+ text,
+ prevAnchors: {
+ forwardAnchorLeft: forwardAnchors.left,
+ forwardAnchorRight: forwardAnchors.right,
+ forwardAnchorCenter: forwardAnchors.center,
+ },
+ };
+}
- // Build result page
- results.push({
- pageNum: page.pageNum,
- width: page.width,
- height: page.height,
- text,
- textItems: page.textItems,
- boundingBoxes: [],
- tables: []
- });
+export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig): ParsedPage[] {
+ const prevAnchors: PrevAnchors = {
+ forwardAnchorLeft: {},
+ forwardAnchorRight: {},
+ forwardAnchorCenter: {},
+ };
+
+ const results: ParsedPage[] = [];
+
+ for (const page of pages) {
+ // Build projection boxes from text items
+ const projectionBoxes = buildBbox(page, config);
+
+ // Project to grid
+ const { text, prevAnchors: newAnchors } = projectToGrid(
+ config,
+ page,
+ projectionBoxes,
+ prevAnchors,
+ pages.length
+ );
+
+ // Update forward anchors if preserving across pages
+ if (config.preserveLayoutAlignmentAcrossPages) {
+ for (const anchor in newAnchors.forwardAnchorLeft) {
+ prevAnchors.forwardAnchorLeft[anchor] = newAnchors.forwardAnchorLeft[anchor];
+ }
+ for (const anchor in newAnchors.forwardAnchorRight) {
+ prevAnchors.forwardAnchorRight[anchor] = newAnchors.forwardAnchorRight[anchor];
+ }
+ for (const anchor in newAnchors.forwardAnchorCenter) {
+ prevAnchors.forwardAnchorCenter[anchor] = newAnchors.forwardAnchorCenter[anchor];
+ }
}
- // Clean raw text (margin detection, etc)
- cleanRawText(results, config);
+ // Build result page
+ results.push({
+ pageNum: page.pageNum,
+ width: page.width,
+ height: page.height,
+ text,
+ textItems: page.textItems,
+ boundingBoxes: [],
+ tables: [],
+ });
+ }
+
+ // Clean raw text (margin detection, etc)
+ cleanRawText(results, config);
- return results;
+ return results;
}
diff --git a/src/processing/markupUtils.ts b/src/processing/markupUtils.ts
index 63bf25a..94d91ab 100644
--- a/src/processing/markupUtils.ts
+++ b/src/processing/markupUtils.ts
@@ -1,4 +1,4 @@
-import { MarkupData } from '../core/types.js';
+import { MarkupData } from "../core/types.js";
/**
* Apply markup tags to text based on markup data
diff --git a/src/processing/ocrUtils.ts b/src/processing/ocrUtils.ts
index 64a6c4d..9ae11f1 100644
--- a/src/processing/ocrUtils.ts
+++ b/src/processing/ocrUtils.ts
@@ -1,4 +1,4 @@
-import { Image, EasyOcrResultLine } from '../engines/pdf/interface.js';
+import { Image, EasyOcrResultLine } from "../engines/pdf/interface.js";
export interface OcrBlock {
c: string; // Content/text
@@ -60,14 +60,12 @@ export function parseImageOcrBlocks(image: Image): OcrBlock[] {
/**
* Parse EasyOCR stdout result into structured format
*/
-export function easyOcrResultLinesToList(
- stdOutResult?: string
-): EasyOcrResultLine[] {
+export function easyOcrResultLinesToList(stdOutResult?: string): EasyOcrResultLine[] {
if (!stdOutResult?.length) {
return [];
}
- const lines = stdOutResult.split('\n');
+ const lines = stdOutResult.split("\n");
const blocks: EasyOcrResultLine[] = [];
for (const line of lines) {
diff --git a/src/processing/pathUtils.ts b/src/processing/pathUtils.ts
index 5ffb374..4686357 100644
--- a/src/processing/pathUtils.ts
+++ b/src/processing/pathUtils.ts
@@ -1,5 +1,5 @@
-import { Path } from '../engines/pdf/interface.js';
-import { LiteParseConfig } from '../core/types.js';
+import { Path } from "../engines/pdf/interface.js";
+import { LiteParseConfig } from "../core/types.js";
export interface Line {
x: number;
@@ -27,7 +27,7 @@ export function getVerticalAndHorizontalLinesFromPagePaths(
// and extract line segments from rectangles and line paths
for (const path of paths) {
- if (path.type === 'line' && path.points.length >= 2) {
+ if (path.type === "line" && path.points.length >= 2) {
const [start, end] = path.points;
const [x1, y1] = start;
const [x2, y2] = end;
@@ -54,7 +54,7 @@ export function getVerticalAndHorizontalLinesFromPagePaths(
}
}
// Handle rectangles as potential table borders
- else if (path.type === 'rectangle' && path.points.length >= 4) {
+ else if (path.type === "rectangle" && path.points.length >= 4) {
const xs = path.points.map((p) => p[0]);
const ys = path.points.map((p) => p[1]);
const minX = Math.min(...xs);
diff --git a/src/processing/tables.ts b/src/processing/tables.ts
index 043d359..ebd5bd0 100644
--- a/src/processing/tables.ts
+++ b/src/processing/tables.ts
@@ -1,5 +1,5 @@
-import { Path } from '../engines/pdf/interface.js';
-import { TextItem } from '../core/types.js';
+import { Path } from "../engines/pdf/interface.js";
+import { TextItem } from "../core/types.js";
export interface DetectedTable {
x1: number;
@@ -43,7 +43,7 @@ function extractLines(paths: Path[]): { horizontal: Line[]; vertical: Line[] } {
const tolerance = 2; // Tolerance for determining if a line is horizontal/vertical
for (const path of paths) {
- if (path.type === 'line' && path.points.length >= 2) {
+ if (path.type === "line" && path.points.length >= 2) {
const [start, end] = path.points;
const [x1, y1] = start;
const [x2, y2] = end;
@@ -73,7 +73,7 @@ function extractLines(paths: Path[]): { horizontal: Line[]; vertical: Line[] } {
}
}
// Extract lines from rectangles
- else if (path.type === 'rectangle' && path.points.length >= 4) {
+ else if (path.type === "rectangle" && path.points.length >= 4) {
const xs = path.points.map((p) => p[0]);
const ys = path.points.map((p) => p[1]);
const minX = Math.min(...xs);
@@ -151,9 +151,7 @@ function clusterLines(lines: Line[], isHorizontal: boolean): Line[] {
const threshold = 5;
// Check if lines are close enough to merge
- const distance = isHorizontal
- ? Math.abs(line.y1 - current.y1)
- : Math.abs(line.x1 - current.x1);
+ const distance = isHorizontal ? Math.abs(line.y1 - current.y1) : Math.abs(line.x1 - current.x1);
if (distance < threshold) {
// Merge lines
@@ -181,9 +179,7 @@ function clusterLines(lines: Line[], isHorizontal: boolean): Line[] {
/**
* Create cells from intersections
*/
-function createCellsFromIntersections(
- intersections: Intersection[]
-): TableCell[] {
+function createCellsFromIntersections(intersections: Intersection[]): TableCell[] {
if (intersections.length < 4) return [];
// Sort intersections
@@ -193,12 +189,8 @@ function createCellsFromIntersections(
});
// Find unique x and y coordinates
- const xCoords = [...new Set(sorted.map((i) => Math.round(i.x)))].sort(
- (a, b) => a - b
- );
- const yCoords = [...new Set(sorted.map((i) => Math.round(i.y)))].sort(
- (a, b) => a - b
- );
+ const xCoords = [...new Set(sorted.map((i) => Math.round(i.x)))].sort((a, b) => a - b);
+ const yCoords = [...new Set(sorted.map((i) => Math.round(i.y)))].sort((a, b) => a - b);
const cells: TableCell[] = [];
@@ -210,7 +202,7 @@ function createCellsFromIntersections(
y1: yCoords[row],
x2: xCoords[col + 1],
y2: yCoords[row + 1],
- text: '',
+ text: "",
row,
col,
});
@@ -242,7 +234,7 @@ function assignTextToCells(cells: TableCell[], textItems: TextItem[]): void {
}
}
- cell.text = cellText.join(' ').trim();
+ cell.text = cellText.join(" ").trim();
}
}
@@ -267,10 +259,7 @@ function isValidTable(
* Detect tables using heuristic methods (outlined paths)
* Uses line intersection detection to find table grid structures
*/
-export function detectTables(
- paths: Path[],
- textItems: TextItem[] = []
-): DetectedTable[] {
+export function detectTables(paths: Path[], textItems: TextItem[] = []): DetectedTable[] {
const tables: DetectedTable[] = [];
// Extract lines from paths
@@ -324,10 +313,7 @@ export function detectTables(
/**
* Extract table structure from detected table region
*/
-export function extractTableStructure(
- table: DetectedTable,
- _paths: Path[]
-): any {
+export function extractTableStructure(table: DetectedTable, _paths: Path[]): any {
return {
rows: table.rows,
cols: table.cols,
diff --git a/src/processing/textUtils.ts b/src/processing/textUtils.ts
index 607033b..99c6b58 100644
--- a/src/processing/textUtils.ts
+++ b/src/processing/textUtils.ts
@@ -3,35 +3,35 @@
*/
export function strToSubscriptString(str: string): string {
const sub: { [key: string]: string } = {
- '0': '₀',
- '1': '₁',
- '2': '₂',
- '3': '₃',
- '4': '₄',
- '5': '₅',
- '6': '₆',
- '7': '₇',
- '8': '₈',
- '9': '₉',
- '+': '₊',
- '-': '₋',
- 'a': 'ₐ',
- 'e': 'ₑ',
- 'o': 'ₒ',
- 'x': 'ₓ',
- 'ə': 'ₔ',
- 'h': 'ₕ',
- 'k': 'ₖ',
- 'l': 'ₗ',
- 'm': 'ₘ',
- 'n': 'ₙ',
- 'p': 'ₚ',
- 'r': 'ᵣ',
- 's': 'ₛ',
- 't': 'ₜ',
+ "0": "₀",
+ "1": "₁",
+ "2": "₂",
+ "3": "₃",
+ "4": "₄",
+ "5": "₅",
+ "6": "₆",
+ "7": "₇",
+ "8": "₈",
+ "9": "₉",
+ "+": "₊",
+ "-": "₋",
+ a: "ₐ",
+ e: "ₑ",
+ o: "ₒ",
+ x: "ₓ",
+ ə: "ₔ",
+ h: "ₕ",
+ k: "ₖ",
+ l: "ₗ",
+ m: "ₘ",
+ n: "ₙ",
+ p: "ₚ",
+ r: "ᵣ",
+ s: "ₛ",
+ t: "ₜ",
};
- let subStr = '';
+ let subStr = "";
for (let i = 0; i < str.length; i++) {
if (!sub[str[i]]) {
return str;
@@ -47,65 +47,65 @@ export function strToSubscriptString(str: string): string {
*/
export function strToPostScript(str: string): string {
const post: { [key: string]: string } = {
- '0': '⁰',
- '1': '¹',
- '2': '²',
- '3': '³',
- '4': '⁴',
- '5': '⁵',
- '6': '⁶',
- '7': '⁷',
- '8': '⁸',
- '9': '⁹',
- '+': '⁺',
- '-': '⁻',
- 'a': 'ᵃ',
- 'b': 'ᵇ',
- 'c': 'ᶜ',
- 'd': 'ᵈ',
- 'e': 'ᵉ',
- 'f': 'ᶠ',
- 'g': 'ᵍ',
- 'h': 'ʰ',
- 'i': 'ⁱ',
- 'j': 'ʲ',
- 'k': 'ᵏ',
- 'l': 'ˡ',
- 'm': 'ᵐ',
- 'n': 'ⁿ',
- 'o': 'ᵒ',
- 'p': 'ᵖ',
- 'r': 'ʳ',
- 's': 'ˢ',
- 't': 'ᵗ',
- 'u': 'ᵘ',
- 'v': 'ᵛ',
- 'w': 'ʷ',
- 'x': 'ˣ',
- 'y': 'ʸ',
- 'z': 'ᶻ',
- 'A': 'ᴬ',
- 'B': 'ᴮ',
- 'D': 'ᴰ',
- 'E': 'ᴱ',
- 'G': 'ᴳ',
- 'H': 'ᴴ',
- 'I': 'ᴵ',
- 'J': 'ᴶ',
- 'K': 'ᴷ',
- 'L': 'ᴸ',
- 'M': 'ᴹ',
- 'N': 'ᴺ',
- 'O': 'ᴼ',
- 'P': 'ᴾ',
- 'R': 'ᴿ',
- 'T': 'ᵀ',
- 'U': 'ᵁ',
- 'V': 'ⱽ',
- 'W': 'ᵂ',
+ "0": "⁰",
+ "1": "¹",
+ "2": "²",
+ "3": "³",
+ "4": "⁴",
+ "5": "⁵",
+ "6": "⁶",
+ "7": "⁷",
+ "8": "⁸",
+ "9": "⁹",
+ "+": "⁺",
+ "-": "⁻",
+ a: "ᵃ",
+ b: "ᵇ",
+ c: "ᶜ",
+ d: "ᵈ",
+ e: "ᵉ",
+ f: "ᶠ",
+ g: "ᵍ",
+ h: "ʰ",
+ i: "ⁱ",
+ j: "ʲ",
+ k: "ᵏ",
+ l: "ˡ",
+ m: "ᵐ",
+ n: "ⁿ",
+ o: "ᵒ",
+ p: "ᵖ",
+ r: "ʳ",
+ s: "ˢ",
+ t: "ᵗ",
+ u: "ᵘ",
+ v: "ᵛ",
+ w: "ʷ",
+ x: "ˣ",
+ y: "ʸ",
+ z: "ᶻ",
+ A: "ᴬ",
+ B: "ᴮ",
+ D: "ᴰ",
+ E: "ᴱ",
+ G: "ᴳ",
+ H: "ᴴ",
+ I: "ᴵ",
+ J: "ᴶ",
+ K: "ᴷ",
+ L: "ᴸ",
+ M: "ᴹ",
+ N: "ᴺ",
+ O: "ᴼ",
+ P: "ᴾ",
+ R: "ᴿ",
+ T: "ᵀ",
+ U: "ᵁ",
+ V: "ⱽ",
+ W: "ᵂ",
};
- let postStr = '';
+ let postStr = "";
for (let i = 0; i < str.length; i++) {
if (!post[str[i]]) {
return str;