A fast, helpful, and open-source document parser
add debug utils
Logan Markewich269d5e5b54a4
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 376b0b0..068816f 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -71,6 +71,55 @@ You can test your changes locally:./dist/src/index.js screenshot document.pdf -o ./screenshots```+### Debugging Grid Projection++When working on the grid projection algorithm (`src/processing/gridProjection.ts`), you can enable built-in debug logging and visual output instead of adding ad-hoc `console.log` statements.++**Debug logging** traces every decision the projection makes — block detection, anchor extraction, snap assignment, rendering, and flowing text classification:++```bash+# Log all projection decisions to stderr+./dist/src/index.js parse document.pdf --debug++# Filter to a specific page+./dist/src/index.js parse document.pdf --debug --debug-page 3++# Filter to elements containing specific text+./dist/src/index.js parse document.pdf --debug --debug-text-filter "Total" "Revenue"++# Filter to a bounding region (x1,y1,x2,y2 in PDF points)+./dist/src/index.js parse document.pdf --debug --debug-region "0,100,300,200"++# Write debug log to a file+./dist/src/index.js parse document.pdf --debug --debug-output ./debug-output+```++**Visual grid export** generates PNG images showing text boxes color-coded by snap type (blue=left, red=right, green=center, gray=floating, yellow=flowing) with anchor lines overlaid. This is useful for comparing against page screenshots to spot projection issues:++```bash+# Generate visualization PNGs (one per page)+./dist/src/index.js parse document.pdf --debug-visualize++# Specify output directory+./dist/src/index.js parse document.pdf --debug-visualize --debug-output ./my-debug+```++These options are also available via the library API:++```typescript+const parser = new LiteParse({+ debug: {+ enabled: true,+ textFilter: ["Total"],+ pageFilter: 2,+ visualize: true,+ visualizePath: "./debug-output",+ }+});+```++See `src/processing/gridDebugLogger.ts` for the full `GridDebugConfig` interface and `src/processing/gridVisualizer.ts` for the visualization renderer.+## Making Changes### Versioning & Changelogsdiff --git a/cli/parse.ts b/cli/parse.tsindex 5d90eba..dd353d4 100644--- a/cli/parse.ts+++ b/cli/parse.ts@@ -1,4 +1,4 @@-import { Command } from "commander";+import { Command, Option } from "commander";import fs from "fs/promises";import { existsSync, readdirSync, statSync } from "fs";import os from "os";@@ -29,6 +29,12 @@ interface ParseCommandOptions {password?: string;config?: string;quiet?: boolean;+ debug?: boolean;+ debugVisualize?: boolean;+ debugOutput?: string;+ debugTextFilter?: string[];+ debugPage?: string;+ debugRegion?: string;}interface ScreenshotCommandOptions {@@ -84,6 +90,18 @@ program.option("--password <password>", "Password for encrypted/protected documents").option("--config <file>", "Config file (JSON)").option("-q, --quiet", "Suppress progress output")+ .addOption(new Option("--debug", "Enable grid projection debug logging").hideHelp())+ .addOption(new Option("--debug-visualize", "Generate grid visualization PNGs").hideHelp())+ .addOption(new Option("--debug-output <path>", "Output directory for debug files").hideHelp())+ .addOption(+ new Option("--debug-text-filter <texts...>", "Filter debug output by text content").hideHelp()+ )+ .addOption(+ new Option("--debug-page <num>", "Filter debug output to specific page number").hideHelp()+ )+ .addOption(+ new Option("--debug-region <coords>", 'Filter to bounding region "x1,y1,x2,y2"').hideHelp()+ ).action(async (file: string, options: ParseCommandOptions) => {try {const quiet = options.quiet || false;@@ -128,6 +146,24 @@ programpassword: options.password,};+ // Build debug config if any debug flags are set+ if (options.debug || options.debugVisualize) {+ let regionFilter: { x1: number; y1: number; x2: number; y2: number } | undefined;+ if (options.debugRegion) {+ const [x1, y1, x2, y2] = options.debugRegion.split(",").map(Number);+ regionFilter = { x1, y1, x2, y2 };+ }+ config.debug = {+ enabled: true,+ visualize: options.debugVisualize,+ visualizePath: options.debugOutput ?? "./debug-output",+ outputPath: options.debugOutput ? `${options.debugOutput}/debug.log` : undefined,+ textFilter: options.debugTextFilter,+ pageFilter: options.debugPage ? parseInt(options.debugPage) : undefined,+ regionFilter,+ };+ }+// Create parserconst parser = new LiteParse(config);diff --git a/src/core/parser.ts b/src/core/parser.tsindex 2557c0f..f066049 100644--- a/src/core/parser.ts+++ b/src/core/parser.ts@@ -172,7 +172,7 @@ export class LiteParse {}// Process pages with complete grid projection (after OCR)- const processedPages = projectPagesToGrid(pages, this.config);+ const processedPages = await projectPagesToGrid(pages, this.config);// Build bounding boxes if enabledif (this.config.preciseBoundingBox) {diff --git a/src/core/types.ts b/src/core/types.tsindex a32593b..3ff1e5d 100644--- a/src/core/types.ts+++ b/src/core/types.ts@@ -4,6 +4,8 @@* - `"json"` — Structured JSON with per-page text items, bounding boxes, and metadata.* - `"text"` — Plain text with spatial layout preserved.*/+import type { GridDebugConfig } from "../processing/gridDebugLogger.js";+export type OutputFormat = "json" | "text";/**@@ -136,6 +138,26 @@ export interface LiteParseConfig {* @defaultValue `undefined`*/password?: string;++ /**+ * Debug configuration for grid projection. When enabled, logs detailed+ * information about how text elements are snapped, anchored, and projected.+ * Can also generate visual PNG overlays of the projection.+ *+ * @example+ * ```typescript+ * const parser = new LiteParse({+ * debug: {+ * enabled: true,+ * textFilter: ["Total", "Revenue"],+ * pageFilter: 2,+ * visualize: true,+ * visualizePath: "./debug-output",+ * }+ * });+ * ```+ */+ debug?: GridDebugConfig;}/**diff --git a/src/lib.ts b/src/lib.tsindex c82775c..b9184da 100644--- a/src/lib.ts+++ b/src/lib.ts@@ -28,3 +28,4 @@ export type {ScreenshotResult,MarkupData,} from "./core/types.js";+export type { GridDebugConfig } from "./processing/gridDebugLogger.js";diff --git a/src/processing/README.md b/src/processing/README.mdindex 546e36a..162f02c 100644--- a/src/processing/README.md+++ b/src/processing/README.md@@ -61,8 +61,8 @@ This is the core algorithm that converts raw PDF text items into readable, prope- `bboxToLine()` - Group text items into lines with Y-tolerance for subscripts- `extractAnchorsPointsFromLines()` - Identify alignment anchors and deduplicate- `tryAlignFloating()` - Align floating bboxes with anchors on adjacent lines-- `projectToGrid()` - Main projection algorithm-- `projectPagesToGrid()` - Process all pages with shared anchors+- `projectToGrid()` - Main projection algorithm (accepts a `GridDebugLogger` for tracing)+- `projectPagesToGrid()` - Process all pages with shared anchors, creates debug logger from config**Constants:**- `FLOATING_SPACES = 2` - Minimum spaces between floating text@@ -79,6 +79,43 @@ This is the core algorithm that converts raw PDF text items into readable, prope---+### gridDebugLogger.ts+**Targeted debug logging for grid projection.**++Provides a `GridDebugLogger` class that traces projection decisions at every stage — block detection, anchor extraction, snap assignment, rendering, flowing text classification, and forward anchor updates. Uses a filter system to narrow output to specific elements.++**`GridDebugConfig` options:**+- `enabled` - Master switch+- `textFilter` - Only log elements whose text contains these substrings (case-insensitive)+- `lineFilter` - Only log elements on these line indices (0-based)+- `pageFilter` - Only log elements on this page number (1-indexed)+- `regionFilter` - Only log elements within a bounding region `{ x1, y1, x2, y2 }`+- `outputPath` - Write log to a file instead of stderr+- `visualize` - Generate PNG visualizations (see gridVisualizer.ts)+- `visualizePath` - Directory for visualization PNGs (default: `./debug-output`)++**Log phases:** `page`, `blocks`, `anchors`, `snap`, `render`, `flowing`, `forward-anchor`, `dedup`, `lines`++When disabled, a zero-cost `NoopGridDebugLogger` singleton is used — no runtime overhead in production.++---++### gridVisualizer.ts+**PNG visualization of grid projection state.**++Renders a page's projection as a PNG image using sharp with an SVG overlay. Text boxes are drawn as colored rectangles by snap type:+- **Blue** — left snap+- **Red** — right snap+- **Green** — center snap+- **Gray** — floating (unsnapped)+- **Yellow** — flowing text++Also draws vertical dashed lines for anchor positions and block boundary outlines. Includes a color legend.++Called automatically when `debug.visualize` is enabled. Output files are named `page-{N}-grid.png` in the configured directory.++---+### bbox.ts**Bounding box construction and OCR integration.**diff --git a/src/processing/grid.ts b/src/processing/grid.tsindex abb2747..f073296 100644--- a/src/processing/grid.ts+++ b/src/processing/grid.ts@@ -6,10 +6,10 @@ import { DEFAULT_CONFIG } from "../core/config.js";/*** Projects text items onto a grid for spatial text extraction*/-export function projectPagesToGrid(+export async function projectPagesToGrid(pages: PageData[],config?: Partial<LiteParseConfig>-): ParsedPage[] {+): Promise<ParsedPage[]> {const fullConfig = {...DEFAULT_CONFIG,...config,diff --git a/src/processing/gridProjection.test.ts b/src/processing/gridProjection.test.tsindex 63a4bb1..11aeaa1 100644--- a/src/processing/gridProjection.test.ts+++ b/src/processing/gridProjection.test.ts@@ -2,6 +2,7 @@ import { expect, describe, it } from "vitest";import { bboxToLine, projectPagesToGrid, projectToGrid } from "./gridProjection";import { ProjectionTextBox } from "../core/types";import { DEFAULT_CONFIG } from "../core/config";+import { NOOP_LOGGER } from "./gridDebugLogger";describe("test bboxToLine", () => {it("test same line that can merge", () => {@@ -279,7 +280,14 @@ describe("test projectToGrid", () => {forwardAnchorCenter: {},},};- const result = projectToGrid(config, page, projectionBoxes, prevAnchors, totalPages);+ const result = projectToGrid(+ config,+ page,+ projectionBoxes,+ prevAnchors,+ totalPages,+ NOOP_LOGGER+ );expect(result).toStrictEqual(expectedOutput);});@@ -311,7 +319,14 @@ describe("test projectToGrid", () => {text: " Name Age\n Alice 30",prevAnchors: prevAnchors,};- const result = projectToGrid(config, page, projectionBoxes, prevAnchors, totalPages);+ const result = projectToGrid(+ config,+ page,+ projectionBoxes,+ prevAnchors,+ totalPages,+ NOOP_LOGGER+ );expect(result).toStrictEqual(expectedOutput);});@@ -335,7 +350,14 @@ describe("test projectToGrid", () => {text: " Revenue 500",prevAnchors: prevAnchors,};- const result = projectToGrid(config, page, projectionBoxes, prevAnchors, totalPages);+ const result = projectToGrid(+ config,+ page,+ projectionBoxes,+ prevAnchors,+ totalPages,+ NOOP_LOGGER+ );expect(result).toStrictEqual(expectedOutput);});});@@ -436,7 +458,7 @@ const mockPageData = [];describe("test projectPagesToGrid", () => {- it("test pages projection", () => {+ it("test pages projection", async () => {const config = { ...DEFAULT_CONFIG, preserveLayoutAlignmentAcrossPages: false };const expectedOutput = [{@@ -515,7 +537,7 @@ describe("test projectPagesToGrid", () => {boundingBoxes: [],},];- const result = projectPagesToGrid(mockPageData, config);+ const result = await projectPagesToGrid(mockPageData, config);expect(result).toStrictEqual(expectedOutput);});});diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.tsindex 3a75505..35cdc6f 100644--- a/src/processing/gridProjection.ts+++ b/src/processing/gridProjection.ts@@ -5,6 +5,8 @@ import { ProjectionTextBox, Coordinates, LiteParseConfig, ParsedPage } from "../import { PageData } from "../engines/pdf/interface.js";import { applyMarkupTags } from "./markupUtils.js";+import { GridDebugLogger, createGridDebugLogger } from "./gridDebugLogger.js";+import { renderAllVisualizations } from "./gridVisualizer.js";// Minimum spaces between unsnapped bboxes (likely justified textconst FLOATING_SPACES = 2;@@ -1287,7 +1289,8 @@ export function projectToGrid(page: PageData,projectionBoxes: ProjectionTextBox[],prevAnchors: PrevAnchors,- totalPages: number+ totalPages: number,+ logger: GridDebugLogger): { text: string; prevAnchors: PrevAnchors } {// detect '.' garbage in the lineslet dotCount = 0;@@ -1341,6 +1344,11 @@ export function projectToGrid(handleRotationReadingOrder(projectionBoxes, page.height);const lines = bboxToLine(projectionBoxes, medianWidth, medianHeight, page.width);+ // Log line composition+ for (let i = 0; i < lines.length; i++) {+ logger.logLineComposition(i, lines[i]);+ }+// remove unprojectable text and apply markup to final linesfor (let i = 0; i < lines.length; ++i) {const line = filterUnprojectableText(config, lines[i]);@@ -1396,6 +1404,11 @@ export function projectToGrid(}}+ // Log block assignments+ for (let bi = 0; bi < blocks.length; bi++) {+ logger.logBlock(bi, blocks[bi].start, blocks[bi].end);+ }+for (const block of blocks) {const blockLines = lines.slice(block.start, block.end);const { anchorLeft, anchorRight, anchorCenter } = extractAnchorsPointsFromLines(@@ -1403,9 +1416,18 @@ export function projectToGrid(page);+ logger.logAnchors(anchorLeft, anchorRight, anchorCenter);+// Block-level classification: if entire block is clearly flowing text,// render it simply and skip grid projectionif (isFlowingTextBlock(blockLines, anchorLeft, anchorRight, anchorCenter, page.width)) {+ logger.logFlowingBlock(block.start, block.end);+ // Capture flowing lines for visualization+ if (logger.shouldVisualize) {+ for (let li = block.start; li < block.end; li++) {+ logger.captureLineBoxes(li, lines[li], true);+ }+ }if (!config.preserveLayoutAlignmentAcrossPages) {const sizes = getMedianTextBoxSize(blockLines.flat());medianWidth = sizes.width;@@ -1414,6 +1436,8 @@ export function projectToGrid(continue;}+ logger.logStructuredBlock(block.start, block.end);+const snapMaps: SnapMaps = {left: [],right: [],@@ -1505,6 +1529,7 @@ export function projectToGrid(}prevBbox = bbox;+ logger.logSnapAssignment(bbox, lineIndex, boxIndex);if (!bbox.snap) {uniqueSnaps.add(Math.round(bbox.x));} else if (bbox.snap == "left") {@@ -1542,7 +1567,7 @@ export function projectToGrid(const columnGapThreshold = medianWidth * FLOWING_COLUMN_GAP_MULTIPLIER;// Helper to mark and render a line as flowing- function markFlowing(lineIndex: number): void {+ function markFlowing(lineIndex: number, reason: string): void {const line = lines[lineIndex];if (!rawLines[lineIndex]) {rawLines[lineIndex] = "";@@ -1550,6 +1575,7 @@ export function projectToGrid(}rawLines[lineIndex] = renderLineAsFlowingText(line, blockMinX, medianWidth);flowingLines.add(lineIndex);+ logger.logFlowingLine(lineIndex, reason);}// First pass: detect clearly flowing lines (wide, no column gaps, enough items)@@ -1565,7 +1591,7 @@ export function projectToGrid(lineSpan > page.width * FLOWING_WIDE_LINE_RATIO &&lineMaxGap(line) < columnGapThreshold) {- markFlowing(lineIndex);+ markFlowing(lineIndex, "wide span, no column gaps");}}@@ -1577,7 +1603,7 @@ export function projectToGrid(const line = lines[lineIndex];if (line.length === 0) continue;if (flowingLines.has(lineIndex - 1) && lineMaxGap(line) < columnGapThreshold) {- markFlowing(lineIndex);+ markFlowing(lineIndex, `forward propagation from line ${lineIndex - 1}`);}}// Backward sweep: propagate flowing status upward@@ -1586,11 +1612,18 @@ export function projectToGrid(const line = lines[lineIndex];if (line.length === 0) continue;if (flowingLines.has(lineIndex + 1) && lineMaxGap(line) < columnGapThreshold) {- markFlowing(lineIndex);+ markFlowing(lineIndex, `backward propagation from line ${lineIndex + 1}`);}}}+ // Capture line boxes for visualization (after snap + flowing assignment)+ if (logger.shouldVisualize) {+ for (let li = block.start; li < block.end; li++) {+ logger.captureLineBoxes(li, lines[li], flowingLines.has(li));+ }+ }+while (hasChanged || snapMaps.right.length || snapMaps.left.length || snapMaps.center.length) {hasChanged = false;@@ -1664,6 +1697,7 @@ export function projectToGrid(bbox.rendered = true;hasChanged = true;+ logger.logRender(bbox, lineIndex, targetX, "floating");let nextBbox: ProjectionTextBox | null = null;if (line.length > boxIndex + 1) {@@ -1739,6 +1773,7 @@ export function projectToGrid(}forwardAnchors.left[snapMaps.left[0]] = targetX;+ logger.logForwardAnchor("left", snapMaps.left[0], targetX);for (const currentLeftSnapBox of thisTurnSnap) {const lineIndex = currentLeftSnapBox.lineIndex;@@ -1748,6 +1783,7 @@ export function projectToGrid(}rawLines[lineIndex] += currentLeftSnapBox.bbox.str;currentLeftSnapBox.bbox.rendered = true;+ logger.logRender(currentLeftSnapBox.bbox, lineIndex, targetX, "left-snap");let nextBbox: ProjectionTextBox | null = null;if (lines[lineIndex].length > currentLeftSnapBox.boxIndex + 1) {@@ -1826,6 +1862,7 @@ export function projectToGrid(targetX = prevAnchors.forwardAnchorRight[snapMaps.right[0]];}forwardAnchors.right[snapMaps.right[0]] = targetX;+ logger.logForwardAnchor("right", snapMaps.right[0], targetX);for (const currentRightSnapBox of thisTurnSnap) {const lineIndex = currentRightSnapBox.lineIndex;@@ -1838,6 +1875,7 @@ export function projectToGrid(}rawLines[lineIndex] += currentRightSnapBox.bbox.str;currentRightSnapBox.bbox.rendered = true;+ logger.logRender(currentRightSnapBox.bbox, lineIndex, targetX, "right-snap");let nextBbox: ProjectionTextBox | null = null;if (lines[lineIndex].length > currentRightSnapBox.boxIndex + 1) {@@ -1913,6 +1951,7 @@ export function projectToGrid(targetX = prevAnchors.forwardAnchorCenter[snapMaps.center[0]];}forwardAnchors.center[snapMaps.center[0]] = targetX;+ logger.logForwardAnchor("center", snapMaps.center[0], targetX);for (const currentCenterSnapBox of thisTurnSnap) {if (flowingLines.has(currentCenterSnapBox.lineIndex)) continue;if (@@ -1928,6 +1967,12 @@ export function projectToGrid(}rawLines[currentCenterSnapBox.lineIndex] += currentCenterSnapBox.bbox.str;currentCenterSnapBox.bbox.rendered = true;+ logger.logRender(+ currentCenterSnapBox.bbox,+ currentCenterSnapBox.lineIndex,+ targetX,+ "center-snap"+ );}snapMaps.center.shift();}@@ -1948,7 +1993,11 @@ export function projectToGrid(};}-export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig): ParsedPage[] {+export async function projectPagesToGrid(+ pages: PageData[],+ config: LiteParseConfig+): Promise<ParsedPage[]> {+ const logger = createGridDebugLogger(config.debug);const prevAnchors: PrevAnchors = {forwardAnchorLeft: {},forwardAnchorRight: {},@@ -1958,6 +2007,8 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):const results: ParsedPage[] = [];for (const page of pages) {+ logger.setPage(page.pageNum, page.width, page.height);+// Build projection boxes from text itemsconst projectionBoxes = buildBbox(page, config);@@ -1967,7 +2018,8 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):page,projectionBoxes,prevAnchors,- pages.length+ pages.length,+ logger);// Update forward anchors if preserving across pages@@ -1997,5 +2049,17 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):// Clean raw text (margin detection, etc)cleanRawText(results, config);+ // Flush debug log and render visualizations+ if (logger.enabled) {+ await logger.flush();+ if (logger.shouldVisualize) {+ const vizPath = logger.debugConfig.visualizePath ?? "./debug-output";+ const paths = await renderAllVisualizations(logger.visualizerPages, vizPath);+ for (const p of paths) {+ process.stderr.write(`[grid-debug] Visualization: ${p}\n`);+ }+ }+ }+return results;}