Skip to content

A fast, helpful, and open-source document parser

add debug utils

Logan Markewich269d5e5b54a4
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 376b0b0..068816f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -71,6 +71,55 @@ You can test your changes locally:
./dist/src/index.js screenshot document.pdf -o ./screenshots
```
+### Debugging Grid Projection
+
+When working on the grid projection algorithm (`src/processing/gridProjection.ts`), you can enable built-in debug logging and visual output instead of adding ad-hoc `console.log` statements.
+
+**Debug logging** traces every decision the projection makes — block detection, anchor extraction, snap assignment, rendering, and flowing text classification:
+
+```bash
+# Log all projection decisions to stderr
+./dist/src/index.js parse document.pdf --debug
+
+# Filter to a specific page
+./dist/src/index.js parse document.pdf --debug --debug-page 3
+
+# Filter to elements containing specific text
+./dist/src/index.js parse document.pdf --debug --debug-text-filter "Total" "Revenue"
+
+# Filter to a bounding region (x1,y1,x2,y2 in PDF points)
+./dist/src/index.js parse document.pdf --debug --debug-region "0,100,300,200"
+
+# Write debug log to a file
+./dist/src/index.js parse document.pdf --debug --debug-output ./debug-output
+```
+
+**Visual grid export** generates PNG images showing text boxes color-coded by snap type (blue=left, red=right, green=center, gray=floating, yellow=flowing) with anchor lines overlaid. This is useful for comparing against page screenshots to spot projection issues:
+
+```bash
+# Generate visualization PNGs (one per page)
+./dist/src/index.js parse document.pdf --debug-visualize
+
+# Specify output directory
+./dist/src/index.js parse document.pdf --debug-visualize --debug-output ./my-debug
+```
+
+These options are also available via the library API:
+
+```typescript
+const parser = new LiteParse({
+ debug: {
+ enabled: true,
+ textFilter: ["Total"],
+ pageFilter: 2,
+ visualize: true,
+ visualizePath: "./debug-output",
+ }
+});
+```
+
+See `src/processing/gridDebugLogger.ts` for the full `GridDebugConfig` interface and `src/processing/gridVisualizer.ts` for the visualization renderer.
+
## Making Changes
### Versioning & Changelogs
diff --git a/cli/parse.ts b/cli/parse.ts
index 5d90eba..dd353d4 100644
--- a/cli/parse.ts
+++ b/cli/parse.ts
@@ -1,4 +1,4 @@
-import { Command } from "commander";
+import { Command, Option } from "commander";
import fs from "fs/promises";
import { existsSync, readdirSync, statSync } from "fs";
import os from "os";
@@ -29,6 +29,12 @@ interface ParseCommandOptions {
password?: string;
config?: string;
quiet?: boolean;
+ debug?: boolean;
+ debugVisualize?: boolean;
+ debugOutput?: string;
+ debugTextFilter?: string[];
+ debugPage?: string;
+ debugRegion?: string;
}
interface ScreenshotCommandOptions {
@@ -84,6 +90,18 @@ program
.option("--password <password>", "Password for encrypted/protected documents")
.option("--config <file>", "Config file (JSON)")
.option("-q, --quiet", "Suppress progress output")
+ .addOption(new Option("--debug", "Enable grid projection debug logging").hideHelp())
+ .addOption(new Option("--debug-visualize", "Generate grid visualization PNGs").hideHelp())
+ .addOption(new Option("--debug-output <path>", "Output directory for debug files").hideHelp())
+ .addOption(
+ new Option("--debug-text-filter <texts...>", "Filter debug output by text content").hideHelp()
+ )
+ .addOption(
+ new Option("--debug-page <num>", "Filter debug output to specific page number").hideHelp()
+ )
+ .addOption(
+ new Option("--debug-region <coords>", 'Filter to bounding region "x1,y1,x2,y2"').hideHelp()
+ )
.action(async (file: string, options: ParseCommandOptions) => {
try {
const quiet = options.quiet || false;
@@ -128,6 +146,24 @@ program
password: options.password,
};
+ // Build debug config if any debug flags are set
+ if (options.debug || options.debugVisualize) {
+ let regionFilter: { x1: number; y1: number; x2: number; y2: number } | undefined;
+ if (options.debugRegion) {
+ const [x1, y1, x2, y2] = options.debugRegion.split(",").map(Number);
+ regionFilter = { x1, y1, x2, y2 };
+ }
+ config.debug = {
+ enabled: true,
+ visualize: options.debugVisualize,
+ visualizePath: options.debugOutput ?? "./debug-output",
+ outputPath: options.debugOutput ? `${options.debugOutput}/debug.log` : undefined,
+ textFilter: options.debugTextFilter,
+ pageFilter: options.debugPage ? parseInt(options.debugPage) : undefined,
+ regionFilter,
+ };
+ }
+
// Create parser
const parser = new LiteParse(config);
diff --git a/src/core/parser.ts b/src/core/parser.ts
index 2557c0f..f066049 100644
--- a/src/core/parser.ts
+++ b/src/core/parser.ts
@@ -172,7 +172,7 @@ export class LiteParse {
}
// Process pages with complete grid projection (after OCR)
- const processedPages = projectPagesToGrid(pages, this.config);
+ const processedPages = await projectPagesToGrid(pages, this.config);
// Build bounding boxes if enabled
if (this.config.preciseBoundingBox) {
diff --git a/src/core/types.ts b/src/core/types.ts
index a32593b..3ff1e5d 100644
--- a/src/core/types.ts
+++ b/src/core/types.ts
@@ -4,6 +4,8 @@
* - `"json"` — Structured JSON with per-page text items, bounding boxes, and metadata.
* - `"text"` — Plain text with spatial layout preserved.
*/
+import type { GridDebugConfig } from "../processing/gridDebugLogger.js";
+
export type OutputFormat = "json" | "text";
/**
@@ -136,6 +138,26 @@ export interface LiteParseConfig {
* @defaultValue `undefined`
*/
password?: string;
+
+ /**
+ * Debug configuration for grid projection. When enabled, logs detailed
+ * information about how text elements are snapped, anchored, and projected.
+ * Can also generate visual PNG overlays of the projection.
+ *
+ * @example
+ * ```typescript
+ * const parser = new LiteParse({
+ * debug: {
+ * enabled: true,
+ * textFilter: ["Total", "Revenue"],
+ * pageFilter: 2,
+ * visualize: true,
+ * visualizePath: "./debug-output",
+ * }
+ * });
+ * ```
+ */
+ debug?: GridDebugConfig;
}
/**
diff --git a/src/lib.ts b/src/lib.ts
index c82775c..b9184da 100644
--- a/src/lib.ts
+++ b/src/lib.ts
@@ -28,3 +28,4 @@ export type {
ScreenshotResult,
MarkupData,
} from "./core/types.js";
+export type { GridDebugConfig } from "./processing/gridDebugLogger.js";
diff --git a/src/processing/README.md b/src/processing/README.md
index 546e36a..162f02c 100644
--- a/src/processing/README.md
+++ b/src/processing/README.md
@@ -61,8 +61,8 @@ This is the core algorithm that converts raw PDF text items into readable, prope
- `bboxToLine()` - Group text items into lines with Y-tolerance for subscripts
- `extractAnchorsPointsFromLines()` - Identify alignment anchors and deduplicate
- `tryAlignFloating()` - Align floating bboxes with anchors on adjacent lines
-- `projectToGrid()` - Main projection algorithm
-- `projectPagesToGrid()` - Process all pages with shared anchors
+- `projectToGrid()` - Main projection algorithm (accepts a `GridDebugLogger` for tracing)
+- `projectPagesToGrid()` - Process all pages with shared anchors, creates debug logger from config
**Constants:**
- `FLOATING_SPACES = 2` - Minimum spaces between floating text
@@ -79,6 +79,43 @@ This is the core algorithm that converts raw PDF text items into readable, prope
---
+### gridDebugLogger.ts
+**Targeted debug logging for grid projection.**
+
+Provides a `GridDebugLogger` class that traces projection decisions at every stage — block detection, anchor extraction, snap assignment, rendering, flowing text classification, and forward anchor updates. Uses a filter system to narrow output to specific elements.
+
+**`GridDebugConfig` options:**
+- `enabled` - Master switch
+- `textFilter` - Only log elements whose text contains these substrings (case-insensitive)
+- `lineFilter` - Only log elements on these line indices (0-based)
+- `pageFilter` - Only log elements on this page number (1-indexed)
+- `regionFilter` - Only log elements within a bounding region `{ x1, y1, x2, y2 }`
+- `outputPath` - Write log to a file instead of stderr
+- `visualize` - Generate PNG visualizations (see gridVisualizer.ts)
+- `visualizePath` - Directory for visualization PNGs (default: `./debug-output`)
+
+**Log phases:** `page`, `blocks`, `anchors`, `snap`, `render`, `flowing`, `forward-anchor`, `dedup`, `lines`
+
+When disabled, a zero-cost `NoopGridDebugLogger` singleton is used — no runtime overhead in production.
+
+---
+
+### gridVisualizer.ts
+**PNG visualization of grid projection state.**
+
+Renders a page's projection as a PNG image using sharp with an SVG overlay. Text boxes are drawn as colored rectangles by snap type:
+- **Blue** — left snap
+- **Red** — right snap
+- **Green** — center snap
+- **Gray** — floating (unsnapped)
+- **Yellow** — flowing text
+
+Also draws vertical dashed lines for anchor positions and block boundary outlines. Includes a color legend.
+
+Called automatically when `debug.visualize` is enabled. Output files are named `page-{N}-grid.png` in the configured directory.
+
+---
+
### bbox.ts
**Bounding box construction and OCR integration.**
diff --git a/src/processing/grid.ts b/src/processing/grid.ts
index abb2747..f073296 100644
--- a/src/processing/grid.ts
+++ b/src/processing/grid.ts
@@ -6,10 +6,10 @@ import { DEFAULT_CONFIG } from "../core/config.js";
/**
* Projects text items onto a grid for spatial text extraction
*/
-export function projectPagesToGrid(
+export async function projectPagesToGrid(
pages: PageData[],
config?: Partial<LiteParseConfig>
-): ParsedPage[] {
+): Promise<ParsedPage[]> {
const fullConfig = {
...DEFAULT_CONFIG,
...config,
diff --git a/src/processing/gridProjection.test.ts b/src/processing/gridProjection.test.ts
index 63a4bb1..11aeaa1 100644
--- a/src/processing/gridProjection.test.ts
+++ b/src/processing/gridProjection.test.ts
@@ -2,6 +2,7 @@ import { expect, describe, it } from "vitest";
import { bboxToLine, projectPagesToGrid, projectToGrid } from "./gridProjection";
import { ProjectionTextBox } from "../core/types";
import { DEFAULT_CONFIG } from "../core/config";
+import { NOOP_LOGGER } from "./gridDebugLogger";
describe("test bboxToLine", () => {
it("test same line that can merge", () => {
@@ -279,7 +280,14 @@ describe("test projectToGrid", () => {
forwardAnchorCenter: {},
},
};
- const result = projectToGrid(config, page, projectionBoxes, prevAnchors, totalPages);
+ const result = projectToGrid(
+ config,
+ page,
+ projectionBoxes,
+ prevAnchors,
+ totalPages,
+ NOOP_LOGGER
+ );
expect(result).toStrictEqual(expectedOutput);
});
@@ -311,7 +319,14 @@ describe("test projectToGrid", () => {
text: " Name Age\n Alice 30",
prevAnchors: prevAnchors,
};
- const result = projectToGrid(config, page, projectionBoxes, prevAnchors, totalPages);
+ const result = projectToGrid(
+ config,
+ page,
+ projectionBoxes,
+ prevAnchors,
+ totalPages,
+ NOOP_LOGGER
+ );
expect(result).toStrictEqual(expectedOutput);
});
@@ -335,7 +350,14 @@ describe("test projectToGrid", () => {
text: " Revenue 500",
prevAnchors: prevAnchors,
};
- const result = projectToGrid(config, page, projectionBoxes, prevAnchors, totalPages);
+ const result = projectToGrid(
+ config,
+ page,
+ projectionBoxes,
+ prevAnchors,
+ totalPages,
+ NOOP_LOGGER
+ );
expect(result).toStrictEqual(expectedOutput);
});
});
@@ -436,7 +458,7 @@ const mockPageData = [
];
describe("test projectPagesToGrid", () => {
- it("test pages projection", () => {
+ it("test pages projection", async () => {
const config = { ...DEFAULT_CONFIG, preserveLayoutAlignmentAcrossPages: false };
const expectedOutput = [
{
@@ -515,7 +537,7 @@ describe("test projectPagesToGrid", () => {
boundingBoxes: [],
},
];
- const result = projectPagesToGrid(mockPageData, config);
+ const result = await projectPagesToGrid(mockPageData, config);
expect(result).toStrictEqual(expectedOutput);
});
});
diff --git a/src/processing/gridProjection.ts b/src/processing/gridProjection.ts
index 3a75505..35cdc6f 100644
--- a/src/processing/gridProjection.ts
+++ b/src/processing/gridProjection.ts
@@ -5,6 +5,8 @@ import { ProjectionTextBox, Coordinates, LiteParseConfig, ParsedPage } from "../
import { PageData } from "../engines/pdf/interface.js";
import { applyMarkupTags } from "./markupUtils.js";
+import { GridDebugLogger, createGridDebugLogger } from "./gridDebugLogger.js";
+import { renderAllVisualizations } from "./gridVisualizer.js";
// Minimum spaces between unsnapped bboxes (likely justified text
const FLOATING_SPACES = 2;
@@ -1287,7 +1289,8 @@ export function projectToGrid(
page: PageData,
projectionBoxes: ProjectionTextBox[],
prevAnchors: PrevAnchors,
- totalPages: number
+ totalPages: number,
+ logger: GridDebugLogger
): { text: string; prevAnchors: PrevAnchors } {
// detect '.' garbage in the lines
let dotCount = 0;
@@ -1341,6 +1344,11 @@ export function projectToGrid(
handleRotationReadingOrder(projectionBoxes, page.height);
const lines = bboxToLine(projectionBoxes, medianWidth, medianHeight, page.width);
+ // Log line composition
+ for (let i = 0; i < lines.length; i++) {
+ logger.logLineComposition(i, lines[i]);
+ }
+
// remove unprojectable text and apply markup to final lines
for (let i = 0; i < lines.length; ++i) {
const line = filterUnprojectableText(config, lines[i]);
@@ -1396,6 +1404,11 @@ export function projectToGrid(
}
}
+ // Log block assignments
+ for (let bi = 0; bi < blocks.length; bi++) {
+ logger.logBlock(bi, blocks[bi].start, blocks[bi].end);
+ }
+
for (const block of blocks) {
const blockLines = lines.slice(block.start, block.end);
const { anchorLeft, anchorRight, anchorCenter } = extractAnchorsPointsFromLines(
@@ -1403,9 +1416,18 @@ export function projectToGrid(
page
);
+ logger.logAnchors(anchorLeft, anchorRight, anchorCenter);
+
// Block-level classification: if entire block is clearly flowing text,
// render it simply and skip grid projection
if (isFlowingTextBlock(blockLines, anchorLeft, anchorRight, anchorCenter, page.width)) {
+ logger.logFlowingBlock(block.start, block.end);
+ // Capture flowing lines for visualization
+ if (logger.shouldVisualize) {
+ for (let li = block.start; li < block.end; li++) {
+ logger.captureLineBoxes(li, lines[li], true);
+ }
+ }
if (!config.preserveLayoutAlignmentAcrossPages) {
const sizes = getMedianTextBoxSize(blockLines.flat());
medianWidth = sizes.width;
@@ -1414,6 +1436,8 @@ export function projectToGrid(
continue;
}
+ logger.logStructuredBlock(block.start, block.end);
+
const snapMaps: SnapMaps = {
left: [],
right: [],
@@ -1505,6 +1529,7 @@ export function projectToGrid(
}
prevBbox = bbox;
+ logger.logSnapAssignment(bbox, lineIndex, boxIndex);
if (!bbox.snap) {
uniqueSnaps.add(Math.round(bbox.x));
} else if (bbox.snap == "left") {
@@ -1542,7 +1567,7 @@ export function projectToGrid(
const columnGapThreshold = medianWidth * FLOWING_COLUMN_GAP_MULTIPLIER;
// Helper to mark and render a line as flowing
- function markFlowing(lineIndex: number): void {
+ function markFlowing(lineIndex: number, reason: string): void {
const line = lines[lineIndex];
if (!rawLines[lineIndex]) {
rawLines[lineIndex] = "";
@@ -1550,6 +1575,7 @@ export function projectToGrid(
}
rawLines[lineIndex] = renderLineAsFlowingText(line, blockMinX, medianWidth);
flowingLines.add(lineIndex);
+ logger.logFlowingLine(lineIndex, reason);
}
// First pass: detect clearly flowing lines (wide, no column gaps, enough items)
@@ -1565,7 +1591,7 @@ export function projectToGrid(
lineSpan > page.width * FLOWING_WIDE_LINE_RATIO &&
lineMaxGap(line) < columnGapThreshold
) {
- markFlowing(lineIndex);
+ markFlowing(lineIndex, "wide span, no column gaps");
}
}
@@ -1577,7 +1603,7 @@ export function projectToGrid(
const line = lines[lineIndex];
if (line.length === 0) continue;
if (flowingLines.has(lineIndex - 1) && lineMaxGap(line) < columnGapThreshold) {
- markFlowing(lineIndex);
+ markFlowing(lineIndex, `forward propagation from line ${lineIndex - 1}`);
}
}
// Backward sweep: propagate flowing status upward
@@ -1586,11 +1612,18 @@ export function projectToGrid(
const line = lines[lineIndex];
if (line.length === 0) continue;
if (flowingLines.has(lineIndex + 1) && lineMaxGap(line) < columnGapThreshold) {
- markFlowing(lineIndex);
+ markFlowing(lineIndex, `backward propagation from line ${lineIndex + 1}`);
}
}
}
+ // Capture line boxes for visualization (after snap + flowing assignment)
+ if (logger.shouldVisualize) {
+ for (let li = block.start; li < block.end; li++) {
+ logger.captureLineBoxes(li, lines[li], flowingLines.has(li));
+ }
+ }
+
while (hasChanged || snapMaps.right.length || snapMaps.left.length || snapMaps.center.length) {
hasChanged = false;
@@ -1664,6 +1697,7 @@ export function projectToGrid(
bbox.rendered = true;
hasChanged = true;
+ logger.logRender(bbox, lineIndex, targetX, "floating");
let nextBbox: ProjectionTextBox | null = null;
if (line.length > boxIndex + 1) {
@@ -1739,6 +1773,7 @@ export function projectToGrid(
}
forwardAnchors.left[snapMaps.left[0]] = targetX;
+ logger.logForwardAnchor("left", snapMaps.left[0], targetX);
for (const currentLeftSnapBox of thisTurnSnap) {
const lineIndex = currentLeftSnapBox.lineIndex;
@@ -1748,6 +1783,7 @@ export function projectToGrid(
}
rawLines[lineIndex] += currentLeftSnapBox.bbox.str;
currentLeftSnapBox.bbox.rendered = true;
+ logger.logRender(currentLeftSnapBox.bbox, lineIndex, targetX, "left-snap");
let nextBbox: ProjectionTextBox | null = null;
if (lines[lineIndex].length > currentLeftSnapBox.boxIndex + 1) {
@@ -1826,6 +1862,7 @@ export function projectToGrid(
targetX = prevAnchors.forwardAnchorRight[snapMaps.right[0]];
}
forwardAnchors.right[snapMaps.right[0]] = targetX;
+ logger.logForwardAnchor("right", snapMaps.right[0], targetX);
for (const currentRightSnapBox of thisTurnSnap) {
const lineIndex = currentRightSnapBox.lineIndex;
@@ -1838,6 +1875,7 @@ export function projectToGrid(
}
rawLines[lineIndex] += currentRightSnapBox.bbox.str;
currentRightSnapBox.bbox.rendered = true;
+ logger.logRender(currentRightSnapBox.bbox, lineIndex, targetX, "right-snap");
let nextBbox: ProjectionTextBox | null = null;
if (lines[lineIndex].length > currentRightSnapBox.boxIndex + 1) {
@@ -1913,6 +1951,7 @@ export function projectToGrid(
targetX = prevAnchors.forwardAnchorCenter[snapMaps.center[0]];
}
forwardAnchors.center[snapMaps.center[0]] = targetX;
+ logger.logForwardAnchor("center", snapMaps.center[0], targetX);
for (const currentCenterSnapBox of thisTurnSnap) {
if (flowingLines.has(currentCenterSnapBox.lineIndex)) continue;
if (
@@ -1928,6 +1967,12 @@ export function projectToGrid(
}
rawLines[currentCenterSnapBox.lineIndex] += currentCenterSnapBox.bbox.str;
currentCenterSnapBox.bbox.rendered = true;
+ logger.logRender(
+ currentCenterSnapBox.bbox,
+ currentCenterSnapBox.lineIndex,
+ targetX,
+ "center-snap"
+ );
}
snapMaps.center.shift();
}
@@ -1948,7 +1993,11 @@ export function projectToGrid(
};
}
-export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig): ParsedPage[] {
+export async function projectPagesToGrid(
+ pages: PageData[],
+ config: LiteParseConfig
+): Promise<ParsedPage[]> {
+ const logger = createGridDebugLogger(config.debug);
const prevAnchors: PrevAnchors = {
forwardAnchorLeft: {},
forwardAnchorRight: {},
@@ -1958,6 +2007,8 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):
const results: ParsedPage[] = [];
for (const page of pages) {
+ logger.setPage(page.pageNum, page.width, page.height);
+
// Build projection boxes from text items
const projectionBoxes = buildBbox(page, config);
@@ -1967,7 +2018,8 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):
page,
projectionBoxes,
prevAnchors,
- pages.length
+ pages.length,
+ logger
);
// Update forward anchors if preserving across pages
@@ -1997,5 +2049,17 @@ export function projectPagesToGrid(pages: PageData[], config: LiteParseConfig):
// Clean raw text (margin detection, etc)
cleanRawText(results, config);
+ // Flush debug log and render visualizations
+ if (logger.enabled) {
+ await logger.flush();
+ if (logger.shouldVisualize) {
+ const vizPath = logger.debugConfig.visualizePath ?? "./debug-output";
+ const paths = await renderAllVisualizations(logger.visualizerPages, vizPath);
+ for (const p of paths) {
+ process.stderr.write(`[grid-debug] Visualization: ${p}\n`);
+ }
+ }
+ }
+
return results;
}