Skip to content

A fast, helpful, and open-source document parser

22.7 KB
Raw Blame
#!/usr/bin/env node

import { program } from "commander";
import { LiteParse, type LiteParseConfig } from "./lib.js";
import { parseResultToCliJson } from "./cli-json.js";
import { formatText } from "./cli-text.js";
import { readFileSync, writeFileSync, mkdirSync, readdirSync, statSync } from "node:fs";
import { join, relative, parse as parsePath } from "node:path";

/**
 * The package version, read from the package.json that ships next to `dist/`
 * (`dist/cli.js` -> `../package.json`, both in a checkout and when installed),
 * so `--version` can never drift from the published version.
 */
function readPackageVersion(): string {
  try {
    const pkg: unknown = JSON.parse(
      readFileSync(new URL("../package.json", import.meta.url), "utf-8"),
    );
    if (
      typeof pkg === "object" &&
      pkg !== null &&
      "version" in pkg &&
      typeof pkg.version === "string"
    ) {
      return pkg.version;
    }
  } catch {
    // Fall through: an unreadable package.json must not break every command.
  }
  return "unknown";
}

program
  .name("liteparse")
  .description("Fast, lightweight PDF and document parsing")
  .version(readPackageVersion());

/**
 * Resolve a CLI `<file>` argument into a parser input. `-` means read the
 * document from stdin (e.g. `curl -sL … | liteparse parse -`); anything else is
 * passed through as a path. Streaming stdin (rather than `readFileSync(0)`)
 * avoids EAGAIN on non-blocking pipes.
 */
async function resolveInput(file: string): Promise<string | Buffer> {
  if (file !== "-") return file;
  const chunks: Buffer[] = [];
  for await (const chunk of process.stdin) {
    chunks.push(chunk as Buffer);
  }
  const bytes = Buffer.concat(chunks);
  if (bytes.length === 0) {
    throw new Error(
      "no data on stdin (input `-` expects a document piped in, e.g. `curl … | liteparse parse -`)",
    );
  }
  return bytes;
}

/** Collect repeated `--ocr-server-header "Name: Value"` flags into an object. */
function collectHeader(
  value: string,
  previous: Record<string, string> = {},
): Record<string, string> {
  const idx = value.indexOf(":");
  if (idx === -1) {
    throw new Error(`invalid header '${value}', expected 'Name: Value'`);
  }
  const name = value.slice(0, idx).trim();
  if (name === "") {
    throw new Error(`invalid header '${value}', empty header name`);
  }
  previous[name] = value.slice(idx + 1).trim();
  return previous;
}

program
  .command("parse")
  .description("Parse a document and extract text")
  .argument("<file>", "Path to the document file")
  .option("-o, --output <file>", "Output file path")
  .option("--format <format>", 'Output format: json|text|markdown (default: "text")')
  .option(
    "--image-mode <mode>",
    "How to surface raster images in markdown: off|placeholder|embed (default: placeholder)",
  )
  .option(
    "--image-output-dir <dir>",
    "Directory to write embedded images to when --image-mode embed is set",
  )
  .option("--extract-images", "Extract embedded image bytes and metadata")
  .option("--no-links", "Disable hyperlink extraction (emit plain anchor text)")
  .option(
    "--keep-headers-footers",
    "Keep running headers/footers in markdown output instead of stripping them",
  )
  .option("--extract-annotations", "Include all PDF annotations in page output")
  .option("--extract-form-fields", "Include AcroForm widget fields and values")
  .option("--extract-structure-tree", "Include the tagged-PDF logical structure tree")
  .option(
    "--extract-blocks",
    "Include each page's classified layout blocks with bounding boxes",
  )
  .option(
    "--extract-xfa-packets",
    "Include raw XFA packets (name + XML content) in JSON output",
  )
  .option(
    "--extract-content-bounds",
    "Include each page's content_bounds in JSON output",
  )
  .option("--ocr-server-url <url>", "HTTP OCR server URL")
  .option(
    "--ocr-server-header <header>",
    'Extra header for OCR server requests, "Name: Value" (repeatable)',
    collectHeader,
  )
  .option("--no-ocr", "Disable OCR")
  .option("--ocr-language <lang>", "OCR language (default: eng)")
  .option("--max-pages <n>", "Max pages to parse", parseInt)
  .option(
    "--target-pages <pages>",
    'Pages to parse (e.g., "1-5,10,15-20")',
  )
  .option(
    "--continue-on-page-error",
    "Continue after page-level extraction errors and report them in JSON",
  )
  .option("--dpi <dpi>", "Rendering DPI", parseFloat)
  .option("--preserve-small-text", "Keep very small text")
  .option(
    "--extract-text-metadata",
    "Include rich PDF text metadata in text items and JSON output",
  )
  .option("--password <password>", "Password for encrypted documents")
  .option("--config <file>", "JSON config file path")
  .option("-q, --quiet", "Suppress progress output")
  .option("--num-workers <n>", "Number of concurrent OCR workers", parseInt)
  .option(
    "--complexity",
    "Include per-page complexity signals in JSON output",
  )
  .option(
    "--extract-vector-graphics",
    "Include page-scoped vector shapes and merged horizontal/vertical lines",
  )
  .action(async (file: string, opts: Record<string, unknown>) => {
    try {
      const config: Partial<LiteParseConfig> = {};

      // Load config file if provided
      if (opts.config) {
        const fileConfig = JSON.parse(
          readFileSync(opts.config as string, "utf-8"),
        );
        Object.assign(config, fileConfig);
      }

      // CLI options override config file
      if (opts.format) config.outputFormat = opts.format as "json" | "text" | "markdown";
      if (opts.imageMode)
        config.imageMode = opts.imageMode as "off" | "placeholder" | "embed";
      if (opts.imageOutputDir)
        config.imageOutputDir = opts.imageOutputDir as string;
      if (opts.extractImages) config.extractImages = true;
      if (opts.links === false) config.extractLinks = false;
      if (opts.keepHeadersFooters) config.keepHeadersFooters = true;
      if (opts.extractAnnotations) config.extractAnnotations = true;
      if (opts.extractFormFields) config.extractFormFields = true;
      if (opts.extractStructureTree) config.extractStructureTree = true;
      if (opts.extractBlocks) config.extractBlocks = true;
      if (opts.extractXfaPackets) config.extractXfaPackets = true;
      if (opts.extractContentBounds) config.extractContentBounds = true;
      if (opts.ocrServerUrl)
        config.ocrServerUrl = opts.ocrServerUrl as string;
      if (opts.ocrServerHeader)
        config.ocrServerHeaders = opts.ocrServerHeader as Record<string, string>;
      if (opts.ocr === false) config.ocrEnabled = false;
      if (opts.ocrLanguage) config.ocrLanguage = opts.ocrLanguage as string;
      if (opts.maxPages) config.maxPages = opts.maxPages as number;
      if (opts.targetPages) config.targetPages = opts.targetPages as string;
      if (opts.continueOnPageError) config.continueOnPageError = true;
      if (opts.dpi) config.dpi = opts.dpi as number;
      if (opts.preserveSmallText) config.preserveVerySmallText = true;
      if (opts.extractTextMetadata) config.extractTextMetadata = true;
      if (opts.password) config.password = opts.password as string;
      if (opts.quiet) config.quiet = true;
      if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;
      if (opts.complexity) config.includeComplexity = true;
      if (opts.extractVectorGraphics) config.extractVectorGraphics = true;

      // Default CLI output to text (library defaults to json)
      if (!config.outputFormat) config.outputFormat = "text";

      const parser = new LiteParse(config);
      const result = await parser.parse(await resolveInput(file));

      // JSON output carries pageErrors itself; text/markdown would silently
      // omit the failed pages, so always surface them on stderr.
      for (const error of result.pageErrors) {
        console.error(
          `[liteparse] page ${error.pageNum} failed to extract and was skipped: ${error.message}`,
        );
      }

      const output =
        config.outputFormat === "json"
          ? JSON.stringify(
              parseResultToCliJson(result, {
                extractTextMetadata: config.extractTextMetadata,
              }),
              null,
              2,
            )
          : config.outputFormat === "text"
            ? formatText(result.pages)
            : result.text;


      if (opts.output) {
        writeFileSync(opts.output as string, output, "utf-8");
      } else {
        process.stdout.write(output);
      }
    } catch (err) {
      console.error(
        `Error: ${err instanceof Error ? err.message : String(err)}`,
      );
      process.exit(1);
    }
  });

program
  .command("is-complex")
  .description(
    "Check if a document is 'complex' enough to require OCR or other advanced parsing",
  )
  .argument("<file>", "Path to the document file")
  .option("--compact", "Emit dense, whitespace-free JSON instead of pretty")
  .option("--max-pages <n>", "Max pages to parse", parseInt)
  .option(
    "--target-pages <pages>",
    'Pages to check (e.g., "1-5,10,15-20")',
  )
  .option("--password <password>", "Password for encrypted documents")
  .option("-q, --quiet", "Suppress progress output")
  .action(async (file: string, opts: Record<string, unknown>) => {
    try {
      const config: Partial<LiteParseConfig> = {};
      if (opts.maxPages) config.maxPages = opts.maxPages as number;
      if (opts.targetPages) config.targetPages = opts.targetPages as string;
      if (opts.password) config.password = opts.password as string;
      if (opts.quiet) config.quiet = true;

      const parser = new LiteParse(config);
      const stats = await parser.isComplex(await resolveInput(file));

      const complexPages = stats.filter((s) => s.needsOcr).length;

      // Always emit JSON on stdout so the command composes with `jq` without a
      // flag. Pretty by default; `--compact` drops the whitespace. Both parse
      // identically for any reader.
      process.stdout.write(
        opts.compact
          ? JSON.stringify(stats)
          : JSON.stringify(stats, null, 2),
      );
      process.stdout.write("\n");

      // Human verdict goes to stderr so it never pollutes the JSON on stdout;
      // the exit code below carries the same signal for scripts.
      if (!opts.quiet) {
        const verdict = complexPages > 0 ? "COMPLEX" : "SIMPLE";
        const layoutCount = (reason: string) =>
          stats.filter((s) => s.layout?.reasons.includes(reason)).length;
        console.error(
          `${verdict} — ${complexPages}/${stats.length} page(s) need OCR; ` +
            `layout: ${layoutCount("multi-column")} multi-column, ` +
            `${layoutCount("table-likely")} table, ` +
            `${layoutCount("dense-graphics")} graphics-dense`,
        );
      }

      // Exit non-zero when any page needs OCR, so the command is usable as a
      // shell predicate (e.g. `is-complex doc.pdf && parse --no-ocr`).
      if (complexPages > 0) process.exit(1);
    } catch (err) {
      console.error(
        `Error: ${err instanceof Error ? err.message : String(err)}`,
      );
      process.exit(1);
    }
  });

program
  .command("screenshot")
  .description("Generate screenshots of document pages")
  .argument("<file>", "Path to the document file")
  .option(
    "-o, --output-dir <dir>",
    "Output directory for screenshots",
    "./screenshots",
  )
  .option(
    "--target-pages <pages>",
    'Pages to screenshot (e.g., "1,3,5" or "1-5")',
  )
  .option("--dpi <dpi>", "Rendering DPI", parseFloat)
  .option("--password <password>", "Password for encrypted documents")
  .option("-q, --quiet", "Suppress progress output")
  .action(async (file: string, opts: Record<string, unknown>) => {
    try {
      const config: Partial<LiteParseConfig> = {};
      if (opts.dpi) config.dpi = opts.dpi as number;
      if (opts.password) config.password = opts.password as string;
      if (opts.quiet) config.quiet = true;
      if (opts.targetPages) config.targetPages = opts.targetPages as string;

      const parser = new LiteParse(config);

      // Parse target pages into number array
      let pageNumbers: number[] | undefined;
      if (opts.targetPages) {
        pageNumbers = [];
        for (const part of (opts.targetPages as string).split(",")) {
          const trimmed = part.trim();
          if (trimmed.includes("-")) {
            const [start, end] = trimmed.split("-").map(Number);
            for (let i = start; i <= end; i++) pageNumbers.push(i);
          } else {
            pageNumbers.push(Number(trimmed));
          }
        }
      }

      const outputDir = opts.outputDir as string;
      mkdirSync(outputDir, { recursive: true });

      const results = await parser.screenshot(file, pageNumbers);

      for (const result of results) {
        const outputPath = join(outputDir, `page_${result.pageNum}.png`);
        writeFileSync(outputPath, result.imageBuffer);
        if (!opts.quiet) {
          console.error(
            `[liteparse] screenshot page ${result.pageNum} → ${outputPath}`,
          );
        }
      }
    } catch (err) {
      console.error(
        `Error: ${err instanceof Error ? err.message : String(err)}`,
      );
      process.exit(1);
    }
  });

program
  .command("batch-parse")
  .description("Parse multiple documents in batch mode")
  .argument("<input-dir>", "Input directory")
  .argument("<output-dir>", "Output directory")
  .option("--format <format>", 'Output format: json|text|markdown (default: "text")')
  .option("--no-ocr", "Disable OCR")
  .option("--ocr-language <lang>", "OCR language (default: eng)")
  .option("--ocr-server-url <url>", "HTTP OCR server URL")
  .option(
    "--ocr-server-header <header>",
    'Extra header for OCR server requests, "Name: Value" (repeatable)',
    collectHeader,
  )
  .option("--max-pages <n>", "Max pages to parse per file", parseInt)
  .option("--dpi <dpi>", "Rendering DPI", parseFloat)
  .option("--recursive", "Recursively search input directory")
  .option("--extension <ext>", "Only process files with this extension")
  .option("--password <password>", "Password for encrypted documents")
  .option("-q, --quiet", "Suppress progress output")
  .option("--num-workers <n>", "Number of concurrent OCR workers", parseInt)
  .option(
    "--extract-text-metadata",
    "Include rich PDF text metadata in text items and JSON output",
  )
  .option("--extract-images", "Extract embedded image bytes and metadata")
  .option("--extract-annotations", "Include all PDF annotations in page output")
  .option("--extract-form-fields", "Include AcroForm widget fields and values")
  .option("--extract-structure-tree", "Include the tagged-PDF logical structure tree")
  .option(
    "--extract-blocks",
    "Include each page's classified layout blocks with bounding boxes",
  )
  .option(
    "--extract-xfa-packets",
    "Include raw XFA packets (name + XML content) in JSON output",
  )
  .option(
    "--extract-content-bounds",
    "Include each page's content_bounds in JSON output",
  )
  .option(
    "--extract-vector-graphics",
    "Include page-scoped vector shapes and merged horizontal/vertical lines",
  )
  .action(
    async (
      inputDir: string,
      outputDir: string,
      opts: Record<string, unknown>,
    ) => {
      try {
        const config: Partial<LiteParseConfig> = {};
        const format = (opts.format as string) ?? "text";
        config.outputFormat = format as "json" | "text" | "markdown";
        if (opts.ocr === false) config.ocrEnabled = false;
        if (opts.ocrLanguage) config.ocrLanguage = opts.ocrLanguage as string;
        if (opts.ocrServerUrl)
          config.ocrServerUrl = opts.ocrServerUrl as string;
        if (opts.ocrServerHeader)
          config.ocrServerHeaders = opts.ocrServerHeader as Record<string, string>;
        if (opts.maxPages) config.maxPages = opts.maxPages as number;
        if (opts.dpi) config.dpi = opts.dpi as number;
        if (opts.password) config.password = opts.password as string;
        if (opts.quiet) config.quiet = true;
        if (opts.numWorkers) config.numWorkers = opts.numWorkers as number;
        if (opts.extractTextMetadata) config.extractTextMetadata = true;
        if (opts.extractImages) config.extractImages = true;
        if (opts.extractAnnotations) config.extractAnnotations = true;
        if (opts.extractFormFields) config.extractFormFields = true;
        if (opts.extractStructureTree) config.extractStructureTree = true;
        if (opts.extractBlocks) config.extractBlocks = true;
        if (opts.extractXfaPackets) config.extractXfaPackets = true;
        if (opts.extractContentBounds) config.extractContentBounds = true;
      if (opts.extractContentBounds) config.extractContentBounds = true;
      if (opts.extractXfaPackets) config.extractXfaPackets = true;
      if (opts.extractContentBounds) config.extractContentBounds = true;
        if (opts.extractVectorGraphics) config.extractVectorGraphics = true;

        const parser = new LiteParse(config);
        const outExt = format === "json" ? ".json" : format === "markdown" ? ".md" : ".txt";

        mkdirSync(outputDir, { recursive: true });

        const extFilter = opts.extension
          ? (opts.extension as string).startsWith(".")
            ? (opts.extension as string).toLowerCase()
            : `.${(opts.extension as string).toLowerCase()}`
          : undefined;

        const files = collectFiles(
          inputDir,
          !!opts.recursive,
          extFilter,
        );

        if (files.length === 0) {
          console.error(
            `[liteparse] no matching files found in ${inputDir}`,
          );
          return;
        }
        if (!opts.quiet) {
          console.error(
            `[liteparse] found ${files.length} files to process`,
          );
        }

        // Outputs are named `<stem><ext>`, so `report.pdf` and `report.png`
        // would both land on `report.txt` and the later one would silently
        // overwrite the earlier one. Refuse up front, before parsing or
        // writing anything.
        assertNoOutputCollisions(files, inputDir, outputDir, outExt);

        let success = 0;
        let errors = 0;

        for (const filePath of files) {
          const t0 = Date.now();
          const rel = relative(inputDir, filePath);
          const parsed = parsePath(rel);
          const outPath = batchOutputPath(filePath, inputDir, outputDir, outExt);
          mkdirSync(join(outputDir, parsed.dir), { recursive: true });

          try {
            const result = await parser.parse(filePath);
            const output =
              format === "json"
                ? JSON.stringify(
                    parseResultToCliJson(result, {
                      extractTextMetadata: config.extractTextMetadata,
                    }),
                    null,
                    2,
                  )
                : format === "text"
                  ? formatText(result.pages)
                  : result.text;
            writeFileSync(outPath, output, "utf-8");
            success++;
            if (!opts.quiet) {
              const elapsed = Date.now() - t0;
              console.error(
                `[liteparse] ${filePath} → ${outPath} (${elapsed}ms)`,
              );
            }
          } catch (err) {
            console.error(
              `[liteparse] error parsing ${filePath}: ${err instanceof Error ? err.message : String(err)}`,
            );
            errors++;
          }
        }

        console.error(
          `[liteparse] batch complete: ${success} succeeded, ${errors} failed`,
        );
        if (errors > 0) process.exit(1);
      } catch (err) {
        console.error(
          `Error: ${err instanceof Error ? err.message : String(err)}`,
        );
        process.exit(1);
      }
    },
  );

// Keep in sync with the extension lists in crates/liteparse/src/conversion.rs.
// Plain-text formats (.txt, .md, .markdown, .log) are deliberately absent: the
// parser rejects them ("unsupported file format"), so scanning them would only
// turn every such file in a directory into a reported failure.
const SUPPORTED_EXTENSIONS = new Set([
  ".pdf",
  ".doc", ".docx", ".docm", ".dot", ".dotm", ".dotx", ".odt", ".ott", ".rtf", ".pages",
  ".ppt", ".pptx", ".pptm", ".pot", ".potm", ".potx", ".odp", ".otp", ".key",
  ".xls", ".xlsx", ".xlsm", ".xlsb", ".ods", ".ots", ".csv", ".tsv", ".numbers",
  ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg",
]);

/** Where `batch-parse` writes the output for `filePath`. */
function batchOutputPath(
  filePath: string,
  inputDir: string,
  outputDir: string,
  outExt: string,
): string {
  const parsed = parsePath(relative(inputDir, filePath));
  return join(outputDir, parsed.dir, parsed.name + outExt);
}

/**
 * Throw if two inputs would be written to the same output file. Paths are
 * compared case-insensitively on Windows and macOS, whose default file systems
 * would make `Report.txt` and `report.txt` the same file.
 */
function assertNoOutputCollisions(
  files: string[],
  inputDir: string,
  outputDir: string,
  outExt: string,
): void {
  const caseInsensitive = process.platform === "win32" || process.platform === "darwin";
  const byOutput = new Map<string, { outPath: string; inputs: string[] }>();
  for (const filePath of files) {
    const outPath = batchOutputPath(filePath, inputDir, outputDir, outExt);
    const key = caseInsensitive ? outPath.toLowerCase() : outPath;
    const entry = byOutput.get(key);
    if (entry) entry.inputs.push(filePath);
    else byOutput.set(key, { outPath, inputs: [filePath] });
  }

  const collisions = [...byOutput.values()].filter((entry) => entry.inputs.length > 1);
  if (collisions.length === 0) return;

  const shown = collisions
    .slice(0, 10)
    .map(({ outPath, inputs }) => `  ${outPath} <- ${inputs.join(", ")}`);
  if (collisions.length > shown.length) {
    shown.push(`  ... and ${collisions.length - shown.length} more`);
  }
  throw new Error(
    `output path collision: multiple input files would be written to the same output file:\n` +
      `${shown.join("\n")}\n` +
      `Rename the inputs, or process one file type at a time with --extension (e.g. --extension pdf).`,
  );
}

function collectFiles(
  dir: string,
  recursive: boolean,
  extFilter?: string,
): string[] {
  const files: string[] = [];
  collectFilesInner(dir, recursive, extFilter, files);
  files.sort();
  return files;
}

function collectFilesInner(
  dir: string,
  recursive: boolean,
  extFilter: string | undefined,
  files: string[],
): void {
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    const fullPath = join(dir, entry.name);
    if (entry.isDirectory()) {
      if (recursive) collectFilesInner(fullPath, recursive, extFilter, files);
      continue;
    }
    const lower = entry.name.toLowerCase();
    if (extFilter) {
      if (!lower.endsWith(extFilter)) continue;
    } else {
      const ext = lower.lastIndexOf(".") >= 0 ? lower.slice(lower.lastIndexOf(".")) : "";
      if (!SUPPORTED_EXTENSIONS.has(ext)) continue;
    }
    files.push(fullPath);
  }
}

program.parse(process.argv);