A fast, helpful, and open-source document parser
Support URL parsing by reading from stdin
Logan Markewich5855ca9d74ba
diff --git a/cli/parse.ts b/cli/parse.tsindex 78883be..b5ee4c6 100644--- a/cli/parse.ts+++ b/cli/parse.ts@@ -87,9 +87,10 @@ program.action(async (file: string, options: ParseCommandOptions) => {try {const quiet = options.quiet || false;+ const isStdin = file === "-";- // Check if file exists- if (!existsSync(file)) {+ // Check if file exists (skip for stdin)+ if (!isStdin && !existsSync(file)) {console.error(`Error: File not found: ${file}`);process.exit(1);}@@ -130,8 +131,24 @@ program// Create parserconst parser = new LiteParse(config);- // Parse PDF (quiet flag controls progress output)- const result = await parser.parse(file, quiet);+ // Read from stdin or file+ let input: string | Buffer;+ if (isStdin) {+ const chunks: Buffer[] = [];+ for await (const chunk of process.stdin) {+ chunks.push(chunk);+ }+ input = Buffer.concat(chunks);+ if (input.length === 0) {+ console.error("Error: No data received from stdin");+ process.exit(1);+ }+ } else {+ input = file;+ }++ // Parse document (quiet flag controls progress output)+ const result = await parser.parse(input, quiet);// Format output based on formatlet output: string;diff --git a/docs/src/content/docs/liteparse/cli-reference.md b/docs/src/content/docs/liteparse/cli-reference.mdindex c0fd208..298aff7 100644--- a/docs/src/content/docs/liteparse/cli-reference.md+++ b/docs/src/content/docs/liteparse/cli-reference.md@@ -19,7 +19,7 @@ lit parse [options] <file>| Argument | Description ||----------|-------------|-| `file` | Path to the document file |+| `file` | Path to the document file, or `-` to read from stdin |### Options@@ -60,6 +60,9 @@ lit parse report.pdf --ocr-server-url http://localhost:8828/ocr# Pipe output to another toollit parse report.pdf -q | wc -l++# Parse a remote file via stdin+curl -sL https://example.com/report.pdf | lit parse --no-ocr -```---diff --git a/docs/src/content/docs/liteparse/guides/parsing-urls.md b/docs/src/content/docs/liteparse/guides/parsing-urls.mdnew file mode 100644index 0000000..9852fb5--- /dev/null+++ b/docs/src/content/docs/liteparse/guides/parsing-urls.md@@ -0,0 +1,38 @@+---+title: Parsing URLs+description: Parse remote documents by reading URLs.+sidebar:+ order: 5+---++To parse remote files, LiteParse support both CLI and library usage for reading bytes and streams. This means the CLI can download them with any tool you like and pipe the bytes to `lit parse` using `-` as the file argument, while the libraries can fetch the bytes directly from the URL and pass them to the parser.++## CLI usage++```bash+# Parse a remote PDF+curl -sL https://example.com/report.pdf | lit parse -++# With options+curl -sL https://example.com/report.pdf | lit parse --no-ocr --format json -++# Save to a file+curl -sL https://example.com/report.pdf | lit parse -o report.txt -+```++The `-` argument tells LiteParse to read from stdin instead of a file path. Any tool that writes to stdout works — `curl`, `wget`, `aws s3 cp - -`, etc.++## Library usage++The TypeScript library accepts `Buffer`/`Uint8Array` directly, so you can handle the download however you like:++```typescript+import { LiteParse } from "@llamaindex/liteparse";++const response = await fetch("https://example.com/report.pdf");+const buffer = Buffer.from(await response.arrayBuffer());++const parser = new LiteParse({ ocrEnabled: false });+const result = await parser.parse(buffer);+console.log(result.text);+```