A fast, helpful, and open-source document parser
improve api generation
Logan Markewichb09fbb82dc65
diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rsindex 82287ce..e01337d 100644--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@ -63,7 +63,7 @@ fn default_num_workers() -> usize {.unwrap_or(1)}-/// Parse a target pages string like "1-5,10,15-20" into a sorted list of page numbers.+#[doc(hidden)]pub fn parse_target_pages(s: &str) -> Result<Vec<u32>, String> {let mut pages = Vec::new();for part in s.split(',') {diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rsindex 9888934..d153655 100644--- a/crates/liteparse/src/lib.rs+++ b/crates/liteparse/src/lib.rs@@ -1,14 +1,37 @@+//! LiteParse — open-source PDF parsing with spatial text extraction, OCR, and bounding boxes.+//!+//! This crate is the core Rust library. Language bindings for Node.js, Python,+//! and WebAssembly re-export the same types with language-idiomatic wrappers.+//!++// ── Public API re-exports ──────────────────────────────────────────────+pub use config::{LiteParseConfig, OutputFormat};+pub use error::LiteParseError;+pub use parser::{LiteParse, ParseResult};+pub use search::{search_items, SearchOptions};+pub use types::{ParsedPage, TextItem};++// ── Modules with user-facing types (visible in docs) ───────────────────pub mod config;+pub mod error;+pub mod parser;+pub mod search;+pub mod types;++// ── Internal modules (available for binding crates, hidden from docs) ──#[cfg(not(target_arch = "wasm32"))]+#[doc(hidden)]pub mod conversion;-pub mod error;+#[doc(hidden)]pub mod extract;+#[doc(hidden)]pub mod ocr;+#[doc(hidden)]pub mod ocr_merge;+#[doc(hidden)]pub mod output;-pub mod parser;+#[doc(hidden)]pub mod projection;#[cfg(not(target_arch = "wasm32"))]+#[doc(hidden)]pub mod render;-pub mod search;-pub mod types;diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rsindex ebc358e..97bdd63 100644--- a/crates/liteparse/src/types.rs+++ b/crates/liteparse/src/types.rs@@ -1,19 +1,14 @@use serde::{Deserialize, Serialize};use std::collections::HashMap;-/// Accepted input types for input documents.-/// - `FilePath(String)` — A file path to a local PDF document.-/// - `Buffer(Vec<u8>)` — A byte buffer containing the PDF data.+#[doc(hidden)]#[derive(Debug, Serialize, Deserialize)]pub enum InputType {FilePath(String),Buffer(Vec<u8>),}-/// Input source for PDF parsing — either a file path or raw bytes.-///-/// When using `Bytes`, the data is kept alive alongside the pdfium `Document`-/// (pdfium requires the buffer to outlive the document handle).+#[doc(hidden)]#[derive(Debug, Clone)]pub enum PdfInput {/// Path to a PDF file on disk.@@ -67,7 +62,7 @@ pub struct TextItem {pub confidence: Option<f32>,}-/// Represents a single page in a PDF document, including its dimensions and extracted text items.+#[doc(hidden)]#[derive(Debug, Serialize)]pub struct Page {pub page_number: usize,@@ -86,6 +81,7 @@ pub struct ParsedPage {pub text_items: Vec<TextItem>,}+#[doc(hidden)]#[derive(Debug, Serialize)]pub enum Snap {Left,@@ -93,6 +89,7 @@ pub enum Snap {Center,}+#[doc(hidden)]#[derive(Debug, Serialize)]pub enum Anchor {Left,@@ -100,7 +97,7 @@ pub enum Anchor {Center,}-/// Represents a Projected piece of text, responsible for keeping track of projection related data+#[doc(hidden)]#[derive(Debug, Serialize)]pub struct ProjectedTextItem {pub item: TextItem,@@ -115,6 +112,7 @@ pub struct ProjectedTextItem {pub d: f32,}+#[doc(hidden)]pub type AnchorMap = HashMap<i32, Vec<(usize, usize)>>;#[cfg(test)]diff --git a/scripts/generate-api-docs.sh b/scripts/generate-api-docs.shindex 79e67cb..7fc443d 100755--- a/scripts/generate-api-docs.sh+++ b/scripts/generate-api-docs.sh@@ -1,13 +1,13 @@#!/usr/bin/env bash-# Generates API reference docs from TypeDoc and outputs a single-# Starlight-compatible markdown file with frontmatter.+# Generates API reference docs from rustdoc JSON via rustdoc-md and outputs+# a single Starlight-compatible markdown file with frontmatter.set -euo pipefailDOCS_DIR="./docs/src/content/docs/liteparse"TMP_DIR="${DOCS_DIR}/.api-tmp"OUT_FILE="${DOCS_DIR}/api.md"-# Generate markdown with TypeDoc+# Generate rustdoc JSON and convert to markdownmkdir -p $TMP_DIRcd crates/liteparse/cargo +nightly rustdoc --lib -- -Z unstable-options --output-format json@@ -19,7 +19,7 @@ rustdoc-md --path ./target/doc/liteparse.json --output ${TMP_DIR}/README.mdcat > "${OUT_FILE}" <<'FRONTMATTER'---title: API Reference-description: API reference for the @llamaindex/liteparse TypeScript library.+description: API reference for the liteparse Rust crate. Types are shared across Node.js, Python, and WASM bindings.sidebar:order: 6---diff --git a/scripts/strip-impls-from-api-docs.py b/scripts/strip-impls-from-api-docs.pyindex 4be834e..96a9bdf 100755--- a/scripts/strip-impls-from-api-docs.py+++ b/scripts/strip-impls-from-api-docs.py@@ -10,17 +10,59 @@ TMP_DIR = f"{DOCS_DIR}/.api-tmp"path = f"{TMP_DIR}/README.md"text = open(path).read()++# Strip implementation and trait implementation sectionstext = re.sub(r"#{2,} Implementations\n.*?(?=\n#{2,} )", "", text, flags=re.DOTALL)text = re.sub(r"#{2,} Trait Implementations\n.*?(?=\n#{2,} )", "", text, flags=re.DOTALL)-text = text.replace(- """# Crate Documentation-**Version:** 2.0.0+# Strip the crate header/version boilerplate+text = re.sub(r"# Crate Documentation\n\n\*\*Version:.*?\*\*Format Version:.*?\n", "", text, flags=re.DOTALL)++# Strip the redundant "Re-exports" section at the end (types are already documented inline)+text = re.sub(r"\n## Re-exports\n.*", "", text, flags=re.DOTALL)++# Strip the top-level "## Modules" line (it's just a bare heading before the actual modules)+text = text.replace("\n## Modules\n", "\n")++# Clean up the "# Module `liteparse`" header to just be a plain intro+text = text.replace("# Module `liteparse`\n", "")++# Remove module grouping headings and their code blocks entirely+text = re.sub(+ r'## Module `\w+`\n\n```rust\npub mod \w+ \{ /\* \.\.\. \*/ \}\n```\n*',+ "",+ text,+)++# Remove bare "### Types" and "### Functions" category headings+text = re.sub(r"### Types\n+", "", text)+text = re.sub(r"### Functions\n+", "", text)++# Promote struct/enum/function headings (####) to ## so they're top-level nav items+# But first, flatten ###### (variant names) to #### so they stay OUT of nav+text = re.sub(r"^######", "####", text, flags=re.MULTILINE)-**Format Version:** 57-""",+# Promote #### (Struct/Enum/Function) -> ##+# Promote ##### (Fields/Variants/Methods) -> ###+text = re.sub(r"^#####", "###", text, flags=re.MULTILINE)+text = re.sub(r"^#### (Struct |Enum |Function )", r"## \1", text, flags=re.MULTILINE)++# Remove the "private fields" placeholder table (it's noise for opaque structs)+text = re.sub(+ r"### Fields\n\n\| Name \| Type \| Documentation \|\n\|[-| ]+\|\n\| \*private fields\* \| \.\.\. \| \*Some fields have been omitted\* \|\n+","",+ text,)++# Collapse verbose enum variant field tables into simple inline type annotations+# Matches pattern: #### `VariantName`\n\nFields:\n\n| Index | Type | ... |\n|---...|\n| 0 | `Type` | |+text = re.sub(+ r"^(#### `\w+`)\n\nFields:\n\n\| Index \| Type \| Documentation \|\n\|[-| ]+\|\n\| 0 \| `([^`]+)` \| \|",+ r"\1(`\2`)",+ text,+ flags=re.MULTILINE,+)+open(path, "w").write(text)