Skip to content

A fast, helpful, and open-source document parser

migrate workflows

Logan Markewichbf5f2feb06b9
diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.yml
index 3cade7b..0929927 100644
--- a/.github/workflows/e2e-output.yml
+++ b/.github/workflows/e2e-output.yml
@@ -39,18 +39,6 @@ jobs:
- name: Build
run: cargo build --release
- # The comparison scripts (compare-dataset.ts, upload-dataset.ts) still use
- # Node/tsx and need to be ported to Rust or rewritten as shell scripts.
- # For now, we install Node to run them.
- - name: Setup Node.js
- uses: actions/setup-node@v4
- with:
- node-version: 20
- cache: "npm"
-
- - name: Install Node dependencies
- run: npm ci
-
- name: Download from HuggingFace
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
@@ -135,21 +123,13 @@ jobs:
- name: Build
run: cargo build --release
- - name: Setup Node.js
- uses: actions/setup-node@v4
- with:
- node-version: 20
- cache: "npm"
-
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install dependencies
- run: |
- npm ci
- pip install huggingface_hub
+ run: pip install huggingface_hub
- name: Download existing dataset from HuggingFace
env:
@@ -165,4 +145,4 @@ jobs:
echo "Warning: HF_TOKEN not set, skipping upload"
exit 0
fi
- npx tsx scripts/upload-dataset.ts expected-dataset ${{ vars.HF_DATASET_REPO || 'llamaindex/liteparse_cicd_data' }}
+ ./scripts/upload-dataset.sh expected-dataset ${{ vars.HF_DATASET_REPO || 'llamaindex/liteparse_cicd_data' }}
diff --git a/scripts/compare-dataset.sh b/scripts/compare-dataset.sh
new file mode 100755
index 0000000..dcbf46c
--- /dev/null
+++ b/scripts/compare-dataset.sh
@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+# Compares current liteparse output against a baseline dataset
+#
+# Usage:
+# ./scripts/compare-dataset.sh [dataset-dir]
+#
+# Exit codes:
+# 0 - No changes detected
+# 1 - Changes detected (requires approval)
+# 2 - Error occurred
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+DATASET_DIR="${1:-$REPO_ROOT/dataset}"
+DOCUMENTS_DIR="$DATASET_DIR/data"
+METADATA_FILE="$DATASET_DIR/metadata.jsonl"
+
+LIT="$REPO_ROOT/target/release/lit"
+if [ ! -x "$LIT" ]; then
+ echo "ERROR: lit binary not found at $LIT. Run 'cargo build --release' first."
+ exit 2
+fi
+
+echo "LiteParse Dataset Comparison"
+echo "============================"
+echo "Dataset: $DATASET_DIR"
+echo "Documents: $DOCUMENTS_DIR"
+echo
+
+if [ ! -f "$METADATA_FILE" ]; then
+ echo "ERROR: metadata.jsonl not found at $METADATA_FILE"
+ exit 2
+fi
+
+ENTRY_COUNT=$(wc -l < "$METADATA_FILE" | tr -d ' ')
+echo "Loaded $ENTRY_COUNT expected entries"
+echo
+
+if [ "$ENTRY_COUNT" -eq 0 ]; then
+ echo "ERROR: metadata.jsonl is empty — dataset has no entries to compare against."
+ echo "The dataset may need to be regenerated with: ./scripts/create-dataset.sh"
+ exit 2
+fi
+
+DIFF_COUNT=0
+ADDED_COUNT=0
+REMOVED_COUNT=0
+CHANGED_COUNT=0
+DIFF_OUTPUT=""
+
+# Collect unique documents from metadata
+DOCUMENTS=$(jq -r '.document' "$METADATA_FILE" | sort -u)
+
+while IFS= read -r document; do
+ [ -z "$document" ] && continue
+ FILE_PATH="$DOCUMENTS_DIR/$document"
+
+ # Check if file exists
+ if [ ! -f "$FILE_PATH" ]; then
+ # All pages for this document are removed
+ while IFS= read -r line; do
+ PAGE=$(echo "$line" | jq -r '.page')
+ DIFF_OUTPUT+="[REMOVED] $document (page $PAGE)"$'\n\n'
+ REMOVED_COUNT=$((REMOVED_COUNT + 1))
+ DIFF_COUNT=$((DIFF_COUNT + 1))
+ done < <(jq -c "select(.document == \"$document\")" "$METADATA_FILE")
+ continue
+ fi
+
+ echo "Checking: $document"
+
+ # Parse current output
+ CURRENT_JSON=""
+ PARSE_ERROR=""
+ if CURRENT_JSON=$("$LIT" parse --format json --no-ocr -q "$FILE_PATH" 2>&1); then
+ # Compare each expected page
+ while IFS= read -r line; do
+ EXPECTED_PAGE=$(echo "$line" | jq -r '.page')
+ EXPECTED_TEXT=$(echo "$line" | jq -r '.output_text')
+ IS_PDF=false
+ if [[ "$document" == *.pdf ]]; then
+ IS_PDF=true
+ fi
+
+ # Get actual text for this page
+ ACTUAL_TEXT=$(echo "$CURRENT_JSON" | jq -r --argjson page "$EXPECTED_PAGE" \
+ '.pages[] | select(.page == $page) | .text // ""' 2>/dev/null)
+
+ if [ -z "$ACTUAL_TEXT" ] && [ "$EXPECTED_PAGE" -ne 0 ]; then
+ # Page not found in current output
+ DIFF_OUTPUT+="[REMOVED] $document (page $EXPECTED_PAGE)"$'\n\n'
+ REMOVED_COUNT=$((REMOVED_COUNT + 1))
+ DIFF_COUNT=$((DIFF_COUNT + 1))
+ continue
+ fi
+
+ # Normalize for comparison
+ if [ "$IS_PDF" = true ]; then
+ EXPECTED_CMP=$(echo "$EXPECTED_TEXT" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+ ACTUAL_CMP=$(echo "$ACTUAL_TEXT" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+ else
+ # For non-PDF: collapse whitespace runs, trim lines, remove empty lines
+ EXPECTED_CMP=$(echo "$EXPECTED_TEXT" | sed 's/[[:space:]]\{1,\}/ /g' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed '/^$/d')
+ ACTUAL_CMP=$(echo "$ACTUAL_TEXT" | sed 's/[[:space:]]\{1,\}/ /g' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed '/^$/d')
+ fi
+
+ if [ "$EXPECTED_CMP" != "$ACTUAL_CMP" ]; then
+ DIFF_OUTPUT+="[CHANGED] $document (page $EXPECTED_PAGE)"$'\n'
+ # Show a unified-style diff
+ DIFF_DETAIL=$(diff <(echo "$EXPECTED_TEXT") <(echo "$ACTUAL_TEXT") --unified=1 2>/dev/null || true)
+ if [ -n "$DIFF_DETAIL" ]; then
+ # Show first 50 lines of diff to avoid massive output
+ DIFF_OUTPUT+="$(echo "$DIFF_DETAIL" | head -50)"$'\n'
+ REMAINING=$(echo "$DIFF_DETAIL" | wc -l | tr -d ' ')
+ if [ "$REMAINING" -gt 50 ]; then
+ DIFF_OUTPUT+="... ($((REMAINING - 50)) more lines)"$'\n'
+ fi
+ fi
+ DIFF_OUTPUT+=$'\n'
+ CHANGED_COUNT=$((CHANGED_COUNT + 1))
+ DIFF_COUNT=$((DIFF_COUNT + 1))
+ fi
+ done < <(jq -c "select(.document == \"$document\")" "$METADATA_FILE")
+
+ # Check for new pages not in expected dataset
+ EXPECTED_PAGES=$(jq -r "select(.document == \"$document\") | .page" "$METADATA_FILE" | sort -n)
+ ACTUAL_PAGES=$(echo "$CURRENT_JSON" | jq -r '.pages[].page' | sort -n)
+
+ while IFS= read -r page; do
+ [ -z "$page" ] && continue
+ if ! echo "$EXPECTED_PAGES" | grep -q "^${page}$"; then
+ DIFF_OUTPUT+="[ADDED] $document (page $page)"$'\n\n'
+ ADDED_COUNT=$((ADDED_COUNT + 1))
+ DIFF_COUNT=$((DIFF_COUNT + 1))
+ fi
+ done <<< "$ACTUAL_PAGES"
+
+ else
+ PARSE_ERROR="$CURRENT_JSON"
+ echo " ERROR: $PARSE_ERROR"
+
+ # Check if error was expected
+ HAS_ERROR_ENTRY=$(jq -r "select(.document == \"$document\") | .output_json.error // false" "$METADATA_FILE" | head -1)
+ if [ "$HAS_ERROR_ENTRY" != "true" ]; then
+ DIFF_OUTPUT+="[CHANGED] $document (page 0)"$'\n'
+ DIFF_OUTPUT+=" Expected: successful parse"$'\n'
+ DIFF_OUTPUT+=" Actual: error: $PARSE_ERROR"$'\n\n'
+ CHANGED_COUNT=$((CHANGED_COUNT + 1))
+ DIFF_COUNT=$((DIFF_COUNT + 1))
+ fi
+ fi
+done <<< "$DOCUMENTS"
+
+echo
+echo "Results"
+echo "-------"
+
+if [ "$DIFF_COUNT" -eq 0 ]; then
+ echo "✓ No changes detected"
+ exit 0
+fi
+
+echo "✗ $DIFF_COUNT change(s) detected:"
+echo
+echo "$DIFF_OUTPUT"
+
+echo "---"
+echo "SUMMARY:"
+echo " Added: $ADDED_COUNT"
+echo " Removed: $REMOVED_COUNT"
+echo " Changed: $CHANGED_COUNT"
+echo
+echo "This PR changes liteparse output and requires manual approval."
+
+exit 1
diff --git a/scripts/compare-dataset.ts b/scripts/compare-dataset.ts
deleted file mode 100644
index 47b3974..0000000
--- a/scripts/compare-dataset.ts
+++ /dev/null
@@ -1,377 +0,0 @@
-/**
- * Compares current liteparse output against a baseline dataset
- *
- * Usage:
- * npx tsx scripts/compare-dataset.ts [dataset-dir]
- *
- * The dataset should be created using create-dataset.ts and contains:
- * - data/: The original document files
- * - metadata.jsonl: Expected outputs for each document/page
- *
- * Exit codes:
- * 0 - No changes detected
- * 1 - Changes detected (requires approval)
- * 2 - Error occurred
- *
- * For CI/CD, this script can be run to detect if a PR changes output.
- * If changes are detected, the PR should require manual approval.
- */
-
-import { LiteParse } from "../src/lib.js";
-import * as fs from "fs/promises";
-import * as path from "path";
-
-const DEFAULT_DATASET_DIR = path.join(import.meta.dirname, "..", "dataset");
-
-interface DatasetRow {
- file_name: string;
- document: string;
- page: number;
- output_text: string;
- output_json: object;
-}
-
-interface DiffResult {
- document: string;
- page: number;
- type: "added" | "removed" | "changed";
- expected?: string;
- actual?: string;
- diff?: string;
-}
-
-async function loadDataset(datasetDir: string): Promise<Map<string, DatasetRow>> {
- const metadataPath = path.join(datasetDir, "metadata.jsonl");
- const content = await fs.readFile(metadataPath, "utf-8");
- const rows = content
- .split("\n")
- .filter((line) => line.trim())
- .map((line) => JSON.parse(line) as DatasetRow);
-
- const map = new Map<string, DatasetRow>();
- for (const row of rows) {
- const key = `${row.document}:${row.page}`;
- map.set(key, row);
- }
-
- return map;
-}
-
-/**
- * Normalize text for comparison - handles cross-platform LibreOffice differences.
- * Collapses whitespace runs to single space and trims each line.
- */
-function normalizeForComparison(text: string): string {
- return text
- .split("\n")
- .map((line) => line.replace(/\s+/g, " ").trim())
- .filter((line) => line.length > 0)
- .join("\n");
-}
-
-interface DiffHunk {
- startLine: number;
- endLine: number;
-}
-
-function computeTextDiff(expected: string, actual: string, maxHunks = 5, contextLines = 1): string {
- const expectedLines = expected.split("\n");
- const actualLines = actual.split("\n");
-
- // Find all changed line indices
- const maxLines = Math.max(expectedLines.length, actualLines.length);
- const changedIndices: number[] = [];
-
- for (let i = 0; i < maxLines; i++) {
- const exp = expectedLines[i] ?? "";
- const act = actualLines[i] ?? "";
- if (exp !== act) {
- changedIndices.push(i);
- }
- }
-
- if (changedIndices.length === 0) {
- return "";
- }
-
- // Group consecutive changes into hunks (with context gap tolerance)
- const hunks: DiffHunk[] = [];
- let currentHunk: DiffHunk | null = null;
-
- for (const idx of changedIndices) {
- // Start new hunk if this change is far from the previous one
- // (more than 2*contextLines apart, so hunks don't overlap)
- if (!currentHunk || idx > currentHunk.endLine + 2 * contextLines + 1) {
- if (currentHunk) {
- hunks.push(currentHunk);
- }
- currentHunk = {
- startLine: idx,
- endLine: idx,
- };
- } else {
- currentHunk.endLine = idx;
- }
- }
- if (currentHunk) {
- hunks.push(currentHunk);
- }
-
- // Build output for each hunk (limited to maxHunks)
- const output: string[] = [];
- const displayHunks = hunks.slice(0, maxHunks);
-
- for (const hunk of displayHunks) {
- // Expand hunk range to include context
- const contextStart = Math.max(0, hunk.startLine - contextLines);
- const contextEnd = Math.min(maxLines - 1, hunk.endLine + contextLines);
-
- // Format hunk header
- const lineRange = hunk.startLine === hunk.endLine
- ? `line ${hunk.startLine + 1}`
- : `lines ${hunk.startLine + 1}-${hunk.endLine + 1}`;
- output.push(`@@ ${lineRange} @@`);
-
- // Collect context before, removed lines, added lines, and context after
- const contextBefore: string[] = [];
- const removedLines: string[] = [];
- const addedLines: string[] = [];
- const contextAfter: string[] = [];
-
- for (let i = contextStart; i <= contextEnd; i++) {
- const exp = expectedLines[i] ?? "";
- const act = actualLines[i] ?? "";
- const isBeforeChanges = i < hunk.startLine;
- const isAfterChanges = i > hunk.endLine;
-
- if (exp === act) {
- // Context line (unchanged)
- if (isBeforeChanges) {
- contextBefore.push(` ${i + 1}: ${exp}`);
- } else if (isAfterChanges) {
- contextAfter.push(` ${i + 1}: ${exp}`);
- }
- } else {
- // Changed line - collect separately
- if (exp) {
- removedLines.push(`- ${i + 1}: ${exp}`);
- }
- if (act) {
- addedLines.push(`+ ${i + 1}: ${act}`);
- }
- }
- }
-
- // Output: context before, then all removed, then all added, then context after
- output.push(...contextBefore);
- output.push(...removedLines);
- output.push(...addedLines);
- output.push(...contextAfter);
- }
-
- if (hunks.length > maxHunks) {
- output.push(`\n... (${hunks.length - maxHunks} more change groups)`);
- }
-
- return output.join("\n");
-}
-
-async function getCurrentOutput(
- filePath: string
-): Promise<Map<number, { text: string; json: object }>> {
- const parser = new LiteParse({
- outputFormat: "json",
- ocrEnabled: false,
- preciseBoundingBox: true,
- });
-
- const result = await parser.parse(filePath, true);
- const outputs = new Map<number, { text: string; json: object }>();
-
- if (result.pages.length === 0 && result.text) {
- // Text-only result
- outputs.set(1, { text: result.text, json: { text: result.text } });
- } else {
- for (const page of result.pages) {
- const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);
- outputs.set(page.pageNum, {
- text: page.text,
- json: jsonPage || { page: page.pageNum, text: page.text },
- });
- }
- }
-
- return outputs;
-}
-
-async function main() {
- const datasetDir = process.argv[2] || DEFAULT_DATASET_DIR;
- const documentsDir = path.join(datasetDir, "data");
-
- console.log("LiteParse Dataset Comparison");
- console.log("============================");
- console.log(`Dataset: ${datasetDir}`);
- console.log(`Documents: ${documentsDir}`);
- console.log();
-
- // Load expected dataset
- let expected: Map<string, DatasetRow>;
- try {
- expected = await loadDataset(datasetDir);
- console.log(`Loaded ${expected.size} expected entries\n`);
-
- if (expected.size === 0) {
- console.error("ERROR: metadata.jsonl is empty — dataset has no entries to compare against.");
- console.error("The dataset may need to be regenerated with: npx tsx scripts/create-dataset.ts");
- process.exit(2);
- }
- } catch (error) {
- console.error(`Failed to load dataset: ${error}`);
- process.exit(2);
- }
-
- const diffs: DiffResult[] = [];
- const processedKeys = new Set<string>();
-
- // Group expected entries by document
- const docEntries = new Map<string, DatasetRow[]>();
- for (const row of expected.values()) {
- const existing = docEntries.get(row.document) || [];
- existing.push(row);
- docEntries.set(row.document, existing);
- }
-
- // Process each document
- for (const [document, rows] of docEntries) {
- const filePath = path.join(documentsDir, document);
-
- // Check if file still exists
- try {
- await fs.access(filePath);
- } catch {
- // File removed
- for (const row of rows) {
- const key = `${row.document}:${row.page}`;
- processedKeys.add(key);
- diffs.push({
- document: row.document,
- page: row.page,
- type: "removed",
- expected: row.output_text,
- });
- }
- continue;
- }
-
- console.log(`Checking: ${document}`);
-
- try {
- const currentOutputs = await getCurrentOutput(filePath);
-
- // Compare each page
- for (const row of rows) {
- const key = `${row.document}:${row.page}`;
- processedKeys.add(key);
-
- const current = currentOutputs.get(row.page);
- if (!current) {
- diffs.push({
- document: row.document,
- page: row.page,
- type: "removed",
- expected: row.output_text,
- });
- continue;
- }
-
- // Compare text output
- // For non-PDF files (PPTX, DOCX, etc.), normalize whitespace because
- // LibreOffice conversion produces different spacing across platforms.
- // For PDFs, use strict comparison since they're parsed directly.
- const expectedText = row.output_text.trim();
- const actualText = current.text.trim();
- const isPdf = row.document.toLowerCase().endsWith(".pdf");
-
- const expectedCompare = isPdf ? expectedText : normalizeForComparison(expectedText);
- const actualCompare = isPdf ? actualText : normalizeForComparison(actualText);
-
- if (expectedCompare !== actualCompare) {
- diffs.push({
- document: row.document,
- page: row.page,
- type: "changed",
- expected: expectedText,
- actual: actualText,
- diff: computeTextDiff(expectedText, actualText),
- });
- }
-
- currentOutputs.delete(row.page);
- }
-
- // Check for new pages
- for (const [pageNum, output] of currentOutputs) {
- diffs.push({
- document,
- page: pageNum,
- type: "added",
- actual: output.text,
- });
- }
- } catch (error) {
- console.error(` ERROR: ${error instanceof Error ? error.message : error}`);
- // Check if it was an expected error
- const errorRow = rows.find(
- (r) => (r.output_json as { error?: boolean }).error === true
- );
- if (!errorRow) {
- diffs.push({
- document,
- page: 0,
- type: "changed",
- expected: "successful parse",
- actual: `error: ${error instanceof Error ? error.message : error}`,
- });
- }
- for (const row of rows) {
- processedKeys.add(`${row.document}:${row.page}`);
- }
- }
- }
-
- // Report results
- console.log();
- console.log("Results");
- console.log("-------");
-
- if (diffs.length === 0) {
- console.log("✓ No changes detected");
- process.exit(0);
- }
-
- console.log(`✗ ${diffs.length} change(s) detected:\n`);
-
- for (const diff of diffs) {
- console.log(`[${diff.type.toUpperCase()}] ${diff.document} (page ${diff.page})`);
- if (diff.diff) {
- console.log(diff.diff);
- }
- console.log();
- }
-
- // Output summary for CI
- console.log("---");
- console.log("SUMMARY:");
- console.log(` Added: ${diffs.filter((d) => d.type === "added").length}`);
- console.log(` Removed: ${diffs.filter((d) => d.type === "removed").length}`);
- console.log(` Changed: ${diffs.filter((d) => d.type === "changed").length}`);
- console.log();
- console.log("This PR changes liteparse output and requires manual approval.");
-
- process.exit(1);
-}
-
-main().catch((error) => {
- console.error("Fatal error:", error);
- process.exit(2);
-});
diff --git a/scripts/compare-outputs.sh b/scripts/compare-outputs.sh
index aa0c649..bf0e8ab 100755
--- a/scripts/compare-outputs.sh
+++ b/scripts/compare-outputs.sh
@@ -9,7 +9,7 @@ OUTPUT_FILE="comparison-output.txt"
# Run comparison script
set +e
-npx tsx scripts/compare-dataset.ts "$EXPECTED_DATASET" > "$OUTPUT_FILE" 2>&1
+./scripts/compare-dataset.sh "$EXPECTED_DATASET" > "$OUTPUT_FILE" 2>&1
EXIT_CODE=$?
set -e
diff --git a/scripts/create-dataset.sh b/scripts/create-dataset.sh
new file mode 100755
index 0000000..ee2ae31
--- /dev/null
+++ b/scripts/create-dataset.sh
@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+# Creates a dataset for regression testing
+#
+# Output structure:
+# dataset/
+# data/
+# doc1.pdf
+# doc2.docx
+# ...
+# metadata.jsonl (each line: {"file_name":"data/doc1.pdf","document":"doc1.pdf","page":1,"output_text":"...","output_json":{...}})
+#
+# Usage:
+# ./scripts/create-dataset.sh [output-dir] [source-docs-dir]
+#
+# Arguments:
+# output-dir - Where to write the dataset (default: ./dataset)
+# source-docs-dir - Where to read source documents from (default: ./e2e-test-docs)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+OUTPUT_DIR="${1:-$REPO_ROOT/dataset}"
+SOURCE_DOCS_DIR="${2:-$REPO_ROOT/e2e-test-docs}"
+DOCUMENTS_DIR="$OUTPUT_DIR/data"
+
+# Resolve paths for comparison
+RESOLVED_SOURCE="$(cd "$SOURCE_DOCS_DIR" 2>/dev/null && pwd)"
+RESOLVED_DOCUMENTS="$(mkdir -p "$DOCUMENTS_DIR" && cd "$DOCUMENTS_DIR" && pwd)"
+SKIP_COPY=false
+if [ "$RESOLVED_SOURCE" = "$RESOLVED_DOCUMENTS" ]; then
+ SKIP_COPY=true
+fi
+
+LIT="$REPO_ROOT/target/release/lit"
+if [ ! -x "$LIT" ]; then
+ echo "ERROR: lit binary not found at $LIT. Run 'cargo build --release' first."
+ exit 1
+fi
+
+echo "LiteParse Dataset Generator"
+echo "==========================="
+echo "Source: $SOURCE_DOCS_DIR"
+echo "Output: $OUTPUT_DIR"
+if [ "$SKIP_COPY" = true ]; then
+ echo "(Source is output documents dir - skipping copy)"
+fi
+echo
+
+mkdir -p "$DOCUMENTS_DIR"
+
+METADATA_FILE="$OUTPUT_DIR/metadata.jsonl"
+: > "$METADATA_FILE"
+
+TOTAL_ENTRIES=0
+TOTAL_FILES=0
+
+# Find all files in source directory
+while IFS= read -r -d '' file; do
+ REL_PATH="${file#"$SOURCE_DOCS_DIR/"}"
+ TOTAL_FILES=$((TOTAL_FILES + 1))
+ echo "Processing: $REL_PATH"
+
+ # Copy file to dataset if needed
+ if [ "$SKIP_COPY" = false ]; then
+ DEST_PATH="$DOCUMENTS_DIR/$REL_PATH"
+ mkdir -p "$(dirname "$DEST_PATH")"
+ cp "$file" "$DEST_PATH"
+ fi
+
+ # Parse with lit and capture JSON output
+ JSON_OUTPUT=""
+ PARSE_ERROR=""
+ if JSON_OUTPUT=$("$LIT" parse --format json --no-ocr -q "$file" 2>&1); then
+ # Extract per-page data from JSON
+ PAGE_COUNT=$(echo "$JSON_OUTPUT" | jq '.pages | length')
+
+ if [ "$PAGE_COUNT" -eq 0 ]; then
+ # No pages - single text entry
+ TEXT=$(echo "$JSON_OUTPUT" | jq -r '.text // ""')
+ jq -nc \
+ --arg fn "data/$REL_PATH" \
+ --arg doc "$REL_PATH" \
+ --arg text "$TEXT" \
+ '{file_name: $fn, document: $doc, page: 1, output_text: $text, output_json: {text: $text}}' \
+ >> "$METADATA_FILE"
+ TOTAL_ENTRIES=$((TOTAL_ENTRIES + 1))
+ echo " -> 1 text entry"
+ else
+ for i in $(seq 0 $((PAGE_COUNT - 1))); do
+ PAGE_JSON=$(echo "$JSON_OUTPUT" | jq ".pages[$i]")
+ PAGE_NUM=$(echo "$PAGE_JSON" | jq '.page')
+ PAGE_TEXT=$(echo "$PAGE_JSON" | jq -r '.text // ""')
+
+ jq -nc \
+ --arg fn "data/$REL_PATH" \
+ --arg doc "$REL_PATH" \
+ --argjson page "$PAGE_NUM" \
+ --arg text "$PAGE_TEXT" \
+ --argjson json "$PAGE_JSON" \
+ '{file_name: $fn, document: $doc, page: $page, output_text: $text, output_json: $json}' \
+ >> "$METADATA_FILE"
+ TOTAL_ENTRIES=$((TOTAL_ENTRIES + 1))
+ done
+ echo " -> $PAGE_COUNT pages"
+ fi
+ else
+ PARSE_ERROR="$JSON_OUTPUT"
+ echo " ERROR: $PARSE_ERROR"
+ jq -nc \
+ --arg fn "data/$REL_PATH" \
+ --arg doc "$REL_PATH" \
+ --arg msg "$PARSE_ERROR" \
+ '{file_name: $fn, document: $doc, page: 0, output_text: "", output_json: {error: true, message: $msg}}' \
+ >> "$METADATA_FILE"
+ TOTAL_ENTRIES=$((TOTAL_ENTRIES + 1))
+ fi
+done < <(find "$SOURCE_DOCS_DIR" -type f -print0 | sort -z)
+
+if [ "$TOTAL_ENTRIES" -eq 0 ]; then
+ echo
+ echo "ERROR: No dataset entries were generated. Check that source documents exist and are parseable."
+ exit 1
+fi
+
+echo
+echo "Dataset generation complete!"
+echo " Total entries: $TOTAL_ENTRIES"
+echo " Documents: $TOTAL_FILES"
+echo " Metadata: $METADATA_FILE"
+echo " Documents dir: $DOCUMENTS_DIR"
+echo
+echo "Use compare-dataset.sh to compare future output against this baseline."
diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.ts
deleted file mode 100644
index 71ade58..0000000
--- a/scripts/create-dataset.ts
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Creates a dataset for regression testing
- *
- * Output structure:
- * dataset/
- * data/
- * doc1.pdf
- * doc2.docx
- * ...
- * metadata.jsonl (each line: {"file_name": "data/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})
- *
- * Usage:
- * npx tsx scripts/create-dataset.ts [output-dir] [source-docs-dir]
- *
- * Arguments:
- * output-dir - Where to write the dataset (default: ./dataset)
- * source-docs-dir - Where to read source documents from (default: ./e2e-test-docs)
- *
- * The dataset can be used with compare-dataset.ts to detect output changes.
- */
-
-import { LiteParse } from "../src/lib.js";
-import * as fs from "fs/promises";
-import * as path from "path";
-
-const DEFAULT_SOURCE_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");
-const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");
-
-interface DatasetRow {
- file_name: string;
- document: string;
- page: number;
- output_text: string;
- output_json: object;
-}
-
-async function findFiles(dir: string, baseDir: string = dir): Promise<string[]> {
- const files: string[] = [];
- const entries = await fs.readdir(dir, { withFileTypes: true });
-
- for (const entry of entries) {
- const fullPath = path.join(dir, entry.name);
-
- if (entry.isDirectory()) {
- // Recursively search subdirectories
- const subFiles = await findFiles(fullPath, baseDir);
- files.push(...subFiles);
- } else if (entry.isFile()) {
- files.push(fullPath);
- }
- }
-
- return files;
-}
-
-async function processFile(filePath: string, baseDocDir: string): Promise<DatasetRow[]> {
- const relativePath = path.relative(baseDocDir, filePath);
-
- console.log(`Processing: ${relativePath}`);
-
- const rows: DatasetRow[] = [];
-
- try {
- // Parse the document
- const parser = new LiteParse({
- outputFormat: "json",
- ocrEnabled: false, // Disable OCR for deterministic results
- preciseBoundingBox: true,
- });
-
- const result = await parser.parse(filePath, true);
-
- // For non-PDF files that return text directly (no pages)
- if (result.pages.length === 0 && result.text) {
- rows.push({
- file_name: `data/${relativePath}`,
- document: relativePath,
- page: 1,
- output_text: result.text,
- output_json: { text: result.text },
- });
- console.log(` -> 1 text entry`);
- return rows;
- }
-
- // Create entries for each page
- for (const page of result.pages) {
- const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);
- rows.push({
- file_name: `data/${relativePath}`,
- document: relativePath,
- page: page.pageNum,
- output_text: page.text,
- output_json: jsonPage || { page: page.pageNum, text: page.text },
- });
- }
-
- console.log(` -> ${rows.length} pages`);
- } catch (error) {
- console.error(` ERROR: ${error instanceof Error ? error.message : error}`);
- // Record the error as a dataset entry (blank result)
- rows.push({
- file_name: `data/${relativePath}`,
- document: relativePath,
- page: 0,
- output_text: "",
- output_json: {
- error: true,
- message: error instanceof Error ? error.message : String(error),
- },
- });
- }
-
- return rows;
-}
-
-async function main() {
- const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;
- const sourceDocsDir = process.argv[3] || DEFAULT_SOURCE_DIR;
- const documentsDir = path.join(outputDir, "data");
-
- // Check if source is already in the output documents folder (skip copy in this case)
- const resolvedSource = path.resolve(sourceDocsDir);
- const resolvedDocuments = path.resolve(documentsDir);
- const skipCopy = resolvedSource === resolvedDocuments;
-
- console.log("LiteParse Dataset Generator");
- console.log("===========================");
- console.log(`Source: ${sourceDocsDir}`);
- console.log(`Output: ${outputDir}`);
- if (skipCopy) {
- console.log(`(Source is output documents dir - skipping copy)`);
- }
- console.log();
-
- // Create output directories
- await fs.mkdir(documentsDir, { recursive: true });
-
- // Find all processable files
- console.log("Finding files...");
- const files = await findFiles(sourceDocsDir);
- console.log(`Found ${files.length} files to process\n`);
-
- // Process each file and copy to documents folder (unless source is already there)
- const allRows: DatasetRow[] = [];
- const copiedDocs = new Set<string>();
-
- for (const file of files) {
- const rows = await processFile(file, sourceDocsDir);
- allRows.push(...rows);
-
- // Copy document file to dataset (only once per document, skip if source is output)
- const relativePath = path.relative(sourceDocsDir, file);
- if (!skipCopy && !copiedDocs.has(relativePath)) {
- const destPath = path.join(documentsDir, relativePath);
- await fs.mkdir(path.dirname(destPath), { recursive: true });
- await fs.copyFile(file, destPath);
- copiedDocs.add(relativePath);
- } else if (skipCopy) {
- copiedDocs.add(relativePath);
- }
- }
-
- // Validate we actually produced entries
- if (allRows.length === 0) {
- console.error("\nERROR: No dataset entries were generated. Check that source documents exist and are parseable.");
- process.exit(1);
- }
-
- // Write metadata.jsonl
- const metadataPath = path.join(outputDir, "metadata.jsonl");
- const metadataContent = allRows.map((row) => JSON.stringify(row)).join("\n");
- await fs.writeFile(metadataPath, metadataContent);
-
- console.log();
- console.log("Dataset generation complete!");
- console.log(` Total entries: ${allRows.length}`);
- console.log(` Documents copied: ${copiedDocs.size}`);
- console.log(` Metadata: ${metadataPath}`);
- console.log(` Documents: ${documentsDir}`);
- console.log();
- console.log("Use compare-dataset.ts to compare future output against this baseline.");
-}
-
-main().catch((error) => {
- console.error("Fatal error:", error);
- process.exit(1);
-});
diff --git a/scripts/upload-dataset.sh b/scripts/upload-dataset.sh
new file mode 100755
index 0000000..511b6ef
--- /dev/null
+++ b/scripts/upload-dataset.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Regenerates and uploads the dataset to HuggingFace
+#
+# Usage:
+# HF_TOKEN=xxx ./scripts/upload-dataset.sh [dataset-dir] [repo-name]
+#
+# Arguments:
+# dataset-dir - Directory containing the dataset with data/ subfolder (default: ./dataset)
+# repo-name - HuggingFace repository name (default: llamaindex/liteparse_cicd_data)
+#
+# Environment variables:
+# HF_TOKEN - HuggingFace API token with write access
+#
+# Requires: hf cli (pip install huggingface_hub)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+DATASET_DIR="${1:-$REPO_ROOT/dataset}"
+REPO_NAME="${2:-llamaindex/liteparse_cicd_data}"
+
+if [ -z "${HF_TOKEN:-}" ]; then
+ echo "Error: HF_TOKEN environment variable is required"
+ echo "Get a token from https://huggingface.co/settings/tokens"
+ exit 1
+fi
+
+DOCUMENTS_DIR="$DATASET_DIR/data"
+
+echo "LiteParse Dataset Upload"
+echo "========================"
+echo "Dataset: $DATASET_DIR"
+echo "Documents: $DOCUMENTS_DIR"
+echo "Repo: $REPO_NAME"
+echo
+
+# Step 1: Regenerate dataset from documents in the dataset directory
+echo "Step 1: Regenerating dataset from existing documents..."
+"$SCRIPT_DIR/create-dataset.sh" "$DATASET_DIR" "$DOCUMENTS_DIR"
+
+# Step 2: Upload to HuggingFace
+echo
+echo "Step 2: Uploading to HuggingFace..."
+hf upload "$REPO_NAME" "$DATASET_DIR" --repo-type dataset --token "$HF_TOKEN"
+
+echo
+echo "✓ Dataset uploaded successfully!"
+echo " View at: https://huggingface.co/datasets/$REPO_NAME"
diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.ts
deleted file mode 100644
index 839959b..0000000
--- a/scripts/upload-dataset.ts
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Regenerates and uploads the dataset to HuggingFace
- *
- * Usage:
- * HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]
- *
- * Arguments:
- * dataset-dir - Directory containing the dataset with data/ subfolder
- * (default: ./dataset)
- * repo-name - HuggingFace repository name (default: llamaindex/liteparse_cicd_data)
- *
- * Environment variables:
- * HF_TOKEN - HuggingFace API token with write access
- *
- * This script:
- * 1. Regenerates metadata.jsonl by re-running liteparse on data/ in the dataset
- * 2. Uploads to HuggingFace using the huggingface_hub API
- */
-
-import { execSync } from "child_process";
-import * as path from "path";
-
-const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");
-const DEFAULT_REPO = "llamaindex/liteparse_cicd_data";
-
-async function main() {
- const datasetDir = process.argv[2] || DEFAULT_OUTPUT_DIR;
- const repoName = process.argv[3] || DEFAULT_REPO;
- const hfToken = process.env.HF_TOKEN;
-
- if (!hfToken) {
- console.error("Error: HF_TOKEN environment variable is required");
- console.error("Get a token from https://huggingface.co/settings/tokens");
- process.exit(1);
- }
-
- const documentsDir = path.join(datasetDir, "data");
-
- console.log("LiteParse Dataset Upload");
- console.log("========================");
- console.log(`Dataset: ${datasetDir}`);
- console.log(`Documents: ${documentsDir}`);
- console.log(`Repo: ${repoName}`);
- console.log();
-
- // Step 1: Regenerate dataset from documents in the dataset directory
- console.log("Step 1: Regenerating dataset from existing documents...");
- try {
- // Pass both output dir and source docs dir (data/ within the dataset)
- execSync(`npx tsx scripts/create-dataset.ts "${datasetDir}" "${documentsDir}"`, {
- cwd: path.join(import.meta.dirname, ".."),
- stdio: "inherit",
- });
- } catch (error) {
- console.error("Failed to generate dataset");
- process.exit(1);
- }
-
- // Step 2: Upload to HuggingFace
- console.log("\nStep 2: Uploading to HuggingFace...");
- try {
- // Use hf cli to upload
- // This requires hf cli to be installed: brew install huggingface-cli
- execSync(
- `hf upload ${repoName} "${datasetDir}" --repo-type dataset --token "${hfToken}"`,
- {
- cwd: path.join(import.meta.dirname, ".."),
- stdio: "inherit",
- }
- );
- console.log("\n✓ Dataset uploaded successfully!");
- console.log(` View at: https://huggingface.co/datasets/${repoName}`);
- } catch (error) {
- console.error("Failed to upload to HuggingFace");
- console.error("Make sure hf cli is installed: brew install huggingface-cli");
- process.exit(1);
- }
-}
-
-main().catch((error) => {
- console.error("Fatal error:", error);
- process.exit(1);
-});