A fast, helpful, and open-source document parser
migrate workflows
Logan Markewichbf5f2feb06b9
diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.ymlindex 3cade7b..0929927 100644--- a/.github/workflows/e2e-output.yml+++ b/.github/workflows/e2e-output.yml@@ -39,18 +39,6 @@ jobs:- name: Buildrun: cargo build --release- # The comparison scripts (compare-dataset.ts, upload-dataset.ts) still use- # Node/tsx and need to be ported to Rust or rewritten as shell scripts.- # For now, we install Node to run them.- - name: Setup Node.js- uses: actions/setup-node@v4- with:- node-version: 20- cache: "npm"-- - name: Install Node dependencies- run: npm ci-- name: Download from HuggingFaceenv:HF_TOKEN: ${{ secrets.HF_TOKEN }}@@ -135,21 +123,13 @@ jobs:- name: Buildrun: cargo build --release- - name: Setup Node.js- uses: actions/setup-node@v4- with:- node-version: 20- cache: "npm"-- name: Setup Pythonuses: actions/setup-python@v5with:python-version: "3.11"- name: Install dependencies- run: |- npm ci- pip install huggingface_hub+ run: pip install huggingface_hub- name: Download existing dataset from HuggingFaceenv:@@ -165,4 +145,4 @@ jobs:echo "Warning: HF_TOKEN not set, skipping upload"exit 0fi- npx tsx scripts/upload-dataset.ts expected-dataset ${{ vars.HF_DATASET_REPO || 'llamaindex/liteparse_cicd_data' }}+ ./scripts/upload-dataset.sh expected-dataset ${{ vars.HF_DATASET_REPO || 'llamaindex/liteparse_cicd_data' }}diff --git a/scripts/compare-dataset.sh b/scripts/compare-dataset.shnew file mode 100755index 0000000..dcbf46c--- /dev/null+++ b/scripts/compare-dataset.sh@@ -0,0 +1,177 @@+#!/usr/bin/env bash+# Compares current liteparse output against a baseline dataset+#+# Usage:+# ./scripts/compare-dataset.sh [dataset-dir]+#+# Exit codes:+# 0 - No changes detected+# 1 - Changes detected (requires approval)+# 2 - Error occurred+set -uo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"++DATASET_DIR="${1:-$REPO_ROOT/dataset}"+DOCUMENTS_DIR="$DATASET_DIR/data"+METADATA_FILE="$DATASET_DIR/metadata.jsonl"++LIT="$REPO_ROOT/target/release/lit"+if [ ! -x "$LIT" ]; then+ echo "ERROR: lit binary not found at $LIT. Run 'cargo build --release' first."+ exit 2+fi++echo "LiteParse Dataset Comparison"+echo "============================"+echo "Dataset: $DATASET_DIR"+echo "Documents: $DOCUMENTS_DIR"+echo++if [ ! -f "$METADATA_FILE" ]; then+ echo "ERROR: metadata.jsonl not found at $METADATA_FILE"+ exit 2+fi++ENTRY_COUNT=$(wc -l < "$METADATA_FILE" | tr -d ' ')+echo "Loaded $ENTRY_COUNT expected entries"+echo++if [ "$ENTRY_COUNT" -eq 0 ]; then+ echo "ERROR: metadata.jsonl is empty — dataset has no entries to compare against."+ echo "The dataset may need to be regenerated with: ./scripts/create-dataset.sh"+ exit 2+fi++DIFF_COUNT=0+ADDED_COUNT=0+REMOVED_COUNT=0+CHANGED_COUNT=0+DIFF_OUTPUT=""++# Collect unique documents from metadata+DOCUMENTS=$(jq -r '.document' "$METADATA_FILE" | sort -u)++while IFS= read -r document; do+ [ -z "$document" ] && continue+ FILE_PATH="$DOCUMENTS_DIR/$document"++ # Check if file exists+ if [ ! -f "$FILE_PATH" ]; then+ # All pages for this document are removed+ while IFS= read -r line; do+ PAGE=$(echo "$line" | jq -r '.page')+ DIFF_OUTPUT+="[REMOVED] $document (page $PAGE)"$'\n\n'+ REMOVED_COUNT=$((REMOVED_COUNT + 1))+ DIFF_COUNT=$((DIFF_COUNT + 1))+ done < <(jq -c "select(.document == \"$document\")" "$METADATA_FILE")+ continue+ fi++ echo "Checking: $document"++ # Parse current output+ CURRENT_JSON=""+ PARSE_ERROR=""+ if CURRENT_JSON=$("$LIT" parse --format json --no-ocr -q "$FILE_PATH" 2>&1); then+ # Compare each expected page+ while IFS= read -r line; do+ EXPECTED_PAGE=$(echo "$line" | jq -r '.page')+ EXPECTED_TEXT=$(echo "$line" | jq -r '.output_text')+ IS_PDF=false+ if [[ "$document" == *.pdf ]]; then+ IS_PDF=true+ fi++ # Get actual text for this page+ ACTUAL_TEXT=$(echo "$CURRENT_JSON" | jq -r --argjson page "$EXPECTED_PAGE" \+ '.pages[] | select(.page == $page) | .text // ""' 2>/dev/null)++ if [ -z "$ACTUAL_TEXT" ] && [ "$EXPECTED_PAGE" -ne 0 ]; then+ # Page not found in current output+ DIFF_OUTPUT+="[REMOVED] $document (page $EXPECTED_PAGE)"$'\n\n'+ REMOVED_COUNT=$((REMOVED_COUNT + 1))+ DIFF_COUNT=$((DIFF_COUNT + 1))+ continue+ fi++ # Normalize for comparison+ if [ "$IS_PDF" = true ]; then+ EXPECTED_CMP=$(echo "$EXPECTED_TEXT" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')+ ACTUAL_CMP=$(echo "$ACTUAL_TEXT" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')+ else+ # For non-PDF: collapse whitespace runs, trim lines, remove empty lines+ EXPECTED_CMP=$(echo "$EXPECTED_TEXT" | sed 's/[[:space:]]\{1,\}/ /g' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed '/^$/d')+ ACTUAL_CMP=$(echo "$ACTUAL_TEXT" | sed 's/[[:space:]]\{1,\}/ /g' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed '/^$/d')+ fi++ if [ "$EXPECTED_CMP" != "$ACTUAL_CMP" ]; then+ DIFF_OUTPUT+="[CHANGED] $document (page $EXPECTED_PAGE)"$'\n'+ # Show a unified-style diff+ DIFF_DETAIL=$(diff <(echo "$EXPECTED_TEXT") <(echo "$ACTUAL_TEXT") --unified=1 2>/dev/null || true)+ if [ -n "$DIFF_DETAIL" ]; then+ # Show first 50 lines of diff to avoid massive output+ DIFF_OUTPUT+="$(echo "$DIFF_DETAIL" | head -50)"$'\n'+ REMAINING=$(echo "$DIFF_DETAIL" | wc -l | tr -d ' ')+ if [ "$REMAINING" -gt 50 ]; then+ DIFF_OUTPUT+="... ($((REMAINING - 50)) more lines)"$'\n'+ fi+ fi+ DIFF_OUTPUT+=$'\n'+ CHANGED_COUNT=$((CHANGED_COUNT + 1))+ DIFF_COUNT=$((DIFF_COUNT + 1))+ fi+ done < <(jq -c "select(.document == \"$document\")" "$METADATA_FILE")++ # Check for new pages not in expected dataset+ EXPECTED_PAGES=$(jq -r "select(.document == \"$document\") | .page" "$METADATA_FILE" | sort -n)+ ACTUAL_PAGES=$(echo "$CURRENT_JSON" | jq -r '.pages[].page' | sort -n)++ while IFS= read -r page; do+ [ -z "$page" ] && continue+ if ! echo "$EXPECTED_PAGES" | grep -q "^${page}$"; then+ DIFF_OUTPUT+="[ADDED] $document (page $page)"$'\n\n'+ ADDED_COUNT=$((ADDED_COUNT + 1))+ DIFF_COUNT=$((DIFF_COUNT + 1))+ fi+ done <<< "$ACTUAL_PAGES"++ else+ PARSE_ERROR="$CURRENT_JSON"+ echo " ERROR: $PARSE_ERROR"++ # Check if error was expected+ HAS_ERROR_ENTRY=$(jq -r "select(.document == \"$document\") | .output_json.error // false" "$METADATA_FILE" | head -1)+ if [ "$HAS_ERROR_ENTRY" != "true" ]; then+ DIFF_OUTPUT+="[CHANGED] $document (page 0)"$'\n'+ DIFF_OUTPUT+=" Expected: successful parse"$'\n'+ DIFF_OUTPUT+=" Actual: error: $PARSE_ERROR"$'\n\n'+ CHANGED_COUNT=$((CHANGED_COUNT + 1))+ DIFF_COUNT=$((DIFF_COUNT + 1))+ fi+ fi+done <<< "$DOCUMENTS"++echo+echo "Results"+echo "-------"++if [ "$DIFF_COUNT" -eq 0 ]; then+ echo "✓ No changes detected"+ exit 0+fi++echo "✗ $DIFF_COUNT change(s) detected:"+echo+echo "$DIFF_OUTPUT"++echo "---"+echo "SUMMARY:"+echo " Added: $ADDED_COUNT"+echo " Removed: $REMOVED_COUNT"+echo " Changed: $CHANGED_COUNT"+echo+echo "This PR changes liteparse output and requires manual approval."++exit 1diff --git a/scripts/compare-dataset.ts b/scripts/compare-dataset.tsdeleted file mode 100644index 47b3974..0000000--- a/scripts/compare-dataset.ts+++ /dev/null@@ -1,377 +0,0 @@-/**- * Compares current liteparse output against a baseline dataset- *- * Usage:- * npx tsx scripts/compare-dataset.ts [dataset-dir]- *- * The dataset should be created using create-dataset.ts and contains:- * - data/: The original document files- * - metadata.jsonl: Expected outputs for each document/page- *- * Exit codes:- * 0 - No changes detected- * 1 - Changes detected (requires approval)- * 2 - Error occurred- *- * For CI/CD, this script can be run to detect if a PR changes output.- * If changes are detected, the PR should require manual approval.- */--import { LiteParse } from "../src/lib.js";-import * as fs from "fs/promises";-import * as path from "path";--const DEFAULT_DATASET_DIR = path.join(import.meta.dirname, "..", "dataset");--interface DatasetRow {- file_name: string;- document: string;- page: number;- output_text: string;- output_json: object;-}--interface DiffResult {- document: string;- page: number;- type: "added" | "removed" | "changed";- expected?: string;- actual?: string;- diff?: string;-}--async function loadDataset(datasetDir: string): Promise<Map<string, DatasetRow>> {- const metadataPath = path.join(datasetDir, "metadata.jsonl");- const content = await fs.readFile(metadataPath, "utf-8");- const rows = content- .split("\n")- .filter((line) => line.trim())- .map((line) => JSON.parse(line) as DatasetRow);-- const map = new Map<string, DatasetRow>();- for (const row of rows) {- const key = `${row.document}:${row.page}`;- map.set(key, row);- }-- return map;-}--/**- * Normalize text for comparison - handles cross-platform LibreOffice differences.- * Collapses whitespace runs to single space and trims each line.- */-function normalizeForComparison(text: string): string {- return text- .split("\n")- .map((line) => line.replace(/\s+/g, " ").trim())- .filter((line) => line.length > 0)- .join("\n");-}--interface DiffHunk {- startLine: number;- endLine: number;-}--function computeTextDiff(expected: string, actual: string, maxHunks = 5, contextLines = 1): string {- const expectedLines = expected.split("\n");- const actualLines = actual.split("\n");-- // Find all changed line indices- const maxLines = Math.max(expectedLines.length, actualLines.length);- const changedIndices: number[] = [];-- for (let i = 0; i < maxLines; i++) {- const exp = expectedLines[i] ?? "";- const act = actualLines[i] ?? "";- if (exp !== act) {- changedIndices.push(i);- }- }-- if (changedIndices.length === 0) {- return "";- }-- // Group consecutive changes into hunks (with context gap tolerance)- const hunks: DiffHunk[] = [];- let currentHunk: DiffHunk | null = null;-- for (const idx of changedIndices) {- // Start new hunk if this change is far from the previous one- // (more than 2*contextLines apart, so hunks don't overlap)- if (!currentHunk || idx > currentHunk.endLine + 2 * contextLines + 1) {- if (currentHunk) {- hunks.push(currentHunk);- }- currentHunk = {- startLine: idx,- endLine: idx,- };- } else {- currentHunk.endLine = idx;- }- }- if (currentHunk) {- hunks.push(currentHunk);- }-- // Build output for each hunk (limited to maxHunks)- const output: string[] = [];- const displayHunks = hunks.slice(0, maxHunks);-- for (const hunk of displayHunks) {- // Expand hunk range to include context- const contextStart = Math.max(0, hunk.startLine - contextLines);- const contextEnd = Math.min(maxLines - 1, hunk.endLine + contextLines);-- // Format hunk header- const lineRange = hunk.startLine === hunk.endLine- ? `line ${hunk.startLine + 1}`- : `lines ${hunk.startLine + 1}-${hunk.endLine + 1}`;- output.push(`@@ ${lineRange} @@`);-- // Collect context before, removed lines, added lines, and context after- const contextBefore: string[] = [];- const removedLines: string[] = [];- const addedLines: string[] = [];- const contextAfter: string[] = [];-- for (let i = contextStart; i <= contextEnd; i++) {- const exp = expectedLines[i] ?? "";- const act = actualLines[i] ?? "";- const isBeforeChanges = i < hunk.startLine;- const isAfterChanges = i > hunk.endLine;-- if (exp === act) {- // Context line (unchanged)- if (isBeforeChanges) {- contextBefore.push(` ${i + 1}: ${exp}`);- } else if (isAfterChanges) {- contextAfter.push(` ${i + 1}: ${exp}`);- }- } else {- // Changed line - collect separately- if (exp) {- removedLines.push(`- ${i + 1}: ${exp}`);- }- if (act) {- addedLines.push(`+ ${i + 1}: ${act}`);- }- }- }-- // Output: context before, then all removed, then all added, then context after- output.push(...contextBefore);- output.push(...removedLines);- output.push(...addedLines);- output.push(...contextAfter);- }-- if (hunks.length > maxHunks) {- output.push(`\n... (${hunks.length - maxHunks} more change groups)`);- }-- return output.join("\n");-}--async function getCurrentOutput(- filePath: string-): Promise<Map<number, { text: string; json: object }>> {- const parser = new LiteParse({- outputFormat: "json",- ocrEnabled: false,- preciseBoundingBox: true,- });-- const result = await parser.parse(filePath, true);- const outputs = new Map<number, { text: string; json: object }>();-- if (result.pages.length === 0 && result.text) {- // Text-only result- outputs.set(1, { text: result.text, json: { text: result.text } });- } else {- for (const page of result.pages) {- const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);- outputs.set(page.pageNum, {- text: page.text,- json: jsonPage || { page: page.pageNum, text: page.text },- });- }- }-- return outputs;-}--async function main() {- const datasetDir = process.argv[2] || DEFAULT_DATASET_DIR;- const documentsDir = path.join(datasetDir, "data");-- console.log("LiteParse Dataset Comparison");- console.log("============================");- console.log(`Dataset: ${datasetDir}`);- console.log(`Documents: ${documentsDir}`);- console.log();-- // Load expected dataset- let expected: Map<string, DatasetRow>;- try {- expected = await loadDataset(datasetDir);- console.log(`Loaded ${expected.size} expected entries\n`);-- if (expected.size === 0) {- console.error("ERROR: metadata.jsonl is empty — dataset has no entries to compare against.");- console.error("The dataset may need to be regenerated with: npx tsx scripts/create-dataset.ts");- process.exit(2);- }- } catch (error) {- console.error(`Failed to load dataset: ${error}`);- process.exit(2);- }-- const diffs: DiffResult[] = [];- const processedKeys = new Set<string>();-- // Group expected entries by document- const docEntries = new Map<string, DatasetRow[]>();- for (const row of expected.values()) {- const existing = docEntries.get(row.document) || [];- existing.push(row);- docEntries.set(row.document, existing);- }-- // Process each document- for (const [document, rows] of docEntries) {- const filePath = path.join(documentsDir, document);-- // Check if file still exists- try {- await fs.access(filePath);- } catch {- // File removed- for (const row of rows) {- const key = `${row.document}:${row.page}`;- processedKeys.add(key);- diffs.push({- document: row.document,- page: row.page,- type: "removed",- expected: row.output_text,- });- }- continue;- }-- console.log(`Checking: ${document}`);-- try {- const currentOutputs = await getCurrentOutput(filePath);-- // Compare each page- for (const row of rows) {- const key = `${row.document}:${row.page}`;- processedKeys.add(key);-- const current = currentOutputs.get(row.page);- if (!current) {- diffs.push({- document: row.document,- page: row.page,- type: "removed",- expected: row.output_text,- });- continue;- }-- // Compare text output- // For non-PDF files (PPTX, DOCX, etc.), normalize whitespace because- // LibreOffice conversion produces different spacing across platforms.- // For PDFs, use strict comparison since they're parsed directly.- const expectedText = row.output_text.trim();- const actualText = current.text.trim();- const isPdf = row.document.toLowerCase().endsWith(".pdf");-- const expectedCompare = isPdf ? expectedText : normalizeForComparison(expectedText);- const actualCompare = isPdf ? actualText : normalizeForComparison(actualText);-- if (expectedCompare !== actualCompare) {- diffs.push({- document: row.document,- page: row.page,- type: "changed",- expected: expectedText,- actual: actualText,- diff: computeTextDiff(expectedText, actualText),- });- }-- currentOutputs.delete(row.page);- }-- // Check for new pages- for (const [pageNum, output] of currentOutputs) {- diffs.push({- document,- page: pageNum,- type: "added",- actual: output.text,- });- }- } catch (error) {- console.error(` ERROR: ${error instanceof Error ? error.message : error}`);- // Check if it was an expected error- const errorRow = rows.find(- (r) => (r.output_json as { error?: boolean }).error === true- );- if (!errorRow) {- diffs.push({- document,- page: 0,- type: "changed",- expected: "successful parse",- actual: `error: ${error instanceof Error ? error.message : error}`,- });- }- for (const row of rows) {- processedKeys.add(`${row.document}:${row.page}`);- }- }- }-- // Report results- console.log();- console.log("Results");- console.log("-------");-- if (diffs.length === 0) {- console.log("✓ No changes detected");- process.exit(0);- }-- console.log(`✗ ${diffs.length} change(s) detected:\n`);-- for (const diff of diffs) {- console.log(`[${diff.type.toUpperCase()}] ${diff.document} (page ${diff.page})`);- if (diff.diff) {- console.log(diff.diff);- }- console.log();- }-- // Output summary for CI- console.log("---");- console.log("SUMMARY:");- console.log(` Added: ${diffs.filter((d) => d.type === "added").length}`);- console.log(` Removed: ${diffs.filter((d) => d.type === "removed").length}`);- console.log(` Changed: ${diffs.filter((d) => d.type === "changed").length}`);- console.log();- console.log("This PR changes liteparse output and requires manual approval.");-- process.exit(1);-}--main().catch((error) => {- console.error("Fatal error:", error);- process.exit(2);-});diff --git a/scripts/compare-outputs.sh b/scripts/compare-outputs.shindex aa0c649..bf0e8ab 100755--- a/scripts/compare-outputs.sh+++ b/scripts/compare-outputs.sh@@ -9,7 +9,7 @@ OUTPUT_FILE="comparison-output.txt"# Run comparison scriptset +e-npx tsx scripts/compare-dataset.ts "$EXPECTED_DATASET" > "$OUTPUT_FILE" 2>&1+./scripts/compare-dataset.sh "$EXPECTED_DATASET" > "$OUTPUT_FILE" 2>&1EXIT_CODE=$?set -ediff --git a/scripts/create-dataset.sh b/scripts/create-dataset.shnew file mode 100755index 0000000..ee2ae31--- /dev/null+++ b/scripts/create-dataset.sh@@ -0,0 +1,133 @@+#!/usr/bin/env bash+# Creates a dataset for regression testing+#+# Output structure:+# dataset/+# data/+# doc1.pdf+# doc2.docx+# ...+# metadata.jsonl (each line: {"file_name":"data/doc1.pdf","document":"doc1.pdf","page":1,"output_text":"...","output_json":{...}})+#+# Usage:+# ./scripts/create-dataset.sh [output-dir] [source-docs-dir]+#+# Arguments:+# output-dir - Where to write the dataset (default: ./dataset)+# source-docs-dir - Where to read source documents from (default: ./e2e-test-docs)+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"++OUTPUT_DIR="${1:-$REPO_ROOT/dataset}"+SOURCE_DOCS_DIR="${2:-$REPO_ROOT/e2e-test-docs}"+DOCUMENTS_DIR="$OUTPUT_DIR/data"++# Resolve paths for comparison+RESOLVED_SOURCE="$(cd "$SOURCE_DOCS_DIR" 2>/dev/null && pwd)"+RESOLVED_DOCUMENTS="$(mkdir -p "$DOCUMENTS_DIR" && cd "$DOCUMENTS_DIR" && pwd)"+SKIP_COPY=false+if [ "$RESOLVED_SOURCE" = "$RESOLVED_DOCUMENTS" ]; then+ SKIP_COPY=true+fi++LIT="$REPO_ROOT/target/release/lit"+if [ ! -x "$LIT" ]; then+ echo "ERROR: lit binary not found at $LIT. Run 'cargo build --release' first."+ exit 1+fi++echo "LiteParse Dataset Generator"+echo "==========================="+echo "Source: $SOURCE_DOCS_DIR"+echo "Output: $OUTPUT_DIR"+if [ "$SKIP_COPY" = true ]; then+ echo "(Source is output documents dir - skipping copy)"+fi+echo++mkdir -p "$DOCUMENTS_DIR"++METADATA_FILE="$OUTPUT_DIR/metadata.jsonl"+: > "$METADATA_FILE"++TOTAL_ENTRIES=0+TOTAL_FILES=0++# Find all files in source directory+while IFS= read -r -d '' file; do+ REL_PATH="${file#"$SOURCE_DOCS_DIR/"}"+ TOTAL_FILES=$((TOTAL_FILES + 1))+ echo "Processing: $REL_PATH"++ # Copy file to dataset if needed+ if [ "$SKIP_COPY" = false ]; then+ DEST_PATH="$DOCUMENTS_DIR/$REL_PATH"+ mkdir -p "$(dirname "$DEST_PATH")"+ cp "$file" "$DEST_PATH"+ fi++ # Parse with lit and capture JSON output+ JSON_OUTPUT=""+ PARSE_ERROR=""+ if JSON_OUTPUT=$("$LIT" parse --format json --no-ocr -q "$file" 2>&1); then+ # Extract per-page data from JSON+ PAGE_COUNT=$(echo "$JSON_OUTPUT" | jq '.pages | length')++ if [ "$PAGE_COUNT" -eq 0 ]; then+ # No pages - single text entry+ TEXT=$(echo "$JSON_OUTPUT" | jq -r '.text // ""')+ jq -nc \+ --arg fn "data/$REL_PATH" \+ --arg doc "$REL_PATH" \+ --arg text "$TEXT" \+ '{file_name: $fn, document: $doc, page: 1, output_text: $text, output_json: {text: $text}}' \+ >> "$METADATA_FILE"+ TOTAL_ENTRIES=$((TOTAL_ENTRIES + 1))+ echo " -> 1 text entry"+ else+ for i in $(seq 0 $((PAGE_COUNT - 1))); do+ PAGE_JSON=$(echo "$JSON_OUTPUT" | jq ".pages[$i]")+ PAGE_NUM=$(echo "$PAGE_JSON" | jq '.page')+ PAGE_TEXT=$(echo "$PAGE_JSON" | jq -r '.text // ""')++ jq -nc \+ --arg fn "data/$REL_PATH" \+ --arg doc "$REL_PATH" \+ --argjson page "$PAGE_NUM" \+ --arg text "$PAGE_TEXT" \+ --argjson json "$PAGE_JSON" \+ '{file_name: $fn, document: $doc, page: $page, output_text: $text, output_json: $json}' \+ >> "$METADATA_FILE"+ TOTAL_ENTRIES=$((TOTAL_ENTRIES + 1))+ done+ echo " -> $PAGE_COUNT pages"+ fi+ else+ PARSE_ERROR="$JSON_OUTPUT"+ echo " ERROR: $PARSE_ERROR"+ jq -nc \+ --arg fn "data/$REL_PATH" \+ --arg doc "$REL_PATH" \+ --arg msg "$PARSE_ERROR" \+ '{file_name: $fn, document: $doc, page: 0, output_text: "", output_json: {error: true, message: $msg}}' \+ >> "$METADATA_FILE"+ TOTAL_ENTRIES=$((TOTAL_ENTRIES + 1))+ fi+done < <(find "$SOURCE_DOCS_DIR" -type f -print0 | sort -z)++if [ "$TOTAL_ENTRIES" -eq 0 ]; then+ echo+ echo "ERROR: No dataset entries were generated. Check that source documents exist and are parseable."+ exit 1+fi++echo+echo "Dataset generation complete!"+echo " Total entries: $TOTAL_ENTRIES"+echo " Documents: $TOTAL_FILES"+echo " Metadata: $METADATA_FILE"+echo " Documents dir: $DOCUMENTS_DIR"+echo+echo "Use compare-dataset.sh to compare future output against this baseline."diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.tsdeleted file mode 100644index 71ade58..0000000--- a/scripts/create-dataset.ts+++ /dev/null@@ -1,188 +0,0 @@-/**- * Creates a dataset for regression testing- *- * Output structure:- * dataset/- * data/- * doc1.pdf- * doc2.docx- * ...- * metadata.jsonl (each line: {"file_name": "data/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})- *- * Usage:- * npx tsx scripts/create-dataset.ts [output-dir] [source-docs-dir]- *- * Arguments:- * output-dir - Where to write the dataset (default: ./dataset)- * source-docs-dir - Where to read source documents from (default: ./e2e-test-docs)- *- * The dataset can be used with compare-dataset.ts to detect output changes.- */--import { LiteParse } from "../src/lib.js";-import * as fs from "fs/promises";-import * as path from "path";--const DEFAULT_SOURCE_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");-const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");--interface DatasetRow {- file_name: string;- document: string;- page: number;- output_text: string;- output_json: object;-}--async function findFiles(dir: string, baseDir: string = dir): Promise<string[]> {- const files: string[] = [];- const entries = await fs.readdir(dir, { withFileTypes: true });-- for (const entry of entries) {- const fullPath = path.join(dir, entry.name);-- if (entry.isDirectory()) {- // Recursively search subdirectories- const subFiles = await findFiles(fullPath, baseDir);- files.push(...subFiles);- } else if (entry.isFile()) {- files.push(fullPath);- }- }-- return files;-}--async function processFile(filePath: string, baseDocDir: string): Promise<DatasetRow[]> {- const relativePath = path.relative(baseDocDir, filePath);-- console.log(`Processing: ${relativePath}`);-- const rows: DatasetRow[] = [];-- try {- // Parse the document- const parser = new LiteParse({- outputFormat: "json",- ocrEnabled: false, // Disable OCR for deterministic results- preciseBoundingBox: true,- });-- const result = await parser.parse(filePath, true);-- // For non-PDF files that return text directly (no pages)- if (result.pages.length === 0 && result.text) {- rows.push({- file_name: `data/${relativePath}`,- document: relativePath,- page: 1,- output_text: result.text,- output_json: { text: result.text },- });- console.log(` -> 1 text entry`);- return rows;- }-- // Create entries for each page- for (const page of result.pages) {- const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);- rows.push({- file_name: `data/${relativePath}`,- document: relativePath,- page: page.pageNum,- output_text: page.text,- output_json: jsonPage || { page: page.pageNum, text: page.text },- });- }-- console.log(` -> ${rows.length} pages`);- } catch (error) {- console.error(` ERROR: ${error instanceof Error ? error.message : error}`);- // Record the error as a dataset entry (blank result)- rows.push({- file_name: `data/${relativePath}`,- document: relativePath,- page: 0,- output_text: "",- output_json: {- error: true,- message: error instanceof Error ? error.message : String(error),- },- });- }-- return rows;-}--async function main() {- const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;- const sourceDocsDir = process.argv[3] || DEFAULT_SOURCE_DIR;- const documentsDir = path.join(outputDir, "data");-- // Check if source is already in the output documents folder (skip copy in this case)- const resolvedSource = path.resolve(sourceDocsDir);- const resolvedDocuments = path.resolve(documentsDir);- const skipCopy = resolvedSource === resolvedDocuments;-- console.log("LiteParse Dataset Generator");- console.log("===========================");- console.log(`Source: ${sourceDocsDir}`);- console.log(`Output: ${outputDir}`);- if (skipCopy) {- console.log(`(Source is output documents dir - skipping copy)`);- }- console.log();-- // Create output directories- await fs.mkdir(documentsDir, { recursive: true });-- // Find all processable files- console.log("Finding files...");- const files = await findFiles(sourceDocsDir);- console.log(`Found ${files.length} files to process\n`);-- // Process each file and copy to documents folder (unless source is already there)- const allRows: DatasetRow[] = [];- const copiedDocs = new Set<string>();-- for (const file of files) {- const rows = await processFile(file, sourceDocsDir);- allRows.push(...rows);-- // Copy document file to dataset (only once per document, skip if source is output)- const relativePath = path.relative(sourceDocsDir, file);- if (!skipCopy && !copiedDocs.has(relativePath)) {- const destPath = path.join(documentsDir, relativePath);- await fs.mkdir(path.dirname(destPath), { recursive: true });- await fs.copyFile(file, destPath);- copiedDocs.add(relativePath);- } else if (skipCopy) {- copiedDocs.add(relativePath);- }- }-- // Validate we actually produced entries- if (allRows.length === 0) {- console.error("\nERROR: No dataset entries were generated. Check that source documents exist and are parseable.");- process.exit(1);- }-- // Write metadata.jsonl- const metadataPath = path.join(outputDir, "metadata.jsonl");- const metadataContent = allRows.map((row) => JSON.stringify(row)).join("\n");- await fs.writeFile(metadataPath, metadataContent);-- console.log();- console.log("Dataset generation complete!");- console.log(` Total entries: ${allRows.length}`);- console.log(` Documents copied: ${copiedDocs.size}`);- console.log(` Metadata: ${metadataPath}`);- console.log(` Documents: ${documentsDir}`);- console.log();- console.log("Use compare-dataset.ts to compare future output against this baseline.");-}--main().catch((error) => {- console.error("Fatal error:", error);- process.exit(1);-});diff --git a/scripts/upload-dataset.sh b/scripts/upload-dataset.shnew file mode 100755index 0000000..511b6ef--- /dev/null+++ b/scripts/upload-dataset.sh@@ -0,0 +1,49 @@+#!/usr/bin/env bash+# Regenerates and uploads the dataset to HuggingFace+#+# Usage:+# HF_TOKEN=xxx ./scripts/upload-dataset.sh [dataset-dir] [repo-name]+#+# Arguments:+# dataset-dir - Directory containing the dataset with data/ subfolder (default: ./dataset)+# repo-name - HuggingFace repository name (default: llamaindex/liteparse_cicd_data)+#+# Environment variables:+# HF_TOKEN - HuggingFace API token with write access+#+# Requires: hf cli (pip install huggingface_hub)+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"++DATASET_DIR="${1:-$REPO_ROOT/dataset}"+REPO_NAME="${2:-llamaindex/liteparse_cicd_data}"++if [ -z "${HF_TOKEN:-}" ]; then+ echo "Error: HF_TOKEN environment variable is required"+ echo "Get a token from https://huggingface.co/settings/tokens"+ exit 1+fi++DOCUMENTS_DIR="$DATASET_DIR/data"++echo "LiteParse Dataset Upload"+echo "========================"+echo "Dataset: $DATASET_DIR"+echo "Documents: $DOCUMENTS_DIR"+echo "Repo: $REPO_NAME"+echo++# Step 1: Regenerate dataset from documents in the dataset directory+echo "Step 1: Regenerating dataset from existing documents..."+"$SCRIPT_DIR/create-dataset.sh" "$DATASET_DIR" "$DOCUMENTS_DIR"++# Step 2: Upload to HuggingFace+echo+echo "Step 2: Uploading to HuggingFace..."+hf upload "$REPO_NAME" "$DATASET_DIR" --repo-type dataset --token "$HF_TOKEN"++echo+echo "✓ Dataset uploaded successfully!"+echo " View at: https://huggingface.co/datasets/$REPO_NAME"diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.tsdeleted file mode 100644index 839959b..0000000--- a/scripts/upload-dataset.ts+++ /dev/null@@ -1,83 +0,0 @@-/**- * Regenerates and uploads the dataset to HuggingFace- *- * Usage:- * HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]- *- * Arguments:- * dataset-dir - Directory containing the dataset with data/ subfolder- * (default: ./dataset)- * repo-name - HuggingFace repository name (default: llamaindex/liteparse_cicd_data)- *- * Environment variables:- * HF_TOKEN - HuggingFace API token with write access- *- * This script:- * 1. Regenerates metadata.jsonl by re-running liteparse on data/ in the dataset- * 2. Uploads to HuggingFace using the huggingface_hub API- */--import { execSync } from "child_process";-import * as path from "path";--const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");-const DEFAULT_REPO = "llamaindex/liteparse_cicd_data";--async function main() {- const datasetDir = process.argv[2] || DEFAULT_OUTPUT_DIR;- const repoName = process.argv[3] || DEFAULT_REPO;- const hfToken = process.env.HF_TOKEN;-- if (!hfToken) {- console.error("Error: HF_TOKEN environment variable is required");- console.error("Get a token from https://huggingface.co/settings/tokens");- process.exit(1);- }-- const documentsDir = path.join(datasetDir, "data");-- console.log("LiteParse Dataset Upload");- console.log("========================");- console.log(`Dataset: ${datasetDir}`);- console.log(`Documents: ${documentsDir}`);- console.log(`Repo: ${repoName}`);- console.log();-- // Step 1: Regenerate dataset from documents in the dataset directory- console.log("Step 1: Regenerating dataset from existing documents...");- try {- // Pass both output dir and source docs dir (data/ within the dataset)- execSync(`npx tsx scripts/create-dataset.ts "${datasetDir}" "${documentsDir}"`, {- cwd: path.join(import.meta.dirname, ".."),- stdio: "inherit",- });- } catch (error) {- console.error("Failed to generate dataset");- process.exit(1);- }-- // Step 2: Upload to HuggingFace- console.log("\nStep 2: Uploading to HuggingFace...");- try {- // Use hf cli to upload- // This requires hf cli to be installed: brew install huggingface-cli- execSync(- `hf upload ${repoName} "${datasetDir}" --repo-type dataset --token "${hfToken}"`,- {- cwd: path.join(import.meta.dirname, ".."),- stdio: "inherit",- }- );- console.log("\n✓ Dataset uploaded successfully!");- console.log(` View at: https://huggingface.co/datasets/${repoName}`);- } catch (error) {- console.error("Failed to upload to HuggingFace");- console.error("Make sure hf cli is installed: brew install huggingface-cli");- process.exit(1);- }-}--main().catch((error) => {- console.error("Fatal error:", error);- process.exit(1);-});