A fast, helpful, and open-source document parser
add regression detection to CICD
Logan Markewich781e442f47c4
diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.ymlnew file mode 100644index 0000000..2873730--- /dev/null+++ b/.github/workflows/e2e-output.yml@@ -0,0 +1,153 @@+name: E2E Output Validation++on:+ pull_request:+ branches: [main]+ push:+ branches: [main]++# For PRs, we need write permissions to add labels+permissions:+ contents: read+ pull-requests: write++jobs:+ compare-outputs:+ runs-on: ubuntu-latest+ outputs:+ has_changes: ${{ steps.compare.outputs.has_changes }}++ steps:+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Install system dependencies+ run: |+ sudo apt-get update+ sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr++ - name: Setup Node.js+ uses: actions/setup-node@v4+ with:+ node-version: 20+ cache: "npm"++ - name: Install dependencies+ run: npm ci++ - name: Build+ run: npm run build++ - name: Download expected dataset+ uses: actions/checkout@v4+ with:+ repository: ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}+ path: expected-dataset+ # Use a GitHub mirror of the HF dataset, or download from HF directly+ continue-on-error: true++ - name: Download from HuggingFace (fallback)+ if: failure()+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN }}+ run: |+ pip install huggingface_hub+ hf download ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }} --local-dir expected-dataset --repo-type dataset --token ${{ secrets.HF_TOKEN }}+ continue-on-error: true++ - name: Compare outputs+ id: compare+ run: |+ # Run comparison script+ set +e+ npx tsx scripts/compare-dataset.ts expected-dataset > comparison-output.txt 2>&1+ EXIT_CODE=$?+ set -e++ cat comparison-output.txt++ if [ $EXIT_CODE -eq 0 ]; then+ echo "has_changes=false" >> $GITHUB_OUTPUT+ echo "✓ No output changes detected"+ elif [ $EXIT_CODE -eq 1 ]; then+ echo "has_changes=true" >> $GITHUB_OUTPUT+ echo "⚠ Output changes detected - requires approval"+ else+ echo "has_changes=error" >> $GITHUB_OUTPUT+ echo "✗ Error running comparison"+ exit 1+ fi++ - name: Upload comparison results+ if: always()+ uses: actions/upload-artifact@v4+ with:+ name: comparison-results+ path: comparison-output.txt++ - name: Add label for changed outputs (PR only)+ if: github.event_name == 'pull_request' && steps.compare.outputs.has_changes == 'true'+ uses: actions/github-script@v7+ with:+ script: |+ github.rest.issues.addLabels({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ labels: ['output-changed']+ })++ - name: Require approval for output changes+ if: steps.compare.outputs.has_changes == 'true'+ run: |+ echo "::warning::This PR changes liteparse output. Please review the comparison results."+ echo ""+ echo "To approve these changes:"+ echo "1. Review the comparison output in the artifacts"+ echo "2. Add the 'output-approved' label to this PR"+ echo "3. Re-run this workflow"+ exit 1++ # This job only runs on main branch after merge+ upload-dataset:+ runs-on: ubuntu-latest+ needs: compare-outputs+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'++ steps:+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Install system dependencies+ run: |+ sudo apt-get update+ sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr++ - name: Setup Node.js+ uses: actions/setup-node@v4+ with:+ node-version: 20+ cache: "npm"++ - name: Setup Python+ uses: actions/setup-python@v5+ with:+ python-version: "3.11"++ - name: Install dependencies+ run: |+ npm ci+ pip install huggingface_hub++ - name: Build+ run: npm run build++ - name: Generate and upload dataset+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN }}+ run: |+ if [ -z "$HF_TOKEN" ]; then+ echo "Warning: HF_TOKEN not set, skipping upload"+ exit 0+ fi+ npx tsx scripts/upload-dataset.ts dataset ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}diff --git a/scripts/compare-dataset.ts b/scripts/compare-dataset.tsnew file mode 100644index 0000000..31b870b--- /dev/null+++ b/scripts/compare-dataset.ts@@ -0,0 +1,271 @@+/**+ * Compares current liteparse output against a baseline dataset+ *+ * Usage:+ * npx tsx scripts/compare-dataset.ts [dataset-dir]+ *+ * The dataset should be created using create-dataset.ts and contains:+ * - documents/: The original document files+ * - metadata.jsonl: Expected outputs for each document/page+ *+ * Exit codes:+ * 0 - No changes detected+ * 1 - Changes detected (requires approval)+ * 2 - Error occurred+ *+ * For CI/CD, this script can be run to detect if a PR changes output.+ * If changes are detected, the PR should require manual approval.+ */++import { LiteParse } from "../src/lib.js";+import * as fs from "fs/promises";+import * as path from "path";++const DEFAULT_DATASET_DIR = path.join(import.meta.dirname, "..", "dataset");++interface DatasetRow {+ file_name: string;+ document: string;+ page: number;+ output_text: string;+ output_json: object;+}++interface DiffResult {+ document: string;+ page: number;+ type: "added" | "removed" | "changed";+ expected?: string;+ actual?: string;+ diff?: string;+}++async function loadDataset(datasetDir: string): Promise<Map<string, DatasetRow>> {+ const metadataPath = path.join(datasetDir, "metadata.jsonl");+ const content = await fs.readFile(metadataPath, "utf-8");+ const rows = content+ .split("\n")+ .filter((line) => line.trim())+ .map((line) => JSON.parse(line) as DatasetRow);++ const map = new Map<string, DatasetRow>();+ for (const row of rows) {+ const key = `${row.document}:${row.page}`;+ map.set(key, row);+ }++ return map;+}++function computeTextDiff(expected: string, actual: string): string {+ // Simple line-by-line diff+ const expectedLines = expected.split("\n");+ const actualLines = actual.split("\n");+ const diff: string[] = [];++ const maxLines = Math.max(expectedLines.length, actualLines.length);+ for (let i = 0; i < maxLines; i++) {+ const exp = expectedLines[i] ?? "";+ const act = actualLines[i] ?? "";++ if (exp !== act) {+ if (exp && !act) {+ diff.push(`- ${exp}`);+ } else if (!exp && act) {+ diff.push(`+ ${act}`);+ } else {+ diff.push(`- ${exp}`);+ diff.push(`+ ${act}`);+ }+ }+ }++ return diff.slice(0, 20).join("\n") + (diff.length > 20 ? `\n... (${diff.length - 20} more)` : "");+}++async function getCurrentOutput(+ filePath: string+): Promise<Map<number, { text: string; json: object }>> {+ const parser = new LiteParse({+ outputFormat: "json",+ ocrEnabled: false,+ preciseBoundingBox: true,+ });++ const result = await parser.parse(filePath, true);+ const outputs = new Map<number, { text: string; json: object }>();++ if (result.pages.length === 0 && result.text) {+ // Text-only result+ outputs.set(1, { text: result.text, json: { text: result.text } });+ } else {+ for (const page of result.pages) {+ const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);+ outputs.set(page.pageNum, {+ text: page.text,+ json: jsonPage || { page: page.pageNum, text: page.text },+ });+ }+ }++ return outputs;+}++async function main() {+ const datasetDir = process.argv[2] || DEFAULT_DATASET_DIR;+ const documentsDir = path.join(datasetDir, "documents");++ console.log("LiteParse Dataset Comparison");+ console.log("============================");+ console.log(`Dataset: ${datasetDir}`);+ console.log(`Documents: ${documentsDir}`);+ console.log();++ // Load expected dataset+ let expected: Map<string, DatasetRow>;+ try {+ expected = await loadDataset(datasetDir);+ console.log(`Loaded ${expected.size} expected entries\n`);+ } catch (error) {+ console.error(`Failed to load dataset: ${error}`);+ process.exit(2);+ }++ const diffs: DiffResult[] = [];+ const processedKeys = new Set<string>();++ // Group expected entries by document+ const docEntries = new Map<string, DatasetRow[]>();+ for (const row of expected.values()) {+ const existing = docEntries.get(row.document) || [];+ existing.push(row);+ docEntries.set(row.document, existing);+ }++ // Process each document+ for (const [document, rows] of docEntries) {+ const filePath = path.join(documentsDir, document);++ // Check if file still exists+ try {+ await fs.access(filePath);+ } catch {+ // File removed+ for (const row of rows) {+ const key = `${row.document}:${row.page}`;+ processedKeys.add(key);+ diffs.push({+ document: row.document,+ page: row.page,+ type: "removed",+ expected: row.output_text,+ });+ }+ continue;+ }++ console.log(`Checking: ${document}`);++ try {+ const currentOutputs = await getCurrentOutput(filePath);++ // Compare each page+ for (const row of rows) {+ const key = `${row.document}:${row.page}`;+ processedKeys.add(key);++ const current = currentOutputs.get(row.page);+ if (!current) {+ diffs.push({+ document: row.document,+ page: row.page,+ type: "removed",+ expected: row.output_text,+ });+ continue;+ }++ // Compare text output (normalize whitespace for comparison)+ const expectedText = row.output_text.trim();+ const actualText = current.text.trim();++ if (expectedText !== actualText) {+ diffs.push({+ document: row.document,+ page: row.page,+ type: "changed",+ expected: expectedText,+ actual: actualText,+ diff: computeTextDiff(expectedText, actualText),+ });+ }++ currentOutputs.delete(row.page);+ }++ // Check for new pages+ for (const [pageNum, output] of currentOutputs) {+ diffs.push({+ document,+ page: pageNum,+ type: "added",+ actual: output.text,+ });+ }+ } catch (error) {+ console.error(` ERROR: ${error instanceof Error ? error.message : error}`);+ // Check if it was an expected error+ const errorRow = rows.find(+ (r) => (r.output_json as { error?: boolean }).error === true+ );+ if (!errorRow) {+ diffs.push({+ document,+ page: 0,+ type: "changed",+ expected: "successful parse",+ actual: `error: ${error instanceof Error ? error.message : error}`,+ });+ }+ for (const row of rows) {+ processedKeys.add(`${row.document}:${row.page}`);+ }+ }+ }++ // Report results+ console.log();+ console.log("Results");+ console.log("-------");++ if (diffs.length === 0) {+ console.log("✓ No changes detected");+ process.exit(0);+ }++ console.log(`✗ ${diffs.length} change(s) detected:\n`);++ for (const diff of diffs) {+ console.log(`[${diff.type.toUpperCase()}] ${diff.document} (page ${diff.page})`);+ if (diff.diff) {+ console.log(diff.diff);+ }+ console.log();+ }++ // Output summary for CI+ console.log("---");+ console.log("SUMMARY:");+ console.log(` Added: ${diffs.filter((d) => d.type === "added").length}`);+ console.log(` Removed: ${diffs.filter((d) => d.type === "removed").length}`);+ console.log(` Changed: ${diffs.filter((d) => d.type === "changed").length}`);+ console.log();+ console.log("This PR changes liteparse output and requires manual approval.");++ process.exit(1);+}++main().catch((error) => {+ console.error("Fatal error:", error);+ process.exit(2);+});diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.tsnew file mode 100644index 0000000..4971139--- /dev/null+++ b/scripts/create-dataset.ts@@ -0,0 +1,167 @@+/**+ * Creates a dataset from e2e-test-docs for regression testing+ *+ * Output structure:+ * dataset/+ * documents/+ * doc1.pdf+ * doc2.docx+ * ...+ * metadata.jsonl (each line: {"file_name": "documents/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})+ *+ * Usage:+ * npx tsx scripts/create-dataset.ts [output-dir]+ *+ * The dataset can be used with compare-dataset.ts to detect output changes.+ */++import { LiteParse } from "../src/lib.js";+import * as fs from "fs/promises";+import * as path from "path";++const E2E_DOCS_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");+const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");++interface DatasetRow {+ file_name: string;+ document: string;+ page: number;+ output_text: string;+ output_json: object;+}++async function findFiles(dir: string, baseDir: string = dir): Promise<string[]> {+ const files: string[] = [];+ const entries = await fs.readdir(dir, { withFileTypes: true });++ for (const entry of entries) {+ const fullPath = path.join(dir, entry.name);++ if (entry.isDirectory()) {+ // Recursively search subdirectories+ const subFiles = await findFiles(fullPath, baseDir);+ files.push(...subFiles);+ } else if (entry.isFile()) {+ files.push(fullPath);+ }+ }++ return files;+}++async function processFile(filePath: string, baseDocDir: string): Promise<DatasetRow[]> {+ const relativePath = path.relative(baseDocDir, filePath);++ console.log(`Processing: ${relativePath}`);++ const rows: DatasetRow[] = [];++ try {+ // Parse the document+ const parser = new LiteParse({+ outputFormat: "json",+ ocrEnabled: false, // Disable OCR for deterministic results+ preciseBoundingBox: true,+ });++ const result = await parser.parse(filePath, true);++ // For non-PDF files that return text directly (no pages)+ if (result.pages.length === 0 && result.text) {+ rows.push({+ file_name: `documents/${relativePath}`,+ document: relativePath,+ page: 1,+ output_text: result.text,+ output_json: { text: result.text },+ });+ console.log(` -> 1 text entry`);+ return rows;+ }++ // Create entries for each page+ for (const page of result.pages) {+ const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);+ rows.push({+ file_name: `documents/${relativePath}`,+ document: relativePath,+ page: page.pageNum,+ output_text: page.text,+ output_json: jsonPage || { page: page.pageNum, text: page.text },+ });+ }++ console.log(` -> ${rows.length} pages`);+ } catch (error) {+ console.error(` ERROR: ${error instanceof Error ? error.message : error}`);+ // Record the error as a dataset entry (blank result)+ rows.push({+ file_name: `documents/${relativePath}`,+ document: relativePath,+ page: 0,+ output_text: "",+ output_json: {+ error: true,+ message: error instanceof Error ? error.message : String(error),+ },+ });+ }++ return rows;+}++async function main() {+ const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;+ const documentsDir = path.join(outputDir, "documents");++ console.log("LiteParse Dataset Generator");+ console.log("===========================");+ console.log(`Source: ${E2E_DOCS_DIR}`);+ console.log(`Output: ${outputDir}`);+ console.log();++ // Create output directories+ await fs.mkdir(documentsDir, { recursive: true });++ // Find all processable files+ console.log("Finding files...");+ const files = await findFiles(E2E_DOCS_DIR);+ console.log(`Found ${files.length} files to process\n`);++ // Process each file and copy to documents folder+ const allRows: DatasetRow[] = [];+ const copiedDocs = new Set<string>();++ for (const file of files) {+ const rows = await processFile(file, E2E_DOCS_DIR);+ allRows.push(...rows);++ // Copy document file to dataset (only once per document)+ const relativePath = path.relative(E2E_DOCS_DIR, file);+ if (!copiedDocs.has(relativePath)) {+ const destPath = path.join(documentsDir, relativePath);+ await fs.mkdir(path.dirname(destPath), { recursive: true });+ await fs.copyFile(file, destPath);+ copiedDocs.add(relativePath);+ }+ }++ // Write metadata.jsonl+ const metadataPath = path.join(outputDir, "metadata.jsonl");+ const metadataContent = allRows.map((row) => JSON.stringify(row)).join("\n");+ await fs.writeFile(metadataPath, metadataContent);++ console.log();+ console.log("Dataset generation complete!");+ console.log(` Total entries: ${allRows.length}`);+ console.log(` Documents copied: ${copiedDocs.size}`);+ console.log(` Metadata: ${metadataPath}`);+ console.log(` Documents: ${documentsDir}`);+ console.log();+ console.log("Use compare-dataset.ts to compare future output against this baseline.");+}++main().catch((error) => {+ console.error("Fatal error:", error);+ process.exit(1);+});diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.tsnew file mode 100644index 0000000..cf850a4--- /dev/null+++ b/scripts/upload-dataset.ts@@ -0,0 +1,74 @@+/**+ * Uploads/updates the dataset to HuggingFace+ *+ * Usage:+ * HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]+ *+ * Environment variables:+ * HF_TOKEN - HuggingFace API token with write access+ *+ * This script:+ * 1. Regenerates the dataset from e2e-test-docs+ * 2. Uploads to HuggingFace using the huggingface_hub API+ */++import { execSync } from "child_process";+import * as path from "path";++const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");+const DEFAULT_REPO = "liteparse/e2e-test-outputs";++async function main() {+ const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;+ const repoName = process.argv[3] || DEFAULT_REPO;+ const hfToken = process.env.HF_TOKEN;++ if (!hfToken) {+ console.error("Error: HF_TOKEN environment variable is required");+ console.error("Get a token from https://huggingface.co/settings/tokens");+ process.exit(1);+ }++ console.log("LiteParse Dataset Upload");+ console.log("========================");+ console.log(`Output: ${outputDir}`);+ console.log(`Repo: ${repoName}`);+ console.log();++ // Step 1: Regenerate dataset+ console.log("Step 1: Regenerating dataset...");+ try {+ execSync(`npx tsx scripts/create-dataset.ts "${outputDir}"`, {+ cwd: path.join(import.meta.dirname, ".."),+ stdio: "inherit",+ });+ } catch (error) {+ console.error("Failed to generate dataset");+ process.exit(1);+ }++ // Step 2: Upload to HuggingFace+ console.log("\nStep 2: Uploading to HuggingFace...");+ try {+ // Use hf cli to upload+ // This requires hf cli to be installed: brew install huggingface-cli+ execSync(+ `hf upload ${repoName} "${outputDir}" --repo-type dataset --token "${hfToken}"`,+ {+ cwd: path.join(import.meta.dirname, ".."),+ stdio: "inherit",+ }+ );+ console.log("\n✓ Dataset uploaded successfully!");+ console.log(` View at: https://huggingface.co/datasets/${repoName}`);+ } catch (error) {+ console.error("Failed to upload to HuggingFace");+ console.error("Make sure hf cli is installed: brew install huggingface-cli");+ process.exit(1);+ }+}++main().catch((error) => {+ console.error("Fatal error:", error);+ process.exit(1);+});