Skip to content

A fast, helpful, and open-source document parser

add regression detection to CICD

Logan Markewich781e442f47c4
diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.yml
new file mode 100644
index 0000000..2873730
--- /dev/null
+++ b/.github/workflows/e2e-output.yml
@@ -0,0 +1,153 @@
+name: E2E Output Validation
+
+on:
+ pull_request:
+ branches: [main]
+ push:
+ branches: [main]
+
+# For PRs, we need write permissions to add labels
+permissions:
+ contents: read
+ pull-requests: write
+
+jobs:
+ compare-outputs:
+ runs-on: ubuntu-latest
+ outputs:
+ has_changes: ${{ steps.compare.outputs.has_changes }}
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: 20
+ cache: "npm"
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Build
+ run: npm run build
+
+ - name: Download expected dataset
+ uses: actions/checkout@v4
+ with:
+ repository: ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}
+ path: expected-dataset
+ # Use a GitHub mirror of the HF dataset, or download from HF directly
+ continue-on-error: true
+
+ - name: Download from HuggingFace (fallback)
+ if: failure()
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
+ run: |
+ pip install huggingface_hub
+ hf download ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }} --local-dir expected-dataset --repo-type dataset --token ${{ secrets.HF_TOKEN }}
+ continue-on-error: true
+
+ - name: Compare outputs
+ id: compare
+ run: |
+ # Run comparison script
+ set +e
+ npx tsx scripts/compare-dataset.ts expected-dataset > comparison-output.txt 2>&1
+ EXIT_CODE=$?
+ set -e
+
+ cat comparison-output.txt
+
+ if [ $EXIT_CODE -eq 0 ]; then
+ echo "has_changes=false" >> $GITHUB_OUTPUT
+ echo "✓ No output changes detected"
+ elif [ $EXIT_CODE -eq 1 ]; then
+ echo "has_changes=true" >> $GITHUB_OUTPUT
+ echo "⚠ Output changes detected - requires approval"
+ else
+ echo "has_changes=error" >> $GITHUB_OUTPUT
+ echo "✗ Error running comparison"
+ exit 1
+ fi
+
+ - name: Upload comparison results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: comparison-results
+ path: comparison-output.txt
+
+ - name: Add label for changed outputs (PR only)
+ if: github.event_name == 'pull_request' && steps.compare.outputs.has_changes == 'true'
+ uses: actions/github-script@v7
+ with:
+ script: |
+ github.rest.issues.addLabels({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ labels: ['output-changed']
+ })
+
+ - name: Require approval for output changes
+ if: steps.compare.outputs.has_changes == 'true'
+ run: |
+ echo "::warning::This PR changes liteparse output. Please review the comparison results."
+ echo ""
+ echo "To approve these changes:"
+ echo "1. Review the comparison output in the artifacts"
+ echo "2. Add the 'output-approved' label to this PR"
+ echo "3. Re-run this workflow"
+ exit 1
+
+ # This job only runs on main branch after merge
+ upload-dataset:
+ runs-on: ubuntu-latest
+ needs: compare-outputs
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: 20
+ cache: "npm"
+
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install dependencies
+ run: |
+ npm ci
+ pip install huggingface_hub
+
+ - name: Build
+ run: npm run build
+
+ - name: Generate and upload dataset
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
+ run: |
+ if [ -z "$HF_TOKEN" ]; then
+ echo "Warning: HF_TOKEN not set, skipping upload"
+ exit 0
+ fi
+ npx tsx scripts/upload-dataset.ts dataset ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}
diff --git a/scripts/compare-dataset.ts b/scripts/compare-dataset.ts
new file mode 100644
index 0000000..31b870b
--- /dev/null
+++ b/scripts/compare-dataset.ts
@@ -0,0 +1,271 @@
+/**
+ * Compares current liteparse output against a baseline dataset
+ *
+ * Usage:
+ * npx tsx scripts/compare-dataset.ts [dataset-dir]
+ *
+ * The dataset should be created using create-dataset.ts and contains:
+ * - documents/: The original document files
+ * - metadata.jsonl: Expected outputs for each document/page
+ *
+ * Exit codes:
+ * 0 - No changes detected
+ * 1 - Changes detected (requires approval)
+ * 2 - Error occurred
+ *
+ * For CI/CD, this script can be run to detect if a PR changes output.
+ * If changes are detected, the PR should require manual approval.
+ */
+
+import { LiteParse } from "../src/lib.js";
+import * as fs from "fs/promises";
+import * as path from "path";
+
+const DEFAULT_DATASET_DIR = path.join(import.meta.dirname, "..", "dataset");
+
+interface DatasetRow {
+ file_name: string;
+ document: string;
+ page: number;
+ output_text: string;
+ output_json: object;
+}
+
+interface DiffResult {
+ document: string;
+ page: number;
+ type: "added" | "removed" | "changed";
+ expected?: string;
+ actual?: string;
+ diff?: string;
+}
+
+async function loadDataset(datasetDir: string): Promise<Map<string, DatasetRow>> {
+ const metadataPath = path.join(datasetDir, "metadata.jsonl");
+ const content = await fs.readFile(metadataPath, "utf-8");
+ const rows = content
+ .split("\n")
+ .filter((line) => line.trim())
+ .map((line) => JSON.parse(line) as DatasetRow);
+
+ const map = new Map<string, DatasetRow>();
+ for (const row of rows) {
+ const key = `${row.document}:${row.page}`;
+ map.set(key, row);
+ }
+
+ return map;
+}
+
+function computeTextDiff(expected: string, actual: string): string {
+ // Simple line-by-line diff
+ const expectedLines = expected.split("\n");
+ const actualLines = actual.split("\n");
+ const diff: string[] = [];
+
+ const maxLines = Math.max(expectedLines.length, actualLines.length);
+ for (let i = 0; i < maxLines; i++) {
+ const exp = expectedLines[i] ?? "";
+ const act = actualLines[i] ?? "";
+
+ if (exp !== act) {
+ if (exp && !act) {
+ diff.push(`- ${exp}`);
+ } else if (!exp && act) {
+ diff.push(`+ ${act}`);
+ } else {
+ diff.push(`- ${exp}`);
+ diff.push(`+ ${act}`);
+ }
+ }
+ }
+
+ return diff.slice(0, 20).join("\n") + (diff.length > 20 ? `\n... (${diff.length - 20} more)` : "");
+}
+
+async function getCurrentOutput(
+ filePath: string
+): Promise<Map<number, { text: string; json: object }>> {
+ const parser = new LiteParse({
+ outputFormat: "json",
+ ocrEnabled: false,
+ preciseBoundingBox: true,
+ });
+
+ const result = await parser.parse(filePath, true);
+ const outputs = new Map<number, { text: string; json: object }>();
+
+ if (result.pages.length === 0 && result.text) {
+ // Text-only result
+ outputs.set(1, { text: result.text, json: { text: result.text } });
+ } else {
+ for (const page of result.pages) {
+ const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);
+ outputs.set(page.pageNum, {
+ text: page.text,
+ json: jsonPage || { page: page.pageNum, text: page.text },
+ });
+ }
+ }
+
+ return outputs;
+}
+
+async function main() {
+ const datasetDir = process.argv[2] || DEFAULT_DATASET_DIR;
+ const documentsDir = path.join(datasetDir, "documents");
+
+ console.log("LiteParse Dataset Comparison");
+ console.log("============================");
+ console.log(`Dataset: ${datasetDir}`);
+ console.log(`Documents: ${documentsDir}`);
+ console.log();
+
+ // Load expected dataset
+ let expected: Map<string, DatasetRow>;
+ try {
+ expected = await loadDataset(datasetDir);
+ console.log(`Loaded ${expected.size} expected entries\n`);
+ } catch (error) {
+ console.error(`Failed to load dataset: ${error}`);
+ process.exit(2);
+ }
+
+ const diffs: DiffResult[] = [];
+ const processedKeys = new Set<string>();
+
+ // Group expected entries by document
+ const docEntries = new Map<string, DatasetRow[]>();
+ for (const row of expected.values()) {
+ const existing = docEntries.get(row.document) || [];
+ existing.push(row);
+ docEntries.set(row.document, existing);
+ }
+
+ // Process each document
+ for (const [document, rows] of docEntries) {
+ const filePath = path.join(documentsDir, document);
+
+ // Check if file still exists
+ try {
+ await fs.access(filePath);
+ } catch {
+ // File removed
+ for (const row of rows) {
+ const key = `${row.document}:${row.page}`;
+ processedKeys.add(key);
+ diffs.push({
+ document: row.document,
+ page: row.page,
+ type: "removed",
+ expected: row.output_text,
+ });
+ }
+ continue;
+ }
+
+ console.log(`Checking: ${document}`);
+
+ try {
+ const currentOutputs = await getCurrentOutput(filePath);
+
+ // Compare each page
+ for (const row of rows) {
+ const key = `${row.document}:${row.page}`;
+ processedKeys.add(key);
+
+ const current = currentOutputs.get(row.page);
+ if (!current) {
+ diffs.push({
+ document: row.document,
+ page: row.page,
+ type: "removed",
+ expected: row.output_text,
+ });
+ continue;
+ }
+
+ // Compare text output (normalize whitespace for comparison)
+ const expectedText = row.output_text.trim();
+ const actualText = current.text.trim();
+
+ if (expectedText !== actualText) {
+ diffs.push({
+ document: row.document,
+ page: row.page,
+ type: "changed",
+ expected: expectedText,
+ actual: actualText,
+ diff: computeTextDiff(expectedText, actualText),
+ });
+ }
+
+ currentOutputs.delete(row.page);
+ }
+
+ // Check for new pages
+ for (const [pageNum, output] of currentOutputs) {
+ diffs.push({
+ document,
+ page: pageNum,
+ type: "added",
+ actual: output.text,
+ });
+ }
+ } catch (error) {
+ console.error(` ERROR: ${error instanceof Error ? error.message : error}`);
+ // Check if it was an expected error
+ const errorRow = rows.find(
+ (r) => (r.output_json as { error?: boolean }).error === true
+ );
+ if (!errorRow) {
+ diffs.push({
+ document,
+ page: 0,
+ type: "changed",
+ expected: "successful parse",
+ actual: `error: ${error instanceof Error ? error.message : error}`,
+ });
+ }
+ for (const row of rows) {
+ processedKeys.add(`${row.document}:${row.page}`);
+ }
+ }
+ }
+
+ // Report results
+ console.log();
+ console.log("Results");
+ console.log("-------");
+
+ if (diffs.length === 0) {
+ console.log("✓ No changes detected");
+ process.exit(0);
+ }
+
+ console.log(`✗ ${diffs.length} change(s) detected:\n`);
+
+ for (const diff of diffs) {
+ console.log(`[${diff.type.toUpperCase()}] ${diff.document} (page ${diff.page})`);
+ if (diff.diff) {
+ console.log(diff.diff);
+ }
+ console.log();
+ }
+
+ // Output summary for CI
+ console.log("---");
+ console.log("SUMMARY:");
+ console.log(` Added: ${diffs.filter((d) => d.type === "added").length}`);
+ console.log(` Removed: ${diffs.filter((d) => d.type === "removed").length}`);
+ console.log(` Changed: ${diffs.filter((d) => d.type === "changed").length}`);
+ console.log();
+ console.log("This PR changes liteparse output and requires manual approval.");
+
+ process.exit(1);
+}
+
+main().catch((error) => {
+ console.error("Fatal error:", error);
+ process.exit(2);
+});
diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.ts
new file mode 100644
index 0000000..4971139
--- /dev/null
+++ b/scripts/create-dataset.ts
@@ -0,0 +1,167 @@
+/**
+ * Creates a dataset from e2e-test-docs for regression testing
+ *
+ * Output structure:
+ * dataset/
+ * documents/
+ * doc1.pdf
+ * doc2.docx
+ * ...
+ * metadata.jsonl (each line: {"file_name": "documents/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})
+ *
+ * Usage:
+ * npx tsx scripts/create-dataset.ts [output-dir]
+ *
+ * The dataset can be used with compare-dataset.ts to detect output changes.
+ */
+
+import { LiteParse } from "../src/lib.js";
+import * as fs from "fs/promises";
+import * as path from "path";
+
+const E2E_DOCS_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");
+const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");
+
+interface DatasetRow {
+ file_name: string;
+ document: string;
+ page: number;
+ output_text: string;
+ output_json: object;
+}
+
+async function findFiles(dir: string, baseDir: string = dir): Promise<string[]> {
+ const files: string[] = [];
+ const entries = await fs.readdir(dir, { withFileTypes: true });
+
+ for (const entry of entries) {
+ const fullPath = path.join(dir, entry.name);
+
+ if (entry.isDirectory()) {
+ // Recursively search subdirectories
+ const subFiles = await findFiles(fullPath, baseDir);
+ files.push(...subFiles);
+ } else if (entry.isFile()) {
+ files.push(fullPath);
+ }
+ }
+
+ return files;
+}
+
+async function processFile(filePath: string, baseDocDir: string): Promise<DatasetRow[]> {
+ const relativePath = path.relative(baseDocDir, filePath);
+
+ console.log(`Processing: ${relativePath}`);
+
+ const rows: DatasetRow[] = [];
+
+ try {
+ // Parse the document
+ const parser = new LiteParse({
+ outputFormat: "json",
+ ocrEnabled: false, // Disable OCR for deterministic results
+ preciseBoundingBox: true,
+ });
+
+ const result = await parser.parse(filePath, true);
+
+ // For non-PDF files that return text directly (no pages)
+ if (result.pages.length === 0 && result.text) {
+ rows.push({
+ file_name: `documents/${relativePath}`,
+ document: relativePath,
+ page: 1,
+ output_text: result.text,
+ output_json: { text: result.text },
+ });
+ console.log(` -> 1 text entry`);
+ return rows;
+ }
+
+ // Create entries for each page
+ for (const page of result.pages) {
+ const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);
+ rows.push({
+ file_name: `documents/${relativePath}`,
+ document: relativePath,
+ page: page.pageNum,
+ output_text: page.text,
+ output_json: jsonPage || { page: page.pageNum, text: page.text },
+ });
+ }
+
+ console.log(` -> ${rows.length} pages`);
+ } catch (error) {
+ console.error(` ERROR: ${error instanceof Error ? error.message : error}`);
+ // Record the error as a dataset entry (blank result)
+ rows.push({
+ file_name: `documents/${relativePath}`,
+ document: relativePath,
+ page: 0,
+ output_text: "",
+ output_json: {
+ error: true,
+ message: error instanceof Error ? error.message : String(error),
+ },
+ });
+ }
+
+ return rows;
+}
+
+async function main() {
+ const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;
+ const documentsDir = path.join(outputDir, "documents");
+
+ console.log("LiteParse Dataset Generator");
+ console.log("===========================");
+ console.log(`Source: ${E2E_DOCS_DIR}`);
+ console.log(`Output: ${outputDir}`);
+ console.log();
+
+ // Create output directories
+ await fs.mkdir(documentsDir, { recursive: true });
+
+ // Find all processable files
+ console.log("Finding files...");
+ const files = await findFiles(E2E_DOCS_DIR);
+ console.log(`Found ${files.length} files to process\n`);
+
+ // Process each file and copy to documents folder
+ const allRows: DatasetRow[] = [];
+ const copiedDocs = new Set<string>();
+
+ for (const file of files) {
+ const rows = await processFile(file, E2E_DOCS_DIR);
+ allRows.push(...rows);
+
+ // Copy document file to dataset (only once per document)
+ const relativePath = path.relative(E2E_DOCS_DIR, file);
+ if (!copiedDocs.has(relativePath)) {
+ const destPath = path.join(documentsDir, relativePath);
+ await fs.mkdir(path.dirname(destPath), { recursive: true });
+ await fs.copyFile(file, destPath);
+ copiedDocs.add(relativePath);
+ }
+ }
+
+ // Write metadata.jsonl
+ const metadataPath = path.join(outputDir, "metadata.jsonl");
+ const metadataContent = allRows.map((row) => JSON.stringify(row)).join("\n");
+ await fs.writeFile(metadataPath, metadataContent);
+
+ console.log();
+ console.log("Dataset generation complete!");
+ console.log(` Total entries: ${allRows.length}`);
+ console.log(` Documents copied: ${copiedDocs.size}`);
+ console.log(` Metadata: ${metadataPath}`);
+ console.log(` Documents: ${documentsDir}`);
+ console.log();
+ console.log("Use compare-dataset.ts to compare future output against this baseline.");
+}
+
+main().catch((error) => {
+ console.error("Fatal error:", error);
+ process.exit(1);
+});
diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.ts
new file mode 100644
index 0000000..cf850a4
--- /dev/null
+++ b/scripts/upload-dataset.ts
@@ -0,0 +1,74 @@
+/**
+ * Uploads/updates the dataset to HuggingFace
+ *
+ * Usage:
+ * HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]
+ *
+ * Environment variables:
+ * HF_TOKEN - HuggingFace API token with write access
+ *
+ * This script:
+ * 1. Regenerates the dataset from e2e-test-docs
+ * 2. Uploads to HuggingFace using the huggingface_hub API
+ */
+
+import { execSync } from "child_process";
+import * as path from "path";
+
+const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");
+const DEFAULT_REPO = "liteparse/e2e-test-outputs";
+
+async function main() {
+ const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;
+ const repoName = process.argv[3] || DEFAULT_REPO;
+ const hfToken = process.env.HF_TOKEN;
+
+ if (!hfToken) {
+ console.error("Error: HF_TOKEN environment variable is required");
+ console.error("Get a token from https://huggingface.co/settings/tokens");
+ process.exit(1);
+ }
+
+ console.log("LiteParse Dataset Upload");
+ console.log("========================");
+ console.log(`Output: ${outputDir}`);
+ console.log(`Repo: ${repoName}`);
+ console.log();
+
+ // Step 1: Regenerate dataset
+ console.log("Step 1: Regenerating dataset...");
+ try {
+ execSync(`npx tsx scripts/create-dataset.ts "${outputDir}"`, {
+ cwd: path.join(import.meta.dirname, ".."),
+ stdio: "inherit",
+ });
+ } catch (error) {
+ console.error("Failed to generate dataset");
+ process.exit(1);
+ }
+
+ // Step 2: Upload to HuggingFace
+ console.log("\nStep 2: Uploading to HuggingFace...");
+ try {
+ // Use hf cli to upload
+ // This requires hf cli to be installed: brew install huggingface-cli
+ execSync(
+ `hf upload ${repoName} "${outputDir}" --repo-type dataset --token "${hfToken}"`,
+ {
+ cwd: path.join(import.meta.dirname, ".."),
+ stdio: "inherit",
+ }
+ );
+ console.log("\n✓ Dataset uploaded successfully!");
+ console.log(` View at: https://huggingface.co/datasets/${repoName}`);
+ } catch (error) {
+ console.error("Failed to upload to HuggingFace");
+ console.error("Make sure hf cli is installed: brew install huggingface-cli");
+ process.exit(1);
+ }
+}
+
+main().catch((error) => {
+ console.error("Fatal error:", error);
+ process.exit(1);
+});