A fast, helpful, and open-source document parser
fix e2e dataset flow
Logan Markewich13122d36e521
diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.ymlindex dac9782..5cfa00b 100644--- a/.github/workflows/e2e-output.yml+++ b/.github/workflows/e2e-output.yml@@ -128,6 +128,12 @@ jobs:- name: Buildrun: npm run build+ - name: Download existing dataset from HuggingFace+ env:+ HF_TOKEN: ${{ secrets.HF_TOKEN }}+ run: |+ hf download 'llamaindex/liteparse_cicd_data' --local-dir expected-dataset --repo-type dataset --token ${{ secrets.HF_TOKEN }}+- name: Generate and upload datasetenv:HF_TOKEN: ${{ secrets.HF_TOKEN }}@@ -136,4 +142,4 @@ jobs:echo "Warning: HF_TOKEN not set, skipping upload"exit 0fi- npx tsx scripts/upload-dataset.ts dataset ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}+ npx tsx scripts/upload-dataset.ts expected-dataset ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.tsindex 4971139..0848fa6 100644--- a/scripts/create-dataset.ts+++ b/scripts/create-dataset.ts@@ -1,5 +1,5 @@/**- * Creates a dataset from e2e-test-docs for regression testing+ * Creates a dataset for regression testing** Output structure:* dataset/@@ -10,7 +10,11 @@* metadata.jsonl (each line: {"file_name": "documents/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})** Usage:- * npx tsx scripts/create-dataset.ts [output-dir]+ * npx tsx scripts/create-dataset.ts [output-dir] [source-docs-dir]+ *+ * Arguments:+ * output-dir - Where to write the dataset (default: ./dataset)+ * source-docs-dir - Where to read source documents from (default: ./e2e-test-docs)** The dataset can be used with compare-dataset.ts to detect output changes.*/@@ -19,7 +23,7 @@ import { LiteParse } from "../src/lib.js";import * as fs from "fs/promises";import * as path from "path";-const E2E_DOCS_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");+const DEFAULT_SOURCE_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");interface DatasetRow {@@ -112,12 +116,21 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Dataseasync function main() {const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;+ const sourceDocsDir = process.argv[3] || DEFAULT_SOURCE_DIR;const documentsDir = path.join(outputDir, "documents");+ // Check if source is already in the output documents folder (skip copy in this case)+ const resolvedSource = path.resolve(sourceDocsDir);+ const resolvedDocuments = path.resolve(documentsDir);+ const skipCopy = resolvedSource === resolvedDocuments;+console.log("LiteParse Dataset Generator");console.log("===========================");- console.log(`Source: ${E2E_DOCS_DIR}`);+ console.log(`Source: ${sourceDocsDir}`);console.log(`Output: ${outputDir}`);+ if (skipCopy) {+ console.log(`(Source is output documents dir - skipping copy)`);+ }console.log();// Create output directories@@ -125,24 +138,26 @@ async function main() {// Find all processable filesconsole.log("Finding files...");- const files = await findFiles(E2E_DOCS_DIR);+ const files = await findFiles(sourceDocsDir);console.log(`Found ${files.length} files to process\n`);- // Process each file and copy to documents folder+ // Process each file and copy to documents folder (unless source is already there)const allRows: DatasetRow[] = [];const copiedDocs = new Set<string>();for (const file of files) {- const rows = await processFile(file, E2E_DOCS_DIR);+ const rows = await processFile(file, sourceDocsDir);allRows.push(...rows);- // Copy document file to dataset (only once per document)- const relativePath = path.relative(E2E_DOCS_DIR, file);- if (!copiedDocs.has(relativePath)) {+ // Copy document file to dataset (only once per document, skip if source is output)+ const relativePath = path.relative(sourceDocsDir, file);+ if (!skipCopy && !copiedDocs.has(relativePath)) {const destPath = path.join(documentsDir, relativePath);await fs.mkdir(path.dirname(destPath), { recursive: true });await fs.copyFile(file, destPath);copiedDocs.add(relativePath);+ } else if (skipCopy) {+ copiedDocs.add(relativePath);}}diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.tsindex cf850a4..a0e9b6d 100644--- a/scripts/upload-dataset.ts+++ b/scripts/upload-dataset.ts@@ -1,14 +1,19 @@/**- * Uploads/updates the dataset to HuggingFace+ * Regenerates and uploads the dataset to HuggingFace** Usage:* HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]*+ * Arguments:+ * dataset-dir - Directory containing the dataset with documents/ subfolder+ * (default: ./dataset)+ * repo-name - HuggingFace repository name (default: liteparse/e2e-test-outputs)+ ** Environment variables:* HF_TOKEN - HuggingFace API token with write access** This script:- * 1. Regenerates the dataset from e2e-test-docs+ * 1. Regenerates metadata.jsonl by re-running liteparse on documents/ in the dataset* 2. Uploads to HuggingFace using the huggingface_hub API*/@@ -19,7 +24,7 @@ const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");const DEFAULT_REPO = "liteparse/e2e-test-outputs";async function main() {- const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;+ const datasetDir = process.argv[2] || DEFAULT_OUTPUT_DIR;const repoName = process.argv[3] || DEFAULT_REPO;const hfToken = process.env.HF_TOKEN;@@ -29,16 +34,20 @@ async function main() {process.exit(1);}+ const documentsDir = path.join(datasetDir, "documents");+console.log("LiteParse Dataset Upload");console.log("========================");- console.log(`Output: ${outputDir}`);+ console.log(`Dataset: ${datasetDir}`);+ console.log(`Documents: ${documentsDir}`);console.log(`Repo: ${repoName}`);console.log();- // Step 1: Regenerate dataset- console.log("Step 1: Regenerating dataset...");+ // Step 1: Regenerate dataset from documents in the dataset directory+ console.log("Step 1: Regenerating dataset from existing documents...");try {- execSync(`npx tsx scripts/create-dataset.ts "${outputDir}"`, {+ // Pass both output dir and source docs dir (documents/ within the dataset)+ execSync(`npx tsx scripts/create-dataset.ts "${datasetDir}" "${documentsDir}"`, {cwd: path.join(import.meta.dirname, ".."),stdio: "inherit",});@@ -53,7 +62,7 @@ async function main() {// Use hf cli to upload// This requires hf cli to be installed: brew install huggingface-cliexecSync(- `hf upload ${repoName} "${outputDir}" --repo-type dataset --token "${hfToken}"`,+ `hf upload ${repoName} "${datasetDir}" --repo-type dataset --token "${hfToken}"`,{cwd: path.join(import.meta.dirname, ".."),stdio: "inherit",