Skip to content

A fast, helpful, and open-source document parser

fix e2e dataset flow

Logan Markewich13122d36e521
diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.yml
index dac9782..5cfa00b 100644
--- a/.github/workflows/e2e-output.yml
+++ b/.github/workflows/e2e-output.yml
@@ -128,6 +128,12 @@ jobs:
- name: Build
run: npm run build
+ - name: Download existing dataset from HuggingFace
+ env:
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
+ run: |
+ hf download 'llamaindex/liteparse_cicd_data' --local-dir expected-dataset --repo-type dataset --token ${{ secrets.HF_TOKEN }}
+
- name: Generate and upload dataset
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
@@ -136,4 +142,4 @@ jobs:
echo "Warning: HF_TOKEN not set, skipping upload"
exit 0
fi
- npx tsx scripts/upload-dataset.ts dataset ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}
+ npx tsx scripts/upload-dataset.ts expected-dataset ${{ vars.HF_DATASET_REPO || 'liteparse/e2e-test-outputs' }}
diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.ts
index 4971139..0848fa6 100644
--- a/scripts/create-dataset.ts
+++ b/scripts/create-dataset.ts
@@ -1,5 +1,5 @@
/**
- * Creates a dataset from e2e-test-docs for regression testing
+ * Creates a dataset for regression testing
*
* Output structure:
* dataset/
@@ -10,7 +10,11 @@
* metadata.jsonl (each line: {"file_name": "documents/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})
*
* Usage:
- * npx tsx scripts/create-dataset.ts [output-dir]
+ * npx tsx scripts/create-dataset.ts [output-dir] [source-docs-dir]
+ *
+ * Arguments:
+ * output-dir - Where to write the dataset (default: ./dataset)
+ * source-docs-dir - Where to read source documents from (default: ./e2e-test-docs)
*
* The dataset can be used with compare-dataset.ts to detect output changes.
*/
@@ -19,7 +23,7 @@ import { LiteParse } from "../src/lib.js";
import * as fs from "fs/promises";
import * as path from "path";
-const E2E_DOCS_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");
+const DEFAULT_SOURCE_DIR = path.join(import.meta.dirname, "..", "e2e-test-docs");
const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");
interface DatasetRow {
@@ -112,12 +116,21 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Datase
async function main() {
const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;
+ const sourceDocsDir = process.argv[3] || DEFAULT_SOURCE_DIR;
const documentsDir = path.join(outputDir, "documents");
+ // Check if source is already in the output documents folder (skip copy in this case)
+ const resolvedSource = path.resolve(sourceDocsDir);
+ const resolvedDocuments = path.resolve(documentsDir);
+ const skipCopy = resolvedSource === resolvedDocuments;
+
console.log("LiteParse Dataset Generator");
console.log("===========================");
- console.log(`Source: ${E2E_DOCS_DIR}`);
+ console.log(`Source: ${sourceDocsDir}`);
console.log(`Output: ${outputDir}`);
+ if (skipCopy) {
+ console.log(`(Source is output documents dir - skipping copy)`);
+ }
console.log();
// Create output directories
@@ -125,24 +138,26 @@ async function main() {
// Find all processable files
console.log("Finding files...");
- const files = await findFiles(E2E_DOCS_DIR);
+ const files = await findFiles(sourceDocsDir);
console.log(`Found ${files.length} files to process\n`);
- // Process each file and copy to documents folder
+ // Process each file and copy to documents folder (unless source is already there)
const allRows: DatasetRow[] = [];
const copiedDocs = new Set<string>();
for (const file of files) {
- const rows = await processFile(file, E2E_DOCS_DIR);
+ const rows = await processFile(file, sourceDocsDir);
allRows.push(...rows);
- // Copy document file to dataset (only once per document)
- const relativePath = path.relative(E2E_DOCS_DIR, file);
- if (!copiedDocs.has(relativePath)) {
+ // Copy document file to dataset (only once per document, skip if source is output)
+ const relativePath = path.relative(sourceDocsDir, file);
+ if (!skipCopy && !copiedDocs.has(relativePath)) {
const destPath = path.join(documentsDir, relativePath);
await fs.mkdir(path.dirname(destPath), { recursive: true });
await fs.copyFile(file, destPath);
copiedDocs.add(relativePath);
+ } else if (skipCopy) {
+ copiedDocs.add(relativePath);
}
}
diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.ts
index cf850a4..a0e9b6d 100644
--- a/scripts/upload-dataset.ts
+++ b/scripts/upload-dataset.ts
@@ -1,14 +1,19 @@
/**
- * Uploads/updates the dataset to HuggingFace
+ * Regenerates and uploads the dataset to HuggingFace
*
* Usage:
* HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]
*
+ * Arguments:
+ * dataset-dir - Directory containing the dataset with documents/ subfolder
+ * (default: ./dataset)
+ * repo-name - HuggingFace repository name (default: liteparse/e2e-test-outputs)
+ *
* Environment variables:
* HF_TOKEN - HuggingFace API token with write access
*
* This script:
- * 1. Regenerates the dataset from e2e-test-docs
+ * 1. Regenerates metadata.jsonl by re-running liteparse on documents/ in the dataset
* 2. Uploads to HuggingFace using the huggingface_hub API
*/
@@ -19,7 +24,7 @@ const DEFAULT_OUTPUT_DIR = path.join(import.meta.dirname, "..", "dataset");
const DEFAULT_REPO = "liteparse/e2e-test-outputs";
async function main() {
- const outputDir = process.argv[2] || DEFAULT_OUTPUT_DIR;
+ const datasetDir = process.argv[2] || DEFAULT_OUTPUT_DIR;
const repoName = process.argv[3] || DEFAULT_REPO;
const hfToken = process.env.HF_TOKEN;
@@ -29,16 +34,20 @@ async function main() {
process.exit(1);
}
+ const documentsDir = path.join(datasetDir, "documents");
+
console.log("LiteParse Dataset Upload");
console.log("========================");
- console.log(`Output: ${outputDir}`);
+ console.log(`Dataset: ${datasetDir}`);
+ console.log(`Documents: ${documentsDir}`);
console.log(`Repo: ${repoName}`);
console.log();
- // Step 1: Regenerate dataset
- console.log("Step 1: Regenerating dataset...");
+ // Step 1: Regenerate dataset from documents in the dataset directory
+ console.log("Step 1: Regenerating dataset from existing documents...");
try {
- execSync(`npx tsx scripts/create-dataset.ts "${outputDir}"`, {
+ // Pass both output dir and source docs dir (documents/ within the dataset)
+ execSync(`npx tsx scripts/create-dataset.ts "${datasetDir}" "${documentsDir}"`, {
cwd: path.join(import.meta.dirname, ".."),
stdio: "inherit",
});
@@ -53,7 +62,7 @@ async function main() {
// Use hf cli to upload
// This requires hf cli to be installed: brew install huggingface-cli
execSync(
- `hf upload ${repoName} "${outputDir}" --repo-type dataset --token "${hfToken}"`,
+ `hf upload ${repoName} "${datasetDir}" --repo-type dataset --token "${hfToken}"`,
{
cwd: path.join(import.meta.dirname, ".."),
stdio: "inherit",