A fast, helpful, and open-source document parser
fix dataset scripts
Logan Markewich01f421e864e4
diff --git a/scripts/compare-dataset.ts b/scripts/compare-dataset.tsindex 020561f..b47b3da 100644--- a/scripts/compare-dataset.ts+++ b/scripts/compare-dataset.ts@@ -5,7 +5,7 @@* npx tsx scripts/compare-dataset.ts [dataset-dir]** The dataset should be created using create-dataset.ts and contains:- * - documents/: The original document files+ * - data/: The original document files* - metadata.jsonl: Expected outputs for each document/page** Exit codes:diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.tsindex 0848fa6..a69daa7 100644--- a/scripts/create-dataset.ts+++ b/scripts/create-dataset.ts@@ -3,11 +3,11 @@** Output structure:* dataset/- * documents/+ * data/* doc1.pdf* doc2.docx* ...- * metadata.jsonl (each line: {"file_name": "documents/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})+ * metadata.jsonl (each line: {"file_name": "data/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})** Usage:* npx tsx scripts/create-dataset.ts [output-dir] [source-docs-dir]@@ -73,7 +73,7 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Datase// For non-PDF files that return text directly (no pages)if (result.pages.length === 0 && result.text) {rows.push({- file_name: `documents/${relativePath}`,+ file_name: `data/${relativePath}`,document: relativePath,page: 1,output_text: result.text,@@ -87,7 +87,7 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Datasefor (const page of result.pages) {const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);rows.push({- file_name: `documents/${relativePath}`,+ file_name: `data/${relativePath}`,document: relativePath,page: page.pageNum,output_text: page.text,@@ -100,7 +100,7 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Dataseconsole.error(` ERROR: ${error instanceof Error ? error.message : error}`);// Record the error as a dataset entry (blank result)rows.push({- file_name: `documents/${relativePath}`,+ file_name: `data/${relativePath}`,document: relativePath,page: 0,output_text: "",diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.tsindex 09ec2ba..9739046 100644--- a/scripts/upload-dataset.ts+++ b/scripts/upload-dataset.ts@@ -5,7 +5,7 @@* HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]** Arguments:- * dataset-dir - Directory containing the dataset with documents/ subfolder+ * dataset-dir - Directory containing the dataset with data/ subfolder* (default: ./dataset)* repo-name - HuggingFace repository name (default: llamaindex/liteparse_cicd_data)*@@ -13,7 +13,7 @@* HF_TOKEN - HuggingFace API token with write access** This script:- * 1. Regenerates metadata.jsonl by re-running liteparse on documents/ in the dataset+ * 1. Regenerates metadata.jsonl by re-running liteparse on data/ in the dataset* 2. Uploads to HuggingFace using the huggingface_hub API*/@@ -46,7 +46,7 @@ async function main() {// Step 1: Regenerate dataset from documents in the dataset directoryconsole.log("Step 1: Regenerating dataset from existing documents...");try {- // Pass both output dir and source docs dir (documents/ within the dataset)+ // Pass both output dir and source docs dir (data/ within the dataset)execSync(`npx tsx scripts/create-dataset.ts "${datasetDir}" "${documentsDir}"`, {cwd: path.join(import.meta.dirname, ".."),stdio: "inherit",