Skip to content

A fast, helpful, and open-source document parser

fix dataset scripts

Logan Markewich01f421e864e4
diff --git a/scripts/compare-dataset.ts b/scripts/compare-dataset.ts
index 020561f..b47b3da 100644
--- a/scripts/compare-dataset.ts
+++ b/scripts/compare-dataset.ts
@@ -5,7 +5,7 @@
* npx tsx scripts/compare-dataset.ts [dataset-dir]
*
* The dataset should be created using create-dataset.ts and contains:
- * - documents/: The original document files
+ * - data/: The original document files
* - metadata.jsonl: Expected outputs for each document/page
*
* Exit codes:
diff --git a/scripts/create-dataset.ts b/scripts/create-dataset.ts
index 0848fa6..a69daa7 100644
--- a/scripts/create-dataset.ts
+++ b/scripts/create-dataset.ts
@@ -3,11 +3,11 @@
*
* Output structure:
* dataset/
- * documents/
+ * data/
* doc1.pdf
* doc2.docx
* ...
- * metadata.jsonl (each line: {"file_name": "documents/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})
+ * metadata.jsonl (each line: {"file_name": "data/doc1.pdf", "document": "doc1.pdf", "page": 1, "output_json": {...}})
*
* Usage:
* npx tsx scripts/create-dataset.ts [output-dir] [source-docs-dir]
@@ -73,7 +73,7 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Datase
// For non-PDF files that return text directly (no pages)
if (result.pages.length === 0 && result.text) {
rows.push({
- file_name: `documents/${relativePath}`,
+ file_name: `data/${relativePath}`,
document: relativePath,
page: 1,
output_text: result.text,
@@ -87,7 +87,7 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Datase
for (const page of result.pages) {
const jsonPage = result.json?.pages.find((p) => p.page === page.pageNum);
rows.push({
- file_name: `documents/${relativePath}`,
+ file_name: `data/${relativePath}`,
document: relativePath,
page: page.pageNum,
output_text: page.text,
@@ -100,7 +100,7 @@ async function processFile(filePath: string, baseDocDir: string): Promise<Datase
console.error(` ERROR: ${error instanceof Error ? error.message : error}`);
// Record the error as a dataset entry (blank result)
rows.push({
- file_name: `documents/${relativePath}`,
+ file_name: `data/${relativePath}`,
document: relativePath,
page: 0,
output_text: "",
diff --git a/scripts/upload-dataset.ts b/scripts/upload-dataset.ts
index 09ec2ba..9739046 100644
--- a/scripts/upload-dataset.ts
+++ b/scripts/upload-dataset.ts
@@ -5,7 +5,7 @@
* HF_TOKEN=xxx npx tsx scripts/upload-dataset.ts [dataset-dir] [repo-name]
*
* Arguments:
- * dataset-dir - Directory containing the dataset with documents/ subfolder
+ * dataset-dir - Directory containing the dataset with data/ subfolder
* (default: ./dataset)
* repo-name - HuggingFace repository name (default: llamaindex/liteparse_cicd_data)
*
@@ -13,7 +13,7 @@
* HF_TOKEN - HuggingFace API token with write access
*
* This script:
- * 1. Regenerates metadata.jsonl by re-running liteparse on documents/ in the dataset
+ * 1. Regenerates metadata.jsonl by re-running liteparse on data/ in the dataset
* 2. Uploads to HuggingFace using the huggingface_hub API
*/
@@ -46,7 +46,7 @@ async function main() {
// Step 1: Regenerate dataset from documents in the dataset directory
console.log("Step 1: Regenerating dataset from existing documents...");
try {
- // Pass both output dir and source docs dir (documents/ within the dataset)
+ // Pass both output dir and source docs dir (data/ within the dataset)
execSync(`npx tsx scripts/create-dataset.ts "${datasetDir}" "${documentsDir}"`, {
cwd: path.join(import.meta.dirname, ".."),
stdio: "inherit",