Skip to content

A fast, helpful, and open-source document parser

fix: remove temporary paths from bytes and conversions after usage; fix: make integration tests dependent on an env var; fix: node ci

Clelia (Astra) Bertellie401ccbbd87d
diff --git a/.github/workflows/ci-node.yml b/.github/workflows/ci-node.yml
index b89e4a9..55e6e88 100644
--- a/.github/workflows/ci-node.yml
+++ b/.github/workflows/ci-node.yml
@@ -4,15 +4,15 @@ on:
push:
branches: [main]
paths:
- - 'crates/**'
- - 'packages/node/**'
- - '.github/workflows/ci-node.yml'
+ - "crates/**"
+ - "packages/node/**"
+ - ".github/workflows/ci-node.yml"
pull_request:
branches: [main]
paths:
- - 'crates/**'
- - 'packages/node/**'
- - '.github/workflows/ci-node.yml'
+ - "crates/**"
+ - "packages/node/**"
+ - ".github/workflows/ci-node.yml"
env:
CARGO_TERM_COLOR: always
@@ -80,7 +80,7 @@ jobs:
- name: Install dependencies
working-directory: packages/node
- run: npm ci
+ run: npm install --ignore-scripts
- name: Build native module
working-directory: packages/node
@@ -147,7 +147,7 @@ jobs:
- name: Install dependencies
working-directory: packages/node
- run: npm ci
+ run: npm install --ignore-scripts
- name: Download artifacts
uses: actions/download-artifact@v4
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c26d9e0..424295a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,6 +8,7 @@ on:
env:
CARGO_TERM_COLOR: always
+ SKIP_INTEGRATION_TESTS: yes
jobs:
fmt:
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index 3495db1..46768e4 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -1,3 +1,5 @@
+use std::path::Path;
+
use crate::config::{LiteParseConfig, parse_target_pages};
#[cfg(not(target_arch = "wasm32"))]
use crate::conversion;
@@ -83,6 +85,7 @@ impl LiteParse {
let t0 = web_time::Instant::now();
let _tmp_dir;
+ let mut is_converted = false;
let validated_input = match input {
PdfInput::Path(p) => PdfInput::Path(p),
@@ -90,6 +93,7 @@ impl LiteParse {
let (converted, tmp_dir) =
convert_data_to_pdf(b, self.config.password.as_deref()).await?;
_tmp_dir = tmp_dir;
+ is_converted = true;
PdfInput::Path(converted.pdf_path)
}
};
@@ -181,6 +185,15 @@ impl LiteParse {
let total = t2.duration_since(t0).as_secs_f64() * 1000.0;
log(&format!("[liteparse] total: {:.1}ms", total));
+ // Office docs and images that are temporarily created from bytes
+ // are removed, but not PDFs, as they pass through the coversion logic
+ // and are used directly as input.
+ // With this block, we insure that temporary PDFs created from bytes and/or
+ // converted from Office/image files are cleaned up as well.
+ if is_converted && let PdfInput::Path(p) = validated_input {
+ tokio::fs::remove_dir_all(Path::new(&p).parent().unwrap()).await?;
+ }
+
Ok(ParseResult {
pages: parsed_pages,
text: full_text,
diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs
index d6a0bb4..bca611a 100644
--- a/crates/liteparse/tests/integration_test.rs
+++ b/crates/liteparse/tests/integration_test.rs
@@ -8,6 +8,10 @@ use serial_test::serial;
#[tokio::test]
#[serial]
async fn test_convert_data_to_pdf_integration() {
+ let env_var = std::env::var("SKIP_INTEGRATION_TESTS");
+ if let Ok(v) = env_var && v == "yes" {
+ return;
+ }
let fixture_path = "../../integration_tests_data/receipt.png";
let data = tokio::fs::read(fixture_path)
.await
@@ -21,6 +25,10 @@ async fn test_convert_data_to_pdf_integration() {
#[tokio::test]
#[serial]
async fn test_parse_bytes_image_integration() {
+ let env_var = std::env::var("SKIP_INTEGRATION_TESTS");
+ if let Ok(v) = env_var && v == "yes" {
+ return;
+ }
let fixture_path = "../../integration_tests_data/receipt.png";
let lit = LiteParse::new(LiteParseConfig::default());
let data = tokio::fs::read(fixture_path)
@@ -37,6 +45,10 @@ async fn test_parse_bytes_image_integration() {
#[tokio::test]
#[serial]
async fn test_parse_bytes_office_integration() {
+ let env_var = std::env::var("SKIP_INTEGRATION_TESTS");
+ if let Ok(v) = env_var && v == "yes" {
+ return;
+ }
let fixture_path = "../../integration_tests_data/sample3.doc";
let lit = LiteParse::new(LiteParseConfig::default());
let data = tokio::fs::read(fixture_path)
@@ -53,6 +65,10 @@ async fn test_parse_bytes_office_integration() {
#[tokio::test]
#[serial]
async fn test_parse_image_integration() {
+ let env_var = std::env::var("SKIP_INTEGRATION_TESTS");
+ if let Ok(v) = env_var && v == "yes" {
+ return;
+ }
let lit = LiteParse::new(LiteParseConfig::default());
let parsed = lit
.parse("../../integration_tests_data/receipt.png")
@@ -64,6 +80,10 @@ async fn test_parse_image_integration() {
#[tokio::test]
#[serial]
async fn test_parse_office_doc_integration() {
+ let env_var = std::env::var("SKIP_INTEGRATION_TESTS");
+ if let Ok(v) = env_var && v == "yes" {
+ return;
+ }
let lit = LiteParse::new(LiteParseConfig::default());
let parsed = lit
.parse("../../integration_tests_data/sample3.doc")