Skip to content

web search done right

Add basic CI (#156)

Oliver Bøving2d8973bcf7dc
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 00000000..06b9cba9
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,48 @@
+# Based on: https://github.com/clap-rs/clap/blob/master/.github/workflows/ci.yml
+
+name: CI
+
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ push:
+
+env:
+ RUST_BACKTRACE: 1
+ CARGO_TERM_COLOR: always
+ CLICOLOR: 1
+
+jobs:
+ test:
+ name: Test
+ strategy:
+ matrix:
+ build: [linux]
+ include:
+ - build: linux
+ os: ubuntu-latest
+ rust: "stable"
+ runs-on: ${{ matrix.os }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+ with:
+ submodules: 'recursive'
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: ${{ matrix.rust }}
+ - uses: Swatinem/rust-cache@v2
+ - name: Install liburing
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y liburing-dev
+ - uses: taiki-e/install-action@v2
+ with:
+ tool: cargo-about
+ - name: Run CI
+ run: ./scripts/ci/all
+ - name: Test
+ run: cargo test
diff --git a/crates/core/src/ranking/models/cross_encoder.rs b/crates/core/src/ranking/models/cross_encoder.rs
index 2d6c8e51..d13096e8 100644
--- a/crates/core/src/ranking/models/cross_encoder.rs
+++ b/crates/core/src/ranking/models/cross_encoder.rs
@@ -165,12 +165,12 @@ mod tests {
#[test]
fn sanity_check() {
- if !Path::new("../../data/cross_encoder").exists() {
+ let data_path = Path::new("../../data/cross_encoder");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
return;
}
-
- let model = CrossEncoderModel::open("../../data/cross_encoder")
- .expect("Failed to find cross-encoder model");
+ let model = CrossEncoderModel::open(data_path).expect("Failed to find cross-encoder model");
let s = model.run(
"how many people live in paris",
diff --git a/crates/core/src/summarizer.rs b/crates/core/src/summarizer.rs
index 901a8c0c..cb4b2591 100644
--- a/crates/core/src/summarizer.rs
+++ b/crates/core/src/summarizer.rs
@@ -545,12 +545,12 @@ mod tests {
#[test]
fn test_dual_encoder() {
- if !Path::new("../../data/summarizer/dual_encoder").exists() {
+ let data_path = Path::new("../../data/summarizer/dual_encoder");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
return;
}
-
- let model =
- DualEncoder::open("../../data/summarizer/dual_encoder").expect("Failed to load model");
+ let model = DualEncoder::open(data_path).expect("Failed to load model");
let query = "What is the capital of France?";
let pos = "The capital of France is Paris.";
let neg = "The best baguette in Paris can be found at Boulangerie Pichard.";
diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rs
index 112ed285..c8664dc4 100644
--- a/crates/core/src/warc.rs
+++ b/crates/core/src/warc.rs
@@ -725,16 +725,15 @@ mod tests {
#[test]
fn internet_archive_parse() {
- if !Path::new("../../data/internet_archive.warc.gz").exists() {
+ let data_path = Path::new("../../data/internet_archive.warc.gz");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
return;
}
let mut records = 0;
- for record in WarcFile::open("../../data/internet_archive.warc.gz")
- .unwrap()
- .records()
- {
+ for record in WarcFile::open(data_path).unwrap().records() {
records += 1;
if let Err(err) = record {
panic!("Error: {:?}", err);
diff --git a/crates/core/src/widgets/thesaurus.rs b/crates/core/src/widgets/thesaurus.rs
index 736eeb63..adeed6a2 100644
--- a/crates/core/src/widgets/thesaurus.rs
+++ b/crates/core/src/widgets/thesaurus.rs
@@ -504,7 +504,12 @@ mod tests {
#[test]
fn build_dict() {
- let dict = Dictionary::build("../../data/english-wordnet-2022-subset.ttl").unwrap();
+ let data_path = Path::new("../../data/english-wordnet-2022-subset.ttl");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
+ return;
+ }
+ let dict = Dictionary::build(data_path).unwrap();
let infos = dict.get(Lemma("barely".to_string()));
diff --git a/crates/zimba/src/lib.rs b/crates/zimba/src/lib.rs
index 8febd32d..c718b83f 100644
--- a/crates/zimba/src/lib.rs
+++ b/crates/zimba/src/lib.rs
@@ -636,11 +636,13 @@ mod tests {
#[test]
fn it_works() {
- if !Path::new("../../data/test.zim").exists() {
+ let data_path = Path::new("../../data/test.zim");
+ if !data_path.exists() {
+ // Skip test if data file is not present
return;
}
- let zim = ZimFile::open("../../data/test.zim").unwrap();
+ let zim = ZimFile::open(data_path).unwrap();
assert_eq!(zim.header.magic, 72173914);
assert_eq!(zim.header.major_version, 5);
diff --git a/crates/zimba/src/wiki.rs b/crates/zimba/src/wiki.rs
index 3dd07e8d..f317e105 100644
--- a/crates/zimba/src/wiki.rs
+++ b/crates/zimba/src/wiki.rs
@@ -256,11 +256,18 @@ impl<'a> Iterator for ImageIterator<'a> {
#[cfg(test)]
mod tests {
+ use std::path::Path;
+
use super::*;
#[test]
fn test_article_iterator() {
- let zim = ZimFile::open("../../data/test.zim").unwrap();
+ let data_path = Path::new("../../data/test.zim");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
+ return;
+ }
+ let zim = ZimFile::open(data_path).unwrap();
let mut iter = ArticleIterator::new(&zim).unwrap();
let article = iter.next().unwrap();
@@ -271,7 +278,12 @@ mod tests {
#[test]
fn test_image_iterator() {
- let zim = ZimFile::open("../../data/test.zim").unwrap();
+ let data_path = Path::new("../../data/test.zim");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
+ return;
+ }
+ let zim = ZimFile::open(data_path).unwrap();
let mut iter = ImageIterator::new(&zim).unwrap();
let image = iter.next().unwrap();
diff --git a/frontend/package.json b/frontend/package.json
index 40eff060..df0fff5a 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -28,6 +28,7 @@
"@tailwindcss/line-clamp": "^0.4.4",
"@tailwindcss/typography": "^0.5.10",
"@types/file-saver": "^2.0.7",
+ "@types/node": "^20.11.19",
"@typescript-eslint/eslint-plugin": "^6.14.0",
"@typescript-eslint/parser": "^6.14.0",
"autoprefixer": "^10.4.16",
@@ -60,4 +61,4 @@
"tailwind-merge": "^2.1.0",
"ts-pattern": "^5.0.6"
}
-}
\ No newline at end of file
+}
diff --git a/scripts/ci/check b/scripts/ci/check
index 8aa9f335..e7e240d6 100755
--- a/scripts/ci/check
+++ b/scripts/ci/check
@@ -3,4 +3,4 @@ set -e
cargo check
cargo check --no-default-features
-cd frontend && npm run check
+cd frontend && npm install && npm run check