Skip to content

web search done right

make ml models optional during setup

Mikkel Denker8f976179042e
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a170b664..615806e7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -45,7 +45,7 @@ if they take our code without making theirs open source, which would breach the
- Install liburing. If you're using a Debian based Linux, you can install it by running `sudo apt install liburing-dev`.
- Install [just](https://github.com/casey/just) by running `cargo install just`. This allows you to run the scripts in the `justfile` file. A justfile is basically a simple Makefile.
- Install [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/) by running `cargo install wasm-pack`. This will allow for generation and packaging of client side WebAssembly code.
-- Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.
+- Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies and creates a small local index which you can use for development. Run `just configure --ml` instead if you also want to setup the ML models.
- Install [cargo-watch](https://github.com/watchexec/cargo-watch) by running `cargo install cargo-watch`.
- (Optional) Install [abeye](https://github.com/oeb25/abeye) by running `cargo install --git https://github.com/oeb25/abeye --locked`. This is used for generating the API client used by the frontend by running `just openapi`.
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index 516b4d7e..36aaef4b 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -179,6 +179,11 @@ fn create_inverted_index() -> Result<()> {
let centrality_path = Path::new(DATA_PATH).join("centrality");
let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");
let dual_encoder_path = Path::new(DATA_PATH).join("dual_encoder");
+ let dual_encoder_path = if !dual_encoder_path.exists() {
+ None
+ } else {
+ Some(dual_encoder_path)
+ };
let worker = indexer::IndexingWorker::new(IndexerConfig {
host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),
@@ -195,8 +200,8 @@ fn create_inverted_index() -> Result<()> {
minimum_clean_words: None,
batch_size: defaults::Indexing::batch_size(),
autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),
- dual_encoder: Some(IndexerDualEncoderConfig {
- model_path: dual_encoder_path.to_str().unwrap().to_string(),
+ dual_encoder: dual_encoder_path.map(|p| IndexerDualEncoderConfig {
+ model_path: p.to_str().unwrap().to_string(),
page_centrality_rank_threshold: Some(100_000),
}),
});
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index bb13cb05..95bb7d2f 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -97,6 +97,8 @@ enum Commands {
Configure {
#[clap(long)]
skip_download: bool,
+ #[clap(long)]
+ ml: bool,
},
// Commands for the live index.
@@ -367,7 +369,10 @@ fn main() -> Result<()> {
.block_on(entity_search_server::run(config))?;
}
#[cfg(feature = "dev")]
- Commands::Configure { skip_download } => {
+ Commands::Configure {
+ skip_download,
+ ml: _,
+ } => {
configure::run(skip_download)?;
}
Commands::Crawler { options } => match options {
diff --git a/justfile b/justfile
index 99b535aa..159e36ee 100644
--- a/justfile
+++ b/justfile
@@ -21,19 +21,12 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
cd frontend && npm run openapi
@setup *ARGS:
- cd crates/client-wasm && wasm-pack build --target web
just setup_python_env
-
-@prepare_models:
- just setup_python_env
- ./scripts/export_crossencoder
- ./scripts/export_dual_encoder
+ ./scripts/setup {{ARGS}}
@configure *ARGS:
just setup {{ARGS}}
- just prepare_models
RUST_LOG="none,stract=info" cargo run --release --all-features -- configure {{ARGS}}
@setup_python_env:
python3 -m venv .venv || true
- .venv/bin/pip install --upgrade -r scripts/requirements.txt
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
deleted file mode 100644
index 92441620..00000000
--- a/scripts/requirements.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-torch
-transformers
-datasets
-gensim
-safetensors
-pandas
diff --git a/scripts/setup b/scripts/setup
new file mode 100755
index 00000000..89d9c04a
--- /dev/null
+++ b/scripts/setup
@@ -0,0 +1,35 @@
+#!.venv/bin/python3
+import argparse
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--ml", action="store_true", help="Flag to configure whether everything for machine learning should also be setup")
+
+args = parser.parse_args()
+
+necesarry_packages = [
+ "pandas",
+]
+
+ml_packages = [
+ "safetensors",
+ "datasets",
+ "torch",
+ "transformers",
+]
+
+packages_to_install = necesarry_packages
+
+if args.ml:
+ packages_to_install += ml_packages
+
+subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade"] + packages_to_install)
+
+
+subprocess.run(["cd", "crates/client-wasm", "&&", "wasm-pack", "build", "--target", "web"])
+
+if args.ml:
+ subprocess.run(["./scripts/export_crossencoder"])
+ subprocess.run(["./scripts/export_dual_encoder"])
\ No newline at end of file
diff --git a/scripts/truncate_word2vec b/scripts/truncate_word2vec
deleted file mode 100755
index 805fb329..00000000
--- a/scripts/truncate_word2vec
+++ /dev/null
@@ -1,7 +0,0 @@
-#!.venv/bin/python3
-import gensim
-
-model = gensim.models.KeyedVectors.load_word2vec_format(
- "data/summarizer/word2vec.bin", limit=1000, binary=True)
-model.save_word2vec_format(
- "data/summarizer/truncated_word2vec.bin", binary=True)