web search done right
make ml models optional during setup
Mikkel Denker8f976179042e
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex a170b664..615806e7 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -45,7 +45,7 @@ if they take our code without making theirs open source, which would breach the- Install liburing. If you're using a Debian based Linux, you can install it by running `sudo apt install liburing-dev`.- Install [just](https://github.com/casey/just) by running `cargo install just`. This allows you to run the scripts in the `justfile` file. A justfile is basically a simple Makefile.- Install [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/) by running `cargo install wasm-pack`. This will allow for generation and packaging of client side WebAssembly code.-- Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.+- Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies and creates a small local index which you can use for development. Run `just configure --ml` instead if you also want to setup the ML models.- Install [cargo-watch](https://github.com/watchexec/cargo-watch) by running `cargo install cargo-watch`.- (Optional) Install [abeye](https://github.com/oeb25/abeye) by running `cargo install --git https://github.com/oeb25/abeye --locked`. This is used for generating the API client used by the frontend by running `just openapi`.diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex 516b4d7e..36aaef4b 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -179,6 +179,11 @@ fn create_inverted_index() -> Result<()> {let centrality_path = Path::new(DATA_PATH).join("centrality");let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");let dual_encoder_path = Path::new(DATA_PATH).join("dual_encoder");+ let dual_encoder_path = if !dual_encoder_path.exists() {+ None+ } else {+ Some(dual_encoder_path)+ };let worker = indexer::IndexingWorker::new(IndexerConfig {host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),@@ -195,8 +200,8 @@ fn create_inverted_index() -> Result<()> {minimum_clean_words: None,batch_size: defaults::Indexing::batch_size(),autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),- dual_encoder: Some(IndexerDualEncoderConfig {- model_path: dual_encoder_path.to_str().unwrap().to_string(),+ dual_encoder: dual_encoder_path.map(|p| IndexerDualEncoderConfig {+ model_path: p.to_str().unwrap().to_string(),page_centrality_rank_threshold: Some(100_000),}),});diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex bb13cb05..95bb7d2f 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -97,6 +97,8 @@ enum Commands {Configure {#[clap(long)]skip_download: bool,+ #[clap(long)]+ ml: bool,},// Commands for the live index.@@ -367,7 +369,10 @@ fn main() -> Result<()> {.block_on(entity_search_server::run(config))?;}#[cfg(feature = "dev")]- Commands::Configure { skip_download } => {+ Commands::Configure {+ skip_download,+ ml: _,+ } => {configure::run(skip_download)?;}Commands::Crawler { options } => match options {diff --git a/justfile b/justfileindex 99b535aa..159e36ee 100644--- a/justfile+++ b/justfile@@ -21,19 +21,12 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")cd frontend && npm run openapi@setup *ARGS:- cd crates/client-wasm && wasm-pack build --target webjust setup_python_env--@prepare_models:- just setup_python_env- ./scripts/export_crossencoder- ./scripts/export_dual_encoder+ ./scripts/setup {{ARGS}}@configure *ARGS:just setup {{ARGS}}- just prepare_modelsRUST_LOG="none,stract=info" cargo run --release --all-features -- configure {{ARGS}}@setup_python_env:python3 -m venv .venv || true- .venv/bin/pip install --upgrade -r scripts/requirements.txtdiff --git a/scripts/requirements.txt b/scripts/requirements.txtdeleted file mode 100644index 92441620..00000000--- a/scripts/requirements.txt+++ /dev/null@@ -1,6 +0,0 @@-torch-transformers-datasets-gensim-safetensors-pandasdiff --git a/scripts/setup b/scripts/setupnew file mode 100755index 00000000..89d9c04a--- /dev/null+++ b/scripts/setup@@ -0,0 +1,35 @@+#!.venv/bin/python3+import argparse+import subprocess+import sys++parser = argparse.ArgumentParser()++parser.add_argument("--ml", action="store_true", help="Flag to configure whether everything for machine learning should also be setup")++args = parser.parse_args()++necesarry_packages = [+ "pandas",+]++ml_packages = [+ "safetensors",+ "datasets",+ "torch",+ "transformers",+]++packages_to_install = necesarry_packages++if args.ml:+ packages_to_install += ml_packages++subprocess.run([sys.executable, "-m", "pip", "install", "--upgrade"] + packages_to_install)+++subprocess.run(["cd", "crates/client-wasm", "&&", "wasm-pack", "build", "--target", "web"])++if args.ml:+ subprocess.run(["./scripts/export_crossencoder"])+ subprocess.run(["./scripts/export_dual_encoder"])\ No newline at end of filediff --git a/scripts/truncate_word2vec b/scripts/truncate_word2vecdeleted file mode 100755index 805fb329..00000000--- a/scripts/truncate_word2vec+++ /dev/null@@ -1,7 +0,0 @@-#!.venv/bin/python3-import gensim--model = gensim.models.KeyedVectors.load_word2vec_format(- "data/summarizer/word2vec.bin", limit=1000, binary=True)-model.save_word2vec_format(- "data/summarizer/truncated_word2vec.bin", binary=True)