web search done right
Refactor `Justfile` and tracing with enabled debug tracing for stract (#87)
Oliver Bøving369d5031df3e
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex b7a1939f..a3e52a89 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -33,5 +33,5 @@ The process is simple and fast. Upon your first pull request, you will be prompt* Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.* (Optional) run `just configure --alice` to download the weights for Alice (the AI assistant that can search on the web).* (Optional) Install cargo-watch by running `cargo install cargo-watch`. This makes frontend development easier.--After the non optional steps you can now run `just cargo test` and should see all tests passing. If you have installced `cargo-watch`, you should be able to run `just frontend` to start the search server and launch the frontend at `0.0.0.0:3000`. You can add the `--alice` flag to enable Alice.\ No newline at end of file++After the non optional steps you can now run `just cargo test` and should see all tests passing. If you have installced `cargo-watch`, you should be able to run `just dev` to start the search server and launch the frontend at `0.0.0.0:3000`. You can add the `--alice` flag to enable Alice.diff --git a/Cargo.toml b/Cargo.tomlindex 74ef9c81..0ad10394 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -28,7 +28,7 @@ tantivy = "0.19.2"thiserror = "1.0.31"toml = "0.5.9"tracing = "0.1.34"-tracing-subscriber = "0.3.11"+tracing-subscriber = { version = "0.3.11", features= ["env-filter"] }whatlang = "0.16.0"tokio = { version = "1.23.1", features = ["full"] }axum = "0.6.9"diff --git a/core/src/distributed/sonic/mod.rs b/core/src/distributed/sonic/mod.rsindex c07167da..3e1b9017 100644--- a/core/src/distributed/sonic/mod.rs+++ b/core/src/distributed/sonic/mod.rs@@ -143,7 +143,7 @@ wherepub async fn accept(&self) -> Result<Request<Req, Res>> {let (mut stream, client) = self.listener.accept().await?;- tracing::debug!("accepted connection from: {}", &client);+ tracing::debug!(?client, "accepted connection");let mut header_buf = vec![0; std::mem::size_of::<Header>()];stream.read_exact(&mut header_buf).await?;@@ -152,7 +152,6 @@ wherelet mut buf = vec![0; header.body_size];stream.read_exact(&mut buf).await?;- tracing::debug!("received bytes: {:?}", &buf);let body = Some(bincode::deserialize(&buf).unwrap());diff --git a/core/src/main.rs b/core/src/main.rsindex 48b4a441..f1cb20cb 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -13,7 +13,7 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use anyhow::Result;+use anyhow::{Context, Result};use clap::{Parser, Subcommand};use serde::de::DeserializeOwned;use std::fs;@@ -27,8 +27,7 @@ use stract::entrypoint::configure;use stract::entrypoint::indexer::IndexPointer;use stract::entrypoint::{self, api, safety_classifier, search_server, webgraph_server};use stract::webgraph::WebgraphBuilder;-use tracing::Level;-use tracing_subscriber::FmtSubscriber;+use tracing_subscriber::prelude::*;#[cfg(not(target_env = "msvc"))]use tikv_jemallocator::Jemalloc;@@ -189,15 +188,23 @@ enum IndexingOptions {}fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {- let raw_config = fs::read_to_string(path).expect("Failed to read config file");- toml::from_str(&raw_config).expect("Failed to parse config")+ let path = path.as_ref();+ let raw_config = fs::read_to_string(path)+ .with_context(|| format!("Failed to read config: '{}'", path.display()))+ .unwrap();+ toml::from_str(&raw_config)+ .with_context(|| format!("Failed to parse config: '{}'", path.display()))+ .unwrap()}fn main() -> Result<()> {- let subscriber = FmtSubscriber::builder()- .with_max_level(Level::INFO)- .finish();- tracing::subscriber::set_global_default(subscriber).unwrap();+ tracing_subscriber::fmt()+ .with_max_level(tracing::Level::INFO)+ .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())+ .without_time()+ .with_target(false)+ .finish()+ .init();let args = Args::parse();diff --git a/frontend/deno.json b/frontend/deno.jsonindex c86a3a28..6bb8f5fe 100644--- a/frontend/deno.json+++ b/frontend/deno.json@@ -6,7 +6,8 @@"build": "deno run -A dev.ts build","preview": "deno run -A main.ts","update": "deno run -A -r https://fresh.deno.dev/update .",- "generateIcons": "deno run -A generateIcons.ts"+ "generateIcons": "deno run -A generateIcons.ts",+ "openapi": "deno run -A npm:openapi-typescript http://localhost:3000/beta/api/docs/openapi.json -o frontend/search/schema.d.ts"},"lint": {"rules": {diff --git a/justfile b/justfileindex 5751f941..9d0aad7b 100644--- a/justfile+++ b/justfile@@ -1,11 +1,23 @@-@frontend-rerun *ARGS:- ./scripts/run_frontend.py {{ARGS}}+@dev *ARGS:+ ./scripts/run_dev.py {{ARGS}}-@frontend *ARGS:- cargo watch -i frontend -s 'just frontend-rerun {{ARGS}}'+export RUST_LOG := "info,stract=debug"-@frontend-openapi:- deno run -A npm:openapi-typescript http://localhost:3000/beta/api/docs/openapi.json -o frontend/search/schema.d.ts+export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")++@dev-api:+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- api configs/api.toml"+@dev-search-server:+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"+@dev-webgraph:+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"+@dev-alice:+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- alice serve configs/alice.toml"+@dev-frontend:+ cd frontend && deno task start++@openapi:+ cd frontend && deno task openapi@setup *ARGS:just setup_python_envdiff --git a/scripts/run_dev.py b/scripts/run_dev.pynew file mode 100755index 00000000..080674ef--- /dev/null+++ b/scripts/run_dev.py@@ -0,0 +1,39 @@+#!.venv/bin/python3+import argparse+import subprocess+import os++os.environ["LIBTORCH"] = "libtorch"+os.environ["LD_LIBRARY_PATH"] = "libtorch/lib"+os.environ["DYLD_LIBRARY_PATH"] = "libtorch/lib"++parser = argparse.ArgumentParser()++parser.add_argument("--release", action="store_true")+parser.add_argument("--alice", action="store_true")++args = parser.parse_args()++if args.release:+ os.environ["STRACT_CARGO_ARGS"] = "--release"++processes = []++processes.append(subprocess.Popen(["just", "dev-api"]))+processes.append(subprocess.Popen(["just", "dev-search-server"]))+processes.append(subprocess.Popen(["just", "dev-webgraph"]))+processes.append(subprocess.Popen(["just", "dev-frontend"]))++if args.alice:+ processes.append(subprocess.Popen(["just", "dev-alice"]))++# kill processes on ctrl-c+import time++while True:+ try:+ time.sleep(1)+ except KeyboardInterrupt:+ for p in processes:+ p.kill()+ breakdiff --git a/scripts/run_frontend.py b/scripts/run_frontend.pydeleted file mode 100755index a06ea5b3..00000000--- a/scripts/run_frontend.py+++ /dev/null@@ -1,69 +0,0 @@-#!.venv/bin/python3-import argparse-import subprocess-import os--os.environ["LIBTORCH"] = "libtorch"-os.environ["LD_LIBRARY_PATH"] = "libtorch/lib"-os.environ["DYLD_LIBRARY_PATH"] = "libtorch/lib"--parser = argparse.ArgumentParser()--parser.add_argument("--release", action="store_true")-parser.add_argument("--alice", action="store_true")---args = parser.parse_args()--release = ["--release"] if args.release else []--processes = []--processes.append(- subprocess.Popen(- ["cargo", "run"] + release + ["--", "api", "configs/api.toml"]- )-)-processes.append(- subprocess.Popen(- ["cargo", "run"]- + release- + ["--", "search-server", "configs/search_server.toml"]- )-)-processes.append(- subprocess.Popen(- ["cargo", "run"]- + release- + [- "--",- "webgraph",- "server",- "configs/webgraph/server.toml",- ]- )-)-processes.append(- subprocess.Popen(- ["deno", "task", "start"],- cwd="frontend"- )-)--if args.alice:- processes.append(- subprocess.Popen(- ["cargo", "run"] + release + ["--", "alice", "serve", "configs/alice.toml"]- )- )--# kill processes on ctrl-c-import time--while True:- try:- time.sleep(1)- except KeyboardInterrupt:- for p in processes:- p.kill()- break