Skip to content

web search done right

Refactor `Justfile` and tracing with enabled debug tracing for stract (#87)

Oliver Bøving369d5031df3e
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b7a1939f..a3e52a89 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -33,5 +33,5 @@ The process is simple and fast. Upon your first pull request, you will be prompt
* Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.
* (Optional) run `just configure --alice` to download the weights for Alice (the AI assistant that can search on the web).
* (Optional) Install cargo-watch by running `cargo install cargo-watch`. This makes frontend development easier.
-
-After the non optional steps you can now run `just cargo test` and should see all tests passing. If you have installced `cargo-watch`, you should be able to run `just frontend` to start the search server and launch the frontend at `0.0.0.0:3000`. You can add the `--alice` flag to enable Alice.
\ No newline at end of file
+
+After the non optional steps you can now run `just cargo test` and should see all tests passing. If you have installced `cargo-watch`, you should be able to run `just dev` to start the search server and launch the frontend at `0.0.0.0:3000`. You can add the `--alice` flag to enable Alice.
diff --git a/Cargo.toml b/Cargo.toml
index 74ef9c81..0ad10394 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,7 +28,7 @@ tantivy = "0.19.2"
thiserror = "1.0.31"
toml = "0.5.9"
tracing = "0.1.34"
-tracing-subscriber = "0.3.11"
+tracing-subscriber = { version = "0.3.11", features= ["env-filter"] }
whatlang = "0.16.0"
tokio = { version = "1.23.1", features = ["full"] }
axum = "0.6.9"
diff --git a/core/src/distributed/sonic/mod.rs b/core/src/distributed/sonic/mod.rs
index c07167da..3e1b9017 100644
--- a/core/src/distributed/sonic/mod.rs
+++ b/core/src/distributed/sonic/mod.rs
@@ -143,7 +143,7 @@ where
pub async fn accept(&self) -> Result<Request<Req, Res>> {
let (mut stream, client) = self.listener.accept().await?;
- tracing::debug!("accepted connection from: {}", &client);
+ tracing::debug!(?client, "accepted connection");
let mut header_buf = vec![0; std::mem::size_of::<Header>()];
stream.read_exact(&mut header_buf).await?;
@@ -152,7 +152,6 @@ where
let mut buf = vec![0; header.body_size];
stream.read_exact(&mut buf).await?;
- tracing::debug!("received bytes: {:?}", &buf);
let body = Some(bincode::deserialize(&buf).unwrap());
diff --git a/core/src/main.rs b/core/src/main.rs
index 48b4a441..f1cb20cb 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -13,7 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use anyhow::Result;
+use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use serde::de::DeserializeOwned;
use std::fs;
@@ -27,8 +27,7 @@ use stract::entrypoint::configure;
use stract::entrypoint::indexer::IndexPointer;
use stract::entrypoint::{self, api, safety_classifier, search_server, webgraph_server};
use stract::webgraph::WebgraphBuilder;
-use tracing::Level;
-use tracing_subscriber::FmtSubscriber;
+use tracing_subscriber::prelude::*;
#[cfg(not(target_env = "msvc"))]
use tikv_jemallocator::Jemalloc;
@@ -189,15 +188,23 @@ enum IndexingOptions {
}
fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {
- let raw_config = fs::read_to_string(path).expect("Failed to read config file");
- toml::from_str(&raw_config).expect("Failed to parse config")
+ let path = path.as_ref();
+ let raw_config = fs::read_to_string(path)
+ .with_context(|| format!("Failed to read config: '{}'", path.display()))
+ .unwrap();
+ toml::from_str(&raw_config)
+ .with_context(|| format!("Failed to parse config: '{}'", path.display()))
+ .unwrap()
}
fn main() -> Result<()> {
- let subscriber = FmtSubscriber::builder()
- .with_max_level(Level::INFO)
- .finish();
- tracing::subscriber::set_global_default(subscriber).unwrap();
+ tracing_subscriber::fmt()
+ .with_max_level(tracing::Level::INFO)
+ .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+ .without_time()
+ .with_target(false)
+ .finish()
+ .init();
let args = Args::parse();
diff --git a/frontend/deno.json b/frontend/deno.json
index c86a3a28..6bb8f5fe 100644
--- a/frontend/deno.json
+++ b/frontend/deno.json
@@ -6,7 +6,8 @@
"build": "deno run -A dev.ts build",
"preview": "deno run -A main.ts",
"update": "deno run -A -r https://fresh.deno.dev/update .",
- "generateIcons": "deno run -A generateIcons.ts"
+ "generateIcons": "deno run -A generateIcons.ts",
+ "openapi": "deno run -A npm:openapi-typescript http://localhost:3000/beta/api/docs/openapi.json -o frontend/search/schema.d.ts"
},
"lint": {
"rules": {
diff --git a/justfile b/justfile
index 5751f941..9d0aad7b 100644
--- a/justfile
+++ b/justfile
@@ -1,11 +1,23 @@
-@frontend-rerun *ARGS:
- ./scripts/run_frontend.py {{ARGS}}
+@dev *ARGS:
+ ./scripts/run_dev.py {{ARGS}}
-@frontend *ARGS:
- cargo watch -i frontend -s 'just frontend-rerun {{ARGS}}'
+export RUST_LOG := "info,stract=debug"
-@frontend-openapi:
- deno run -A npm:openapi-typescript http://localhost:3000/beta/api/docs/openapi.json -o frontend/search/schema.d.ts
+export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
+
+@dev-api:
+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- api configs/api.toml"
+@dev-search-server:
+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"
+@dev-webgraph:
+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"
+@dev-alice:
+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- alice serve configs/alice.toml"
+@dev-frontend:
+ cd frontend && deno task start
+
+@openapi:
+ cd frontend && deno task openapi
@setup *ARGS:
just setup_python_env
diff --git a/scripts/run_dev.py b/scripts/run_dev.py
new file mode 100755
index 00000000..080674ef
--- /dev/null
+++ b/scripts/run_dev.py
@@ -0,0 +1,39 @@
+#!.venv/bin/python3
+import argparse
+import subprocess
+import os
+
+os.environ["LIBTORCH"] = "libtorch"
+os.environ["LD_LIBRARY_PATH"] = "libtorch/lib"
+os.environ["DYLD_LIBRARY_PATH"] = "libtorch/lib"
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--release", action="store_true")
+parser.add_argument("--alice", action="store_true")
+
+args = parser.parse_args()
+
+if args.release:
+ os.environ["STRACT_CARGO_ARGS"] = "--release"
+
+processes = []
+
+processes.append(subprocess.Popen(["just", "dev-api"]))
+processes.append(subprocess.Popen(["just", "dev-search-server"]))
+processes.append(subprocess.Popen(["just", "dev-webgraph"]))
+processes.append(subprocess.Popen(["just", "dev-frontend"]))
+
+if args.alice:
+ processes.append(subprocess.Popen(["just", "dev-alice"]))
+
+# kill processes on ctrl-c
+import time
+
+while True:
+ try:
+ time.sleep(1)
+ except KeyboardInterrupt:
+ for p in processes:
+ p.kill()
+ break
diff --git a/scripts/run_frontend.py b/scripts/run_frontend.py
deleted file mode 100755
index a06ea5b3..00000000
--- a/scripts/run_frontend.py
+++ /dev/null
@@ -1,69 +0,0 @@
-#!.venv/bin/python3
-import argparse
-import subprocess
-import os
-
-os.environ["LIBTORCH"] = "libtorch"
-os.environ["LD_LIBRARY_PATH"] = "libtorch/lib"
-os.environ["DYLD_LIBRARY_PATH"] = "libtorch/lib"
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument("--release", action="store_true")
-parser.add_argument("--alice", action="store_true")
-
-
-args = parser.parse_args()
-
-release = ["--release"] if args.release else []
-
-processes = []
-
-processes.append(
- subprocess.Popen(
- ["cargo", "run"] + release + ["--", "api", "configs/api.toml"]
- )
-)
-processes.append(
- subprocess.Popen(
- ["cargo", "run"]
- + release
- + ["--", "search-server", "configs/search_server.toml"]
- )
-)
-processes.append(
- subprocess.Popen(
- ["cargo", "run"]
- + release
- + [
- "--",
- "webgraph",
- "server",
- "configs/webgraph/server.toml",
- ]
- )
-)
-processes.append(
- subprocess.Popen(
- ["deno", "task", "start"],
- cwd="frontend"
- )
-)
-
-if args.alice:
- processes.append(
- subprocess.Popen(
- ["cargo", "run"] + release + ["--", "alice", "serve", "configs/alice.toml"]
- )
- )
-
-# kill processes on ctrl-c
-import time
-
-while True:
- try:
- time.sleep(1)
- except KeyboardInterrupt:
- for p in processes:
- p.kill()
- break