web search done right
option to limit number of concurrent search requests gives better rate limit control to slow down requests instead of crashing if we exceed the limits of the servers
Mikkel Denkerbe5bb09fcfe3
diff --git a/Cargo.lock b/Cargo.lockindex 213c5a0c..de7be79b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4607,6 +4607,7 @@ dependencies = ["tokio","tokio-stream","toml",+ "tower","tower-http","tracing","tracing-subscriber",@@ -5253,6 +5254,7 @@ dependencies = ["pin-project","pin-project-lite","tokio",+ "tokio-util","tower-layer","tower-service","tracing",diff --git a/Cargo.toml b/Cargo.tomlindex a0e16e54..bf1fc0f1 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -104,6 +104,7 @@ tokenizers = "0.13.2"tokio = { version = "1.23.1", features = ["full"] }tokio-stream = "0.1.11"toml = "0.8.2"+tower = { version = "0.4.13", features = ["limit"] }tower-http = { version = "0.5.0", features = ["compression-gzip", "cors"] }tracing = { version = "0.1.34", features = ["release_max_level_info"] }tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 4fc3035d..9642695f 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -99,6 +99,7 @@ tokenizers = { workspace = true }tokio = { workspace = true }tokio-stream = { workspace = true }toml = { workspace = true }+tower = { workspace = true }tower-http = { workspace = true }tracing = { workspace = true }tracing-subscriber = { workspace = true }diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 04ef9fd0..37c23151 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -3,8 +3,8 @@ use rand::seq::SliceRandom;use stract::{bangs::Bangs,config::{- ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, LLMConfig, SnippetConfig,- WidgetsConfig,+ defaults, ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, LLMConfig,+ SnippetConfig, WidgetsConfig,},distributed::member::ShardId,image_store::Image,@@ -120,6 +120,7 @@ pub async fn main() {model: "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf".to_string(),api_key: None,},+ max_concurrent_searches: defaults::Api::max_concurrent_searches(),};let mut queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex cc5bcbd0..0b5177c6 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -19,6 +19,7 @@use axum::{body::Body, extract, middleware, Router};use tokio::sync::Mutex;+use tower::limit::ConcurrencyLimitLayer;use tower_http::compression::CompressionLayer;use crate::{@@ -92,13 +93,17 @@ pub async fn favicon() -> impl IntoResponse {}fn build_router(state: Arc<State>) -> Router {+ let mut search = Router::new()+ .route("/beta/api/search", post(search::search))+ .route_layer(middleware::from_fn_with_state(state.clone(), search_metric))+ .layer(cors_layer());++ if let Some(limit) = state.config.max_concurrent_searches {+ search = search.layer(ConcurrencyLimitLayer::new(limit));+ }+Router::new()- .merge(- Router::new()- .route("/beta/api/search", post(search::search))- .route_layer(middleware::from_fn_with_state(state.clone(), search_metric))- .layer(cors_layer()),- )+ .merge(search).route("/favicon.ico", get(favicon)).merge(Router::new()diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex 3815ab11..db8ffc6c 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -48,6 +48,10 @@ impl Api {pub fn entity_sidebar() -> f64 {10.0}++ pub fn max_concurrent_searches() -> Option<usize> {+ None+ }}pub struct Snippet;diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 9162055a..273c43b2 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -234,6 +234,9 @@ pub struct ApiConfig {#[serde(default)]pub correction_config: CorrectionConfig,++ #[serde(default = "defaults::Api::max_concurrent_searches")]+ pub max_concurrent_searches: Option<usize>,}#[derive(Debug, Serialize, Deserialize, Clone)]