web search done right
Log number of failed searches separately from number of successful searches
Mikkel Denker9ae23fc7a69b
diff --git a/Cargo.lock b/Cargo.lockindex 7ca744fe..80d3dcce 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -179,9 +179,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"[[package]]name = "axum"-version = "0.5.17"+version = "0.6.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"+checksum = "6137c6234afb339e75e764c866e3594900f0211e1315d33779f269bbe2ec6967"dependencies = ["async-trait","axum-core",@@ -197,8 +197,10 @@ dependencies = ["mime","percent-encoding","pin-project-lite",+ "rustversion","serde","serde_json",+ "serde_path_to_error","serde_urlencoded","sync_wrapper","tokio",@@ -210,9 +212,9 @@ dependencies = [[[package]]name = "axum-core"-version = "0.2.9"+version = "0.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc"+checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34"dependencies = ["async-trait","bytes",@@ -220,15 +222,16 @@ dependencies = ["http","http-body","mime",+ "rustversion","tower-layer","tower-service",][[package]]name = "axum-extra"-version = "0.3.7"+version = "0.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "69034b3b0fd97923eee2ce8a47540edb21e07f48f87f67d44bb4271cec622bdb"+checksum = "72f6236b9edabc109ec6d5836ce90e359854a4e267d4241d29202f73261ad622"dependencies = ["axum","bytes",@@ -245,9 +248,9 @@ dependencies = [[[package]]name = "axum-macros"-version = "0.2.3"+version = "0.3.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6293dae2ec708e679da6736e857cf8532886ef258e92930f38279c12641628b8"+checksum = "5fbf955307ff8addb48d2399393c9e2740dd491537ec562b66ab364fc4a38841"dependencies = ["heck 0.4.1","proc-macro2",@@ -1539,9 +1542,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"[[package]]name = "http"-version = "0.2.8"+version = "0.2.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"+checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"dependencies = ["bytes","fnv",@@ -2050,9 +2053,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"[[package]]name = "matchit"-version = "0.5.0"+version = "0.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"+checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"[[package]]name = "matrixmultiply"@@ -3272,6 +3275,15 @@ dependencies = ["serde",]+[[package]]+name = "serde_path_to_error"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341"+dependencies = [+ "serde",+]+[[package]]name = "serde_urlencoded"version = "0.7.1"@@ -3926,9 +3938,9 @@ dependencies = [[[package]]name = "tower-http"-version = "0.3.5"+version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"+checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"dependencies = ["async-compression","bitflags",@@ -3948,6 +3960,7 @@ dependencies = ["tower","tower-layer","tower-service",+ "tracing",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 93c501f4..48fd28e8 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -27,9 +27,10 @@ tracing-subscriber = "0.3.11"whatlang = "0.16.0"tokio = { version = "1.23.1", features = ["full"] }askama = "0.11.1"-axum = "0.5.9"+axum = "0.6.9"+axum-extra = { version = "0.6.0", features = ["spa"] }+axum-macros = "0.3.4"anyhow = "1.0.58"-axum-extra = { version = "0.3.4", features = ["spa"] }html-escape = "0.2.11"logos = "0.12.1"csv = "1.1.6"@@ -48,7 +49,7 @@ once_cell = "1.13.1"regex = "1.6.0"http = "0.2.8"kuchiki = "0.8.1"-tower-http = { version = "0.3.4", features = ["compression-gzip"] }+tower-http = { version = "0.4.0", features = ["compression-gzip"] }lalrpop-util = { version = "0.19.8", features = ["lexer"] }lalrpop = { version = "0.19.8", features = ["lexer"] }serde_urlencoded = "0.7.1"@@ -56,7 +57,6 @@ rust-stemmers = "1.2.0"min-max-heap = "1.3.0"base64 = "0.13.0"async-trait = "0.1.57"-axum-macros = "0.2.3"urlencoding = "2.1.2"bitvec = "1.0.1"criterion = "0.3.6"diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rsindex 2163856c..d7b59c73 100644--- a/core/src/api/improvement.rs+++ b/core/src/api/improvement.rs@@ -61,8 +61,8 @@ impl From<StoreParams> for StoredQuery {#[allow(clippy::unused_async)]pub async fn store(- extract::Json(params): extract::Json<StoreParams>,Extension(state): Extension<Arc<State>>,+ extract::Json(params): extract::Json<StoreParams>,) -> impl IntoResponse {match state.improvement_queue.as_ref() {Some(q) => {diff --git a/core/src/api/metrics.rs b/core/src/api/metrics.rsindex 5a36de69..0d57d1f8 100644--- a/core/src/api/metrics.rs+++ b/core/src/api/metrics.rs@@ -16,14 +16,14 @@use std::sync::Arc;-use axum::{response::IntoResponse, Extension};+use axum::{extract, response::IntoResponse};use axum_macros::debug_handler;#[allow(clippy::unused_async)]#[allow(clippy::match_wild_err_arm)]#[debug_handler]pub async fn route(- Extension(registry): Extension<Arc<crate::metrics::PrometheusRegistry>>,+ extract::State(registry): extract::State<Arc<crate::metrics::PrometheusRegistry>>,) -> impl IntoResponse {format!("{}", registry)}diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 03c9b289..ab73c816 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use axum::{body::Body, routing::get_service, Extension, Router};+use axum::{body::Body, extract, middleware, routing::get_service, Router};use tokio::sync::Mutex;use tower_http::{compression::CompressionLayer, services::ServeDir};@@ -57,7 +57,8 @@ pub struct HtmlTemplate<T>(T);pub struct State {pub searcher: DistributedSearcher,pub autosuggest: Autosuggest,- pub search_counter: crate::metrics::Counter,+ pub search_counter_success: crate::metrics::Counter,+ pub search_counter_fail: crate::metrics::Counter,pub summarizer: Arc<Summarizer>,pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,}@@ -88,7 +89,11 @@ pub async fn favicon() -> impl IntoResponse {.unwrap()}-pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter) -> Result<Router> {+pub fn router(+ config: &FrontendConfig,+ search_counter_success: crate::metrics::Counter,+ search_counter_fail: crate::metrics::Counter,+) -> Result<Router> {let shards: Vec<_> = config.search_servers.clone()@@ -120,15 +125,20 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)let state = Arc::new(State {searcher,autosuggest,- search_counter,+ search_counter_success,+ search_counter_fail,summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),improvement_queue: query_store_queue,});Ok(Router::new().route("/", get(index::route))- .route("/search", get(search::route))- .route("/beta/api/search", post(search::api))+ .merge(+ Router::new()+ .route("/search", get(search::route))+ .route("/beta/api/search", post(search::api))+ .route_layer(middleware::from_fn_with_state(state.clone(), search_metric)),+ ).route("/autosuggest", get(autosuggest::route)).route("/autosuggest/browser", get(autosuggest::browser)).route("/favicon.ico", get(favicon))@@ -141,21 +151,30 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter).route("/opensearch.xml", get(opensearch::route)).route("/improvement/click", post(improvement::click)).route("/improvement/store", post(improvement::store))- .fallback(get_service(ServeDir::new("frontend/dist/")).handle_error(- |error: std::io::Error| async move {- (- StatusCode::INTERNAL_SERVER_ERROR,- format!("Unhandled internal error: {error}"),- )- },- ))+ .fallback(get_service(ServeDir::new("frontend/dist/"))).layer(CompressionLayer::new()).merge(Router::new().route("/summarize", get(summarize::route)))- .layer(Extension(state)))+ .with_state(state))}-pub fn metrics_router(registry: crate::metrics::PrometheusRegistry) -> Result<Router> {- Ok(Router::new()+pub fn metrics_router(registry: crate::metrics::PrometheusRegistry) -> Router {+ Router::new().route("/metrics", get(metrics::route))- .layer(Extension(Arc::new(registry))))+ .with_state(Arc::new(registry))+}++async fn search_metric<B>(+ extract::State(state): extract::State<Arc<State>>,+ request: axum::http::Request<B>,+ next: middleware::Next<B>,+) -> Response {+ let response = next.run(request).await;++ if response.status().is_success() {+ state.search_counter_success.inc();+ } else {+ state.search_counter_fail.inc();+ }++ response}diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex c1a9af45..d2a12c37 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -18,7 +18,7 @@ use http::{StatusCode, Uri};use optics::SiteRankings;use std::{collections::HashMap, sync::Arc};-use axum::{Extension, Json};+use axum::Json;use axum_macros::debug_handler;use crate::{@@ -94,11 +94,9 @@ fn extract_site_rankings(params: &HashMap<String, String>) -> Option<SiteRanking#[debug_handler]pub async fn route(extract::Query(params): extract::Query<HashMap<String, String>>,- Extension(state): Extension<Arc<State>>,+ extract::State(state): extract::State<Arc<State>>,extract::OriginalUri(uri): extract::OriginalUri,) -> Result<impl IntoResponse, StatusCode> {- state.search_counter.inc();-let query = params.get("q").cloned().unwrap_or_default();let skip_pages: usize = params@@ -242,11 +240,9 @@ fn next_page_url(uri: &Uri, params: HashMap<String, String>, skip_pages: usize)#[allow(clippy::match_wild_err_arm)]#[debug_handler]pub async fn api(+ extract::State(state): extract::State<Arc<State>>,extract::Json(mut query): extract::Json<SearchQuery>,- Extension(state): Extension<Arc<State>>,) -> impl IntoResponse {- state.search_counter.inc();-query.num_results = query.num_results.min(50 * NUM_RESULTS_PER_PAGE);match state.searcher.search(&query).await {diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rsindex 29395cce..5a860d0c 100644--- a/core/src/entrypoint/frontend.rs+++ b/core/src/entrypoint/frontend.rs@@ -18,23 +18,39 @@ use anyhow::Result;use crate::{api::{metrics_router, router},+ metrics::Label,FrontendConfig,};pub async fn run(config: FrontendConfig) -> Result<()> {- let search_counter = crate::metrics::Counter::default();+ let search_counter_success = crate::metrics::Counter::default();+ let search_counter_fail = crate::metrics::Counter::default();let mut registry = crate::metrics::PrometheusRegistry::default();let group = registry.new_group(- "total_search_requests".to_string(),+ "search_requests".to_string(),Some("Total number of incoming search requests.".to_string()),).unwrap();- group.register(search_counter.clone(), Vec::new());- let app = router(&config, search_counter)?;- let metrics_app = metrics_router(registry)?;+ group.register(+ search_counter_success.clone(),+ vec![Label {+ key: "status".to_string(),+ val: "success".to_string(),+ }],+ );+ group.register(+ search_counter_fail.clone(),+ vec![Label {+ key: "status".to_string(),+ val: "fail".to_string(),+ }],+ );++ let app = router(&config, search_counter_success, search_counter_fail)?;+ let metrics_app = metrics_router(registry);let addr = config.host.parse()?;tracing::info!("frontend server listening on {}", addr);