Skip to content

web search done right

Log number of failed searches separately from number of successful searches

Mikkel Denker9ae23fc7a69b
diff --git a/Cargo.lock b/Cargo.lock
index 7ca744fe..80d3dcce 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -179,9 +179,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "axum"
-version = "0.5.17"
+version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"
+checksum = "6137c6234afb339e75e764c866e3594900f0211e1315d33779f269bbe2ec6967"
dependencies = [
"async-trait",
"axum-core",
@@ -197,8 +197,10 @@ dependencies = [
"mime",
"percent-encoding",
"pin-project-lite",
+ "rustversion",
"serde",
"serde_json",
+ "serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
@@ -210,9 +212,9 @@ dependencies = [
[[package]]
name = "axum-core"
-version = "0.2.9"
+version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc"
+checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34"
dependencies = [
"async-trait",
"bytes",
@@ -220,15 +222,16 @@ dependencies = [
"http",
"http-body",
"mime",
+ "rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-extra"
-version = "0.3.7"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69034b3b0fd97923eee2ce8a47540edb21e07f48f87f67d44bb4271cec622bdb"
+checksum = "72f6236b9edabc109ec6d5836ce90e359854a4e267d4241d29202f73261ad622"
dependencies = [
"axum",
"bytes",
@@ -245,9 +248,9 @@ dependencies = [
[[package]]
name = "axum-macros"
-version = "0.2.3"
+version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6293dae2ec708e679da6736e857cf8532886ef258e92930f38279c12641628b8"
+checksum = "5fbf955307ff8addb48d2399393c9e2740dd491537ec562b66ab364fc4a38841"
dependencies = [
"heck 0.4.1",
"proc-macro2",
@@ -1539,9 +1542,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
[[package]]
name = "http"
-version = "0.2.8"
+version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
+checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
dependencies = [
"bytes",
"fnv",
@@ -2050,9 +2053,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]]
name = "matchit"
-version = "0.5.0"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
+checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
[[package]]
name = "matrixmultiply"
@@ -3272,6 +3275,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "serde_path_to_error"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -3926,9 +3938,9 @@ dependencies = [
[[package]]
name = "tower-http"
-version = "0.3.5"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
+checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"
dependencies = [
"async-compression",
"bitflags",
@@ -3948,6 +3960,7 @@ dependencies = [
"tower",
"tower-layer",
"tower-service",
+ "tracing",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 93c501f4..48fd28e8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,9 +27,10 @@ tracing-subscriber = "0.3.11"
whatlang = "0.16.0"
tokio = { version = "1.23.1", features = ["full"] }
askama = "0.11.1"
-axum = "0.5.9"
+axum = "0.6.9"
+axum-extra = { version = "0.6.0", features = ["spa"] }
+axum-macros = "0.3.4"
anyhow = "1.0.58"
-axum-extra = { version = "0.3.4", features = ["spa"] }
html-escape = "0.2.11"
logos = "0.12.1"
csv = "1.1.6"
@@ -48,7 +49,7 @@ once_cell = "1.13.1"
regex = "1.6.0"
http = "0.2.8"
kuchiki = "0.8.1"
-tower-http = { version = "0.3.4", features = ["compression-gzip"] }
+tower-http = { version = "0.4.0", features = ["compression-gzip"] }
lalrpop-util = { version = "0.19.8", features = ["lexer"] }
lalrpop = { version = "0.19.8", features = ["lexer"] }
serde_urlencoded = "0.7.1"
@@ -56,7 +57,6 @@ rust-stemmers = "1.2.0"
min-max-heap = "1.3.0"
base64 = "0.13.0"
async-trait = "0.1.57"
-axum-macros = "0.2.3"
urlencoding = "2.1.2"
bitvec = "1.0.1"
criterion = "0.3.6"
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rs
index 2163856c..d7b59c73 100644
--- a/core/src/api/improvement.rs
+++ b/core/src/api/improvement.rs
@@ -61,8 +61,8 @@ impl From<StoreParams> for StoredQuery {
#[allow(clippy::unused_async)]
pub async fn store(
- extract::Json(params): extract::Json<StoreParams>,
Extension(state): Extension<Arc<State>>,
+ extract::Json(params): extract::Json<StoreParams>,
) -> impl IntoResponse {
match state.improvement_queue.as_ref() {
Some(q) => {
diff --git a/core/src/api/metrics.rs b/core/src/api/metrics.rs
index 5a36de69..0d57d1f8 100644
--- a/core/src/api/metrics.rs
+++ b/core/src/api/metrics.rs
@@ -16,14 +16,14 @@
use std::sync::Arc;
-use axum::{response::IntoResponse, Extension};
+use axum::{extract, response::IntoResponse};
use axum_macros::debug_handler;
#[allow(clippy::unused_async)]
#[allow(clippy::match_wild_err_arm)]
#[debug_handler]
pub async fn route(
- Extension(registry): Extension<Arc<crate::metrics::PrometheusRegistry>>,
+ extract::State(registry): extract::State<Arc<crate::metrics::PrometheusRegistry>>,
) -> impl IntoResponse {
format!("{}", registry)
}
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 03c9b289..ab73c816 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use axum::{body::Body, routing::get_service, Extension, Router};
+use axum::{body::Body, extract, middleware, routing::get_service, Router};
use tokio::sync::Mutex;
use tower_http::{compression::CompressionLayer, services::ServeDir};
@@ -57,7 +57,8 @@ pub struct HtmlTemplate<T>(T);
pub struct State {
pub searcher: DistributedSearcher,
pub autosuggest: Autosuggest,
- pub search_counter: crate::metrics::Counter,
+ pub search_counter_success: crate::metrics::Counter,
+ pub search_counter_fail: crate::metrics::Counter,
pub summarizer: Arc<Summarizer>,
pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,
}
@@ -88,7 +89,11 @@ pub async fn favicon() -> impl IntoResponse {
.unwrap()
}
-pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter) -> Result<Router> {
+pub fn router(
+ config: &FrontendConfig,
+ search_counter_success: crate::metrics::Counter,
+ search_counter_fail: crate::metrics::Counter,
+) -> Result<Router> {
let shards: Vec<_> = config
.search_servers
.clone()
@@ -120,15 +125,20 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)
let state = Arc::new(State {
searcher,
autosuggest,
- search_counter,
+ search_counter_success,
+ search_counter_fail,
summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),
improvement_queue: query_store_queue,
});
Ok(Router::new()
.route("/", get(index::route))
- .route("/search", get(search::route))
- .route("/beta/api/search", post(search::api))
+ .merge(
+ Router::new()
+ .route("/search", get(search::route))
+ .route("/beta/api/search", post(search::api))
+ .route_layer(middleware::from_fn_with_state(state.clone(), search_metric)),
+ )
.route("/autosuggest", get(autosuggest::route))
.route("/autosuggest/browser", get(autosuggest::browser))
.route("/favicon.ico", get(favicon))
@@ -141,21 +151,30 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)
.route("/opensearch.xml", get(opensearch::route))
.route("/improvement/click", post(improvement::click))
.route("/improvement/store", post(improvement::store))
- .fallback(get_service(ServeDir::new("frontend/dist/")).handle_error(
- |error: std::io::Error| async move {
- (
- StatusCode::INTERNAL_SERVER_ERROR,
- format!("Unhandled internal error: {error}"),
- )
- },
- ))
+ .fallback(get_service(ServeDir::new("frontend/dist/")))
.layer(CompressionLayer::new())
.merge(Router::new().route("/summarize", get(summarize::route)))
- .layer(Extension(state)))
+ .with_state(state))
}
-pub fn metrics_router(registry: crate::metrics::PrometheusRegistry) -> Result<Router> {
- Ok(Router::new()
+pub fn metrics_router(registry: crate::metrics::PrometheusRegistry) -> Router {
+ Router::new()
.route("/metrics", get(metrics::route))
- .layer(Extension(Arc::new(registry))))
+ .with_state(Arc::new(registry))
+}
+
+async fn search_metric<B>(
+ extract::State(state): extract::State<Arc<State>>,
+ request: axum::http::Request<B>,
+ next: middleware::Next<B>,
+) -> Response {
+ let response = next.run(request).await;
+
+ if response.status().is_success() {
+ state.search_counter_success.inc();
+ } else {
+ state.search_counter_fail.inc();
+ }
+
+ response
}
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index c1a9af45..d2a12c37 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -18,7 +18,7 @@ use http::{StatusCode, Uri};
use optics::SiteRankings;
use std::{collections::HashMap, sync::Arc};
-use axum::{Extension, Json};
+use axum::Json;
use axum_macros::debug_handler;
use crate::{
@@ -94,11 +94,9 @@ fn extract_site_rankings(params: &HashMap<String, String>) -> Option<SiteRanking
#[debug_handler]
pub async fn route(
extract::Query(params): extract::Query<HashMap<String, String>>,
- Extension(state): Extension<Arc<State>>,
+ extract::State(state): extract::State<Arc<State>>,
extract::OriginalUri(uri): extract::OriginalUri,
) -> Result<impl IntoResponse, StatusCode> {
- state.search_counter.inc();
-
let query = params.get("q").cloned().unwrap_or_default();
let skip_pages: usize = params
@@ -242,11 +240,9 @@ fn next_page_url(uri: &Uri, params: HashMap<String, String>, skip_pages: usize)
#[allow(clippy::match_wild_err_arm)]
#[debug_handler]
pub async fn api(
+ extract::State(state): extract::State<Arc<State>>,
extract::Json(mut query): extract::Json<SearchQuery>,
- Extension(state): Extension<Arc<State>>,
) -> impl IntoResponse {
- state.search_counter.inc();
-
query.num_results = query.num_results.min(50 * NUM_RESULTS_PER_PAGE);
match state.searcher.search(&query).await {
diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rs
index 29395cce..5a860d0c 100644
--- a/core/src/entrypoint/frontend.rs
+++ b/core/src/entrypoint/frontend.rs
@@ -18,23 +18,39 @@ use anyhow::Result;
use crate::{
api::{metrics_router, router},
+ metrics::Label,
FrontendConfig,
};
pub async fn run(config: FrontendConfig) -> Result<()> {
- let search_counter = crate::metrics::Counter::default();
+ let search_counter_success = crate::metrics::Counter::default();
+ let search_counter_fail = crate::metrics::Counter::default();
let mut registry = crate::metrics::PrometheusRegistry::default();
let group = registry
.new_group(
- "total_search_requests".to_string(),
+ "search_requests".to_string(),
Some("Total number of incoming search requests.".to_string()),
)
.unwrap();
- group.register(search_counter.clone(), Vec::new());
- let app = router(&config, search_counter)?;
- let metrics_app = metrics_router(registry)?;
+ group.register(
+ search_counter_success.clone(),
+ vec![Label {
+ key: "status".to_string(),
+ val: "success".to_string(),
+ }],
+ );
+ group.register(
+ search_counter_fail.clone(),
+ vec![Label {
+ key: "status".to_string(),
+ val: "fail".to_string(),
+ }],
+ );
+
+ let app = router(&config, search_counter_success, search_counter_fail)?;
+ let metrics_app = metrics_router(registry);
let addr = config.host.parse()?;
tracing::info!("frontend server listening on {}", addr);