web search done right
json api
Mikkel Denker02559946c63e
diff --git a/Cargo.lock b/Cargo.lockindex b9211342..85511793 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -388,6 +388,7 @@ dependencies = ["js-sys","num-integer","num-traits",+ "serde","time 0.1.44","wasm-bindgen","winapi",diff --git a/Cargo.toml b/Cargo.tomlindex 224274a6..823388a6 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -39,7 +39,7 @@ csv = "1.1.6"fst = { version = "0.4.7", features = ["levenshtein"] }rocksdb = "0.19.0"image = "0.24.3"-chrono = "0.4.20"+chrono = {version = "0.4.20", features = ["serde"] }uuid = "1.1.2"futures = "0.3.21"quick-xml = "0.23.0"diff --git a/src/bangs.rs b/src/bangs.rsindex c212adb6..a41025e8 100644--- a/src/bangs.rs+++ b/src/bangs.rs@@ -50,7 +50,7 @@ pub struct Bang {url: String,}-#[derive(Debug, PartialEq, Eq)]+#[derive(Debug, Serialize, PartialEq, Eq)]pub struct BangHit {bang: Bang,pub redirect_to: Url,diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rsindex ac463634..cd23cca2 100644--- a/src/entity_index/mod.rs+++ b/src/entity_index/mod.rs@@ -22,6 +22,7 @@ use std::{time::Duration,};+use serde::Serialize;use tantivy::{collector::TopDocs,query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},@@ -144,7 +145,7 @@ fn wikipedify_url(url: &Url) -> Vec<Url> {]}-#[derive(Debug, PartialEq, Eq)]+#[derive(Debug, Serialize, PartialEq, Eq)]pub struct StoredEntity {pub title: String,pub entity_abstract: String,diff --git a/src/frontend/api.rs b/src/frontend/api.rsnew file mode 100644index 00000000..97333707--- /dev/null+++ b/src/frontend/api.rs@@ -0,0 +1,48 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{collections::HashMap, sync::Arc};++use crate::webpage::region::Region;++use super::State;+use axum::{extract, response::IntoResponse, Extension, Json};++#[allow(clippy::unused_async)]+pub async fn search(+ extract::Query(params): extract::Query<HashMap<String, String>>,+ Extension(state): Extension<Arc<State>>,+) -> impl IntoResponse {+ let query = params.get("q").cloned().unwrap_or_default();++ let skip_pages = params.get("p").and_then(|p| p.parse().ok());++ let selected_region = params.get("gl").and_then(|gl| {+ if let Ok(region) = Region::from_gl(gl) {+ Some(region)+ } else {+ None+ }+ });++ match state+ .searcher+ .search(query.as_str(), selected_region, None, skip_pages)+ {+ Ok(result) => Json(result),+ Err(_) => panic!("Search failed"), // TODO: show 500 status to user here+ }+}diff --git a/src/frontend/mod.rs b/src/frontend/mod.rsindex 458b4bb7..93ed84bd 100644--- a/src/frontend/mod.rs+++ b/src/frontend/mod.rs@@ -33,6 +33,7 @@ use axum::{use axum_extra::routing::SpaRouter;mod about;+mod api;mod autosuggest;mod entity_image;mod favicons;@@ -101,6 +102,7 @@ pub fn router(.route("/favicon.ico", get(favicon)).route("/about", get(about::route)).route("/privacy-and-happy-lawyers", get(privacy::route))+ .route("/api/beta/search", get(api::search)).merge(SpaRouter::new("/static", "static")).layer(Extension(state)).layer(CompressionLayer::new()))diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex cf12fd68..9bc94d65 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use chrono::NaiveDateTime;+use serde::Serialize;use tantivy::collector::{Collector, Count};use tantivy::merge_policy::NoMergePolicy;use tantivy::schema::Schema;@@ -44,13 +45,15 @@ impl InvertedIndex {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let schema = create_schema();- let tantivy_index = if path.as_ref().exists() {+ let mut tantivy_index = if path.as_ref().exists() {tantivy::Index::open_in_dir(&path)?} else {fs::create_dir_all(&path)?;tantivy::Index::create_in_dir(&path, schema.clone())?};+ tantivy_index.set_default_multithread_executor()?;+let tokenizer = Tokenizer::default();tantivy_index.tokenizers()@@ -227,13 +230,13 @@ impl InvertedIndex {}}-#[derive(Debug)]+#[derive(Debug, Serialize)]pub struct InvertedIndexSearchResult {pub num_docs: usize,pub documents: Vec<RetrievedWebpage>,}-#[derive(Default, Debug)]+#[derive(Default, Debug, Serialize)]pub struct RetrievedWebpage {pub title: String,pub url: String,diff --git a/src/searcher.rs b/src/searcher.rsindex 4d3975c1..dbdef47e 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -17,6 +17,7 @@use std::str::FromStr;use std::time::Instant;+use serde::Serialize;use uuid::Uuid;use crate::bangs::{BangHit, Bangs};@@ -33,7 +34,7 @@ use crate::{Error, Result};pub const NUM_RESULTS_PER_PAGE: usize = 20;-#[derive(Debug)]+#[derive(Debug, Serialize)]pub struct WebsitesResult {pub spell_corrected_query: Option<String>,pub webpages: InvertedIndexSearchResult,@@ -41,7 +42,7 @@ pub struct WebsitesResult {pub search_duration_ms: u128,}-#[derive(Debug)]+#[derive(Debug, Serialize)]pub enum SearchResult {Websites(WebsitesResult),Bang(BangHit),diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex b0932de9..fe4e4037 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -16,9 +16,10 @@use std::{fmt::Display, time::Duration};+use serde::Serialize;use tracing::debug;-#[derive(Debug, PartialEq, Eq, Clone, Hash)]+#[derive(Debug, Serialize, PartialEq, Eq, Clone, Hash)]pub struct Url(String);impl Display for Url {