Skip to content

web search done right

json api

Mikkel Denker02559946c63e
diff --git a/Cargo.lock b/Cargo.lock
index b9211342..85511793 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -388,6 +388,7 @@ dependencies = [
"js-sys",
"num-integer",
"num-traits",
+ "serde",
"time 0.1.44",
"wasm-bindgen",
"winapi",
diff --git a/Cargo.toml b/Cargo.toml
index 224274a6..823388a6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,7 +39,7 @@ csv = "1.1.6"
fst = { version = "0.4.7", features = ["levenshtein"] }
rocksdb = "0.19.0"
image = "0.24.3"
-chrono = "0.4.20"
+chrono = {version = "0.4.20", features = ["serde"] }
uuid = "1.1.2"
futures = "0.3.21"
quick-xml = "0.23.0"
diff --git a/src/bangs.rs b/src/bangs.rs
index c212adb6..a41025e8 100644
--- a/src/bangs.rs
+++ b/src/bangs.rs
@@ -50,7 +50,7 @@ pub struct Bang {
url: String,
}
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct BangHit {
bang: Bang,
pub redirect_to: Url,
diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rs
index ac463634..cd23cca2 100644
--- a/src/entity_index/mod.rs
+++ b/src/entity_index/mod.rs
@@ -22,6 +22,7 @@ use std::{
time::Duration,
};
+use serde::Serialize;
use tantivy::{
collector::TopDocs,
query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},
@@ -144,7 +145,7 @@ fn wikipedify_url(url: &Url) -> Vec<Url> {
]
}
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct StoredEntity {
pub title: String,
pub entity_abstract: String,
diff --git a/src/frontend/api.rs b/src/frontend/api.rs
new file mode 100644
index 00000000..97333707
--- /dev/null
+++ b/src/frontend/api.rs
@@ -0,0 +1,48 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::webpage::region::Region;
+
+use super::State;
+use axum::{extract, response::IntoResponse, Extension, Json};
+
+#[allow(clippy::unused_async)]
+pub async fn search(
+ extract::Query(params): extract::Query<HashMap<String, String>>,
+ Extension(state): Extension<Arc<State>>,
+) -> impl IntoResponse {
+ let query = params.get("q").cloned().unwrap_or_default();
+
+ let skip_pages = params.get("p").and_then(|p| p.parse().ok());
+
+ let selected_region = params.get("gl").and_then(|gl| {
+ if let Ok(region) = Region::from_gl(gl) {
+ Some(region)
+ } else {
+ None
+ }
+ });
+
+ match state
+ .searcher
+ .search(query.as_str(), selected_region, None, skip_pages)
+ {
+ Ok(result) => Json(result),
+ Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
+ }
+}
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index 458b4bb7..93ed84bd 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -33,6 +33,7 @@ use axum::{
use axum_extra::routing::SpaRouter;
mod about;
+mod api;
mod autosuggest;
mod entity_image;
mod favicons;
@@ -101,6 +102,7 @@ pub fn router(
.route("/favicon.ico", get(favicon))
.route("/about", get(about::route))
.route("/privacy-and-happy-lawyers", get(privacy::route))
+ .route("/api/beta/search", get(api::search))
.merge(SpaRouter::new("/static", "static"))
.layer(Extension(state))
.layer(CompressionLayer::new()))
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index cf12fd68..9bc94d65 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use chrono::NaiveDateTime;
+use serde::Serialize;
use tantivy::collector::{Collector, Count};
use tantivy::merge_policy::NoMergePolicy;
use tantivy::schema::Schema;
@@ -44,13 +45,15 @@ impl InvertedIndex {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let schema = create_schema();
- let tantivy_index = if path.as_ref().exists() {
+ let mut tantivy_index = if path.as_ref().exists() {
tantivy::Index::open_in_dir(&path)?
} else {
fs::create_dir_all(&path)?;
tantivy::Index::create_in_dir(&path, schema.clone())?
};
+ tantivy_index.set_default_multithread_executor()?;
+
let tokenizer = Tokenizer::default();
tantivy_index
.tokenizers()
@@ -227,13 +230,13 @@ impl InvertedIndex {
}
}
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
pub struct InvertedIndexSearchResult {
pub num_docs: usize,
pub documents: Vec<RetrievedWebpage>,
}
-#[derive(Default, Debug)]
+#[derive(Default, Debug, Serialize)]
pub struct RetrievedWebpage {
pub title: String,
pub url: String,
diff --git a/src/searcher.rs b/src/searcher.rs
index 4d3975c1..dbdef47e 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -17,6 +17,7 @@
use std::str::FromStr;
use std::time::Instant;
+use serde::Serialize;
use uuid::Uuid;
use crate::bangs::{BangHit, Bangs};
@@ -33,7 +34,7 @@ use crate::{Error, Result};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
pub struct WebsitesResult {
pub spell_corrected_query: Option<String>,
pub webpages: InvertedIndexSearchResult,
@@ -41,7 +42,7 @@ pub struct WebsitesResult {
pub search_duration_ms: u128,
}
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
pub enum SearchResult {
Websites(WebsitesResult),
Bang(BangHit),
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index b0932de9..fe4e4037 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -16,9 +16,10 @@
use std::{fmt::Display, time::Duration};
+use serde::Serialize;
use tracing::debug;
-#[derive(Debug, PartialEq, Eq, Clone, Hash)]
+#[derive(Debug, Serialize, PartialEq, Eq, Clone, Hash)]
pub struct Url(String);
impl Display for Url {