web search done right
Very simple searching working. Still a lot of bugs to figure out, but it actually works
Mikkel Denker085d6cae555a
diff --git a/Cargo.toml b/Cargo.tomlindex b4da6c4d..962635ee 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,8 +11,10 @@ authors = ['Cuely ApS. <[email protected]>']name = "cuely"[dependencies]+askama = "0.11.1"async-channel = "1.6.1"async-trait = "0.1.53"+axum = "0.5.9"bincode = "1.3.3"clap = { version = "3.1.18", features = ["derive"] }encoding_rs = "0.8.31"@@ -34,6 +36,7 @@ sled = { version = "0.34.7", features = ["compression"] }tantivy = "0.18.0"thiserror = "1.0.31"tl = "0.7.5"+tokio = { version = "1.19.2", features = ["full"] }toml = "0.5.9"tracing = "0.1.34"tracing-subscriber = "0.3.11"diff --git a/configs/indexer.toml b/configs/indexer/local.tomlsimilarity index 72%rename from configs/indexer.tomlrename to configs/indexer/local.tomlindex 3b2b1f95..e65aee09 100644--- a/configs/indexer.toml+++ b/configs/indexer/local.toml@@ -7,6 +7,4 @@ centrality_store_path = "centrality"[warc_source]type = "Local"folder = "./warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]-# type = "HTTP"-# base_url = "https://data.commoncrawl.org/"\ No newline at end of file+names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]\ No newline at end of filediff --git a/configs/indexer/master.toml b/configs/indexer/master.tomlnew file mode 100644index 00000000..5fae3cff--- /dev/null+++ b/configs/indexer/master.toml@@ -0,0 +1,10 @@+type = "Indexer"+mode = "Master"++limit_warc_files = 100+workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]++[warc_source]+type = "HTTP"+base_url = "https://data.commoncrawl.org/"+warc_paths_file = "warc.paths"\ No newline at end of filediff --git a/configs/indexer/worker.toml b/configs/indexer/worker.tomlnew file mode 100644index 00000000..46172815--- /dev/null+++ b/configs/indexer/worker.toml@@ -0,0 +1,3 @@+type = "Indexer"+mode = "Worker"+centrality_store_path = "centrality"\ No newline at end of filediff --git a/configs/server.toml b/configs/server.tomlnew file mode 100644index 00000000..7e5b1d47--- /dev/null+++ b/configs/server.toml@@ -0,0 +1,2 @@+type = "Server"+index_path= "index/CC-MAIN-20220116093137-20220116123137-00001.warc.gz/"\ No newline at end of filediff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex 6b80bff8..8d97ff25 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use clap::Parser;-use cuely::entrypoint::{CentralityEntrypoint, Indexer, WebgraphEntrypoint};+use cuely::entrypoint::{CentralityEntrypoint, Indexer, ServerEntrypoint, WebgraphEntrypoint};use cuely::Config;use std::fs;use tracing::Level;@@ -26,7 +26,8 @@ struct Args {mapreduce_worker_addr: Option<String>,}-fn main() {+#[tokio::main]+async fn main() {let subscriber = FmtSubscriber::builder().with_max_level(Level::INFO).finish();@@ -49,5 +50,6 @@ fn main() {.expect("Failed to build webgraph"),Config::Centrality(config) => CentralityEntrypoint::from(config).run(),+ Config::Server(config) => ServerEntrypoint::from(config).run().await.unwrap(),}}diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex c7ca9826..88fd1614 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -15,8 +15,10 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.mod centrality;mod indexer;+mod server;mod webgraph;pub use centrality::CentralityEntrypoint;pub use indexer::Indexer;+pub use server::ServerEntrypoint;pub use webgraph::WebgraphEntrypoint;diff --git a/src/entrypoint/server.rs b/src/entrypoint/server.rsnew file mode 100644index 00000000..61220a72--- /dev/null+++ b/src/entrypoint/server.rs@@ -0,0 +1,112 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use axum::{Extension, Router};++use crate::{+ index::{Index, RetrievedWebpage},+ server::Server,+ Result, ServerConfig,+};+use std::collections::HashMap;+use std::sync::Arc;++use crate::query::Query;+use crate::ranking::Ranker;+use askama::Template;+use axum::{+ extract,+ http::StatusCode,+ response::{Html, IntoResponse, Response},+ routing::get,+};++pub struct ServerEntrypoint {+ config: ServerConfig,+}++impl From<ServerConfig> for ServerEntrypoint {+ fn from(config: ServerConfig) -> Self {+ Self { config }+ }+}++impl ServerEntrypoint {+ pub async fn run(self) -> Result<()> {+ let search_index = Index::open(&self.config.index_path)?;+ let state = Arc::new(Server {+ index: search_index,+ });++ let app = Router::new().route("/", get(index)).layer(Extension(state));+ let addr = SocketAddr::from(([127, 0, 0, 1], 3000));+ tracing::debug!("listening on {}", addr);+ axum::Server::bind(&addr)+ .serve(app.into_make_service())+ .await+ .unwrap();++ Ok(())+ }+}++async fn index(+ extract::Query(params): extract::Query<HashMap<String, String>>,+ Extension(state): Extension<Arc<Server>>,+) -> impl IntoResponse {+ let mut search_result = Vec::new();++ if let Some(query) = params.get("q") {+ let query = Query::parse(query).expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());++ let result = state+ .index+ .search(&query, ranker.collector())+ .expect("Search failed");++ search_result = result.documents;+ }++ let template = IndexTemplate { search_result };+ HtmlTemplate(template)+}++#[derive(Template)]+#[template(path = "index.html")]+struct IndexTemplate {+ search_result: Vec<RetrievedWebpage>,+}++struct HtmlTemplate<T>(T);++impl<T> IntoResponse for HtmlTemplate<T>+where+ T: Template,+{+ fn into_response(self) -> Response {+ match self.0.render() {+ Ok(html) => Html(html).into_response(),+ Err(err) => (+ StatusCode::INTERNAL_SERVER_ERROR,+ format!("Failed to render template. Error: {}", err),+ )+ .into_response(),+ }+ }+}diff --git a/src/lib.rs b/src/lib.rsindex 0bef245c..de3f3f2c 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -32,6 +32,7 @@ mod query;pub mod ranking;mod schema;pub mod searcher;+pub mod server;mod snippet;mod tokenizer;mod warc;@@ -44,6 +45,12 @@ pub enum Config {Indexer(IndexingConfig),Webgraph(WebgraphConfig),Centrality(CentralityConfig),+ Server(ServerConfig),+}++#[derive(Debug, Deserialize, Clone)]+pub struct ServerConfig {+ pub index_path: String,}#[derive(Debug, Deserialize, Clone)]diff --git a/src/searcher.rs b/src/searcher.rsindex c6862c47..9c2fba94 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -19,6 +19,7 @@ use crate::query::Query;use crate::ranking::Ranker;use crate::Result;+#[derive(Debug)]pub struct SearchResult {pub num_docs: usize,pub documents: Vec<RetrievedWebpage>,diff --git a/src/server.rs b/src/server.rsnew file mode 100644index 00000000..d0ae6e6e--- /dev/null+++ b/src/server.rs@@ -0,0 +1,21 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::index::Index;++pub struct Server {+ pub index: Index,+}diff --git a/templates/index.html b/templates/index.htmlnew file mode 100644index 00000000..3cd4b4c8--- /dev/null+++ b/templates/index.html@@ -0,0 +1,11 @@+<form method="GET">+ <input name="q" />+ <input type="submit" />+</form>++{% for item in search_result %}+<div>+ <p><a href="{{ item.url }}">{{ item.title }}</a></p>+ <p>{{ item.snippet }} </p>+</div>+{% endfor %}\ No newline at end of file