Skip to content

web search done right

Very simple searching working. Still a lot of bugs to figure out, but it actually works

Mikkel Denker085d6cae555a
diff --git a/Cargo.toml b/Cargo.toml
index b4da6c4d..962635ee 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,8 +11,10 @@ authors = ['Cuely ApS. <[email protected]>']
name = "cuely"
[dependencies]
+askama = "0.11.1"
async-channel = "1.6.1"
async-trait = "0.1.53"
+axum = "0.5.9"
bincode = "1.3.3"
clap = { version = "3.1.18", features = ["derive"] }
encoding_rs = "0.8.31"
@@ -34,6 +36,7 @@ sled = { version = "0.34.7", features = ["compression"] }
tantivy = "0.18.0"
thiserror = "1.0.31"
tl = "0.7.5"
+tokio = { version = "1.19.2", features = ["full"] }
toml = "0.5.9"
tracing = "0.1.34"
tracing-subscriber = "0.3.11"
diff --git a/configs/indexer.toml b/configs/indexer/local.toml
similarity index 72%
rename from configs/indexer.toml
rename to configs/indexer/local.toml
index 3b2b1f95..e65aee09 100644
--- a/configs/indexer.toml
+++ b/configs/indexer/local.toml
@@ -7,6 +7,4 @@ centrality_store_path = "centrality"
[warc_source]
type = "Local"
folder = "./warc_files"
-names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]
-# type = "HTTP"
-# base_url = "https://data.commoncrawl.org/"
\ No newline at end of file
+names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]
\ No newline at end of file
diff --git a/configs/indexer/master.toml b/configs/indexer/master.toml
new file mode 100644
index 00000000..5fae3cff
--- /dev/null
+++ b/configs/indexer/master.toml
@@ -0,0 +1,10 @@
+type = "Indexer"
+mode = "Master"
+
+limit_warc_files = 100
+workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]
+
+[warc_source]
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
+warc_paths_file = "warc.paths"
\ No newline at end of file
diff --git a/configs/indexer/worker.toml b/configs/indexer/worker.toml
new file mode 100644
index 00000000..46172815
--- /dev/null
+++ b/configs/indexer/worker.toml
@@ -0,0 +1,3 @@
+type = "Indexer"
+mode = "Worker"
+centrality_store_path = "centrality"
\ No newline at end of file
diff --git a/configs/server.toml b/configs/server.toml
new file mode 100644
index 00000000..7e5b1d47
--- /dev/null
+++ b/configs/server.toml
@@ -0,0 +1,2 @@
+type = "Server"
+index_path= "index/CC-MAIN-20220116093137-20220116123137-00001.warc.gz/"
\ No newline at end of file
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index 6b80bff8..8d97ff25 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use clap::Parser;
-use cuely::entrypoint::{CentralityEntrypoint, Indexer, WebgraphEntrypoint};
+use cuely::entrypoint::{CentralityEntrypoint, Indexer, ServerEntrypoint, WebgraphEntrypoint};
use cuely::Config;
use std::fs;
use tracing::Level;
@@ -26,7 +26,8 @@ struct Args {
mapreduce_worker_addr: Option<String>,
}
-fn main() {
+#[tokio::main]
+async fn main() {
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::INFO)
.finish();
@@ -49,5 +50,6 @@ fn main() {
.expect("Failed to build webgraph"),
Config::Centrality(config) => CentralityEntrypoint::from(config).run(),
+ Config::Server(config) => ServerEntrypoint::from(config).run().await.unwrap(),
}
}
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index c7ca9826..88fd1614 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -15,8 +15,10 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod centrality;
mod indexer;
+mod server;
mod webgraph;
pub use centrality::CentralityEntrypoint;
pub use indexer::Indexer;
+pub use server::ServerEntrypoint;
pub use webgraph::WebgraphEntrypoint;
diff --git a/src/entrypoint/server.rs b/src/entrypoint/server.rs
new file mode 100644
index 00000000..61220a72
--- /dev/null
+++ b/src/entrypoint/server.rs
@@ -0,0 +1,112 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::net::SocketAddr;
+
+use axum::{Extension, Router};
+
+use crate::{
+ index::{Index, RetrievedWebpage},
+ server::Server,
+ Result, ServerConfig,
+};
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::query::Query;
+use crate::ranking::Ranker;
+use askama::Template;
+use axum::{
+ extract,
+ http::StatusCode,
+ response::{Html, IntoResponse, Response},
+ routing::get,
+};
+
+pub struct ServerEntrypoint {
+ config: ServerConfig,
+}
+
+impl From<ServerConfig> for ServerEntrypoint {
+ fn from(config: ServerConfig) -> Self {
+ Self { config }
+ }
+}
+
+impl ServerEntrypoint {
+ pub async fn run(self) -> Result<()> {
+ let search_index = Index::open(&self.config.index_path)?;
+ let state = Arc::new(Server {
+ index: search_index,
+ });
+
+ let app = Router::new().route("/", get(index)).layer(Extension(state));
+ let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
+ tracing::debug!("listening on {}", addr);
+ axum::Server::bind(&addr)
+ .serve(app.into_make_service())
+ .await
+ .unwrap();
+
+ Ok(())
+ }
+}
+
+async fn index(
+ extract::Query(params): extract::Query<HashMap<String, String>>,
+ Extension(state): Extension<Arc<Server>>,
+) -> impl IntoResponse {
+ let mut search_result = Vec::new();
+
+ if let Some(query) = params.get("q") {
+ let query = Query::parse(query).expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ let result = state
+ .index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+
+ search_result = result.documents;
+ }
+
+ let template = IndexTemplate { search_result };
+ HtmlTemplate(template)
+}
+
+#[derive(Template)]
+#[template(path = "index.html")]
+struct IndexTemplate {
+ search_result: Vec<RetrievedWebpage>,
+}
+
+struct HtmlTemplate<T>(T);
+
+impl<T> IntoResponse for HtmlTemplate<T>
+where
+ T: Template,
+{
+ fn into_response(self) -> Response {
+ match self.0.render() {
+ Ok(html) => Html(html).into_response(),
+ Err(err) => (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ format!("Failed to render template. Error: {}", err),
+ )
+ .into_response(),
+ }
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 0bef245c..de3f3f2c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -32,6 +32,7 @@ mod query;
pub mod ranking;
mod schema;
pub mod searcher;
+pub mod server;
mod snippet;
mod tokenizer;
mod warc;
@@ -44,6 +45,12 @@ pub enum Config {
Indexer(IndexingConfig),
Webgraph(WebgraphConfig),
Centrality(CentralityConfig),
+ Server(ServerConfig),
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct ServerConfig {
+ pub index_path: String,
}
#[derive(Debug, Deserialize, Clone)]
diff --git a/src/searcher.rs b/src/searcher.rs
index c6862c47..9c2fba94 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -19,6 +19,7 @@ use crate::query::Query;
use crate::ranking::Ranker;
use crate::Result;
+#[derive(Debug)]
pub struct SearchResult {
pub num_docs: usize,
pub documents: Vec<RetrievedWebpage>,
diff --git a/src/server.rs b/src/server.rs
new file mode 100644
index 00000000..d0ae6e6e
--- /dev/null
+++ b/src/server.rs
@@ -0,0 +1,21 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::index::Index;
+
+pub struct Server {
+ pub index: Index,
+}
diff --git a/templates/index.html b/templates/index.html
new file mode 100644
index 00000000..3cd4b4c8
--- /dev/null
+++ b/templates/index.html
@@ -0,0 +1,11 @@
+<form method="GET">
+ <input name="q" />
+ <input type="submit" />
+</form>
+
+{% for item in search_result %}
+<div>
+ <p><a href="{{ item.url }}">{{ item.title }}</a></p>
+ <p>{{ item.snippet }} </p>
+</div>
+{% endfor %}
\ No newline at end of file