Skip to content

web search done right

Way better CLI

Mikkel Denkercb46c3417abd
diff --git a/.gitignore b/.gitignore
index 129ac8b8..e9230917 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,5 @@
/target
/Cargo.lock
+.DS_Store
flamegraph.svg
-warc_files/
-webgraph
-centrality
-index
+data
diff --git a/Cargo.toml b/Cargo.toml
index 962635ee..ce791c63 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,10 +11,6 @@ authors = ['Cuely ApS. <[email protected]>']
name = "cuely"
[dependencies]
-askama = "0.11.1"
-async-channel = "1.6.1"
-async-trait = "0.1.53"
-axum = "0.5.9"
bincode = "1.3.3"
clap = { version = "3.1.18", features = ["derive"] }
encoding_rs = "0.8.31"
@@ -36,11 +32,14 @@ sled = { version = "0.34.7", features = ["compression"] }
tantivy = "0.18.0"
thiserror = "1.0.31"
tl = "0.7.5"
-tokio = { version = "1.19.2", features = ["full"] }
toml = "0.5.9"
tracing = "0.1.34"
tracing-subscriber = "0.3.11"
whatlang = "0.16.0"
+tokio = { version = "1.19.2", features = ["full"] }
+askama = "0.11.1"
+axum = "0.5.9"
+anyhow = "1.0.58"
[dev-dependencies]
maplit = "1.0.2"
diff --git a/configs/centrality.toml b/configs/centrality.toml
deleted file mode 100644
index a7300b3b..00000000
--- a/configs/centrality.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-type = "Centrality"
-webgraph_path = "webgraph"
-output_path = "centrality"
\ No newline at end of file
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index e65aee09..4b4335d8 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,10 +1,7 @@
-type = "Indexer"
-mode = "Local"
-
warc_paths_file = "warc.paths"
centrality_store_path = "centrality"
[warc_source]
type = "Local"
folder = "./warc_files"
-names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]
\ No newline at end of file
+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
\ No newline at end of file
diff --git a/configs/indexer/master.toml b/configs/indexer/master.toml
index 5fae3cff..0122d89b 100644
--- a/configs/indexer/master.toml
+++ b/configs/indexer/master.toml
@@ -1,6 +1,3 @@
-type = "Indexer"
-mode = "Master"
-
limit_warc_files = 100
workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]
diff --git a/configs/indexer/worker.toml b/configs/indexer/worker.toml
deleted file mode 100644
index 46172815..00000000
--- a/configs/indexer/worker.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-type = "Indexer"
-mode = "Worker"
-centrality_store_path = "centrality"
\ No newline at end of file
diff --git a/configs/server.toml b/configs/server.toml
deleted file mode 100644
index 7e5b1d47..00000000
--- a/configs/server.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-type = "Server"
-index_path= "index/CC-MAIN-20220116093137-20220116123137-00001.warc.gz/"
\ No newline at end of file
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
deleted file mode 100644
index 8d97ff25..00000000
--- a/src/bin/cuely.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use clap::Parser;
-use cuely::entrypoint::{CentralityEntrypoint, Indexer, ServerEntrypoint, WebgraphEntrypoint};
-use cuely::Config;
-use std::fs;
-use tracing::Level;
-use tracing_subscriber::FmtSubscriber;
-
-#[derive(Parser, Debug)]
-struct Args {
- config_file: String,
- mapreduce_worker_addr: Option<String>,
-}
-
-#[tokio::main]
-async fn main() {
- let subscriber = FmtSubscriber::builder()
- .with_max_level(Level::INFO)
- .finish();
- tracing::subscriber::set_global_default(subscriber).unwrap();
-
- let args = Args::parse();
-
- let raw_config = fs::read_to_string(args.config_file).expect("Failed to read config file");
-
- let config: Config = toml::from_str(&raw_config).expect("Failed to parse config");
-
- match config {
- Config::Indexer(config) => {
- Indexer::new(config, args.mapreduce_worker_addr)
- .run()
- .expect("Failed to index documents");
- }
- Config::Webgraph(config) => WebgraphEntrypoint::new(config, args.mapreduce_worker_addr)
- .run()
- .expect("Failed to build webgraph"),
-
- Config::Centrality(config) => CentralityEntrypoint::from(config).run(),
- Config::Server(config) => ServerEntrypoint::from(config).run().await.unwrap(),
- }
-}
diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rs
index 0d85a9ef..f7a40fbb 100644
--- a/src/entrypoint/centrality.rs
+++ b/src/entrypoint/centrality.rs
@@ -14,27 +14,17 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{
- ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder, CentralityConfig,
-};
+use std::path::Path;
-pub struct CentralityEntrypoint {
- config: CentralityConfig,
-}
+use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};
-impl From<CentralityConfig> for CentralityEntrypoint {
- fn from(config: CentralityConfig) -> Self {
- Self { config }
- }
-}
+pub struct CentralityEntrypoint {}
impl CentralityEntrypoint {
- pub fn run(self) {
- let graph = WebgraphBuilder::new(&self.config.webgraph_path)
- .with_host_graph()
- .open();
+ pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
+ let graph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();
- let mut centrality_store = CentralityStore::new(&self.config.output_path);
+ let mut centrality_store = CentralityStore::new(output_path);
centrality_store.append(
graph
diff --git a/src/server.rs b/src/entrypoint/frontend.rs
similarity index 70%
rename from src/server.rs
rename to src/entrypoint/frontend.rs
index d0ae6e6e..4a3eda82 100644
--- a/src/server.rs
+++ b/src/entrypoint/frontend.rs
@@ -14,8 +14,17 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::index::Index;
+use anyhow::Result;
-pub struct Server {
- pub index: Index,
+use crate::frontend::router;
+
+pub async fn run(index_path: &str, host: &str) -> Result<()> {
+ let app = router(index_path)?;
+ let addr = host.parse()?;
+ tracing::info!("listening on {}", addr);
+ axum::Server::bind(&addr)
+ .serve(app.into_make_service())
+ .await?;
+
+ Ok(())
}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 52ba8ca2..f8ee8558 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -25,14 +25,10 @@ use crate::ranking::centrality_store::CentralityStore;
use crate::warc::WarcFile;
use crate::webpage::{Html, Link, Webpage};
use crate::{
- HttpConfig, IndexingConfig, IndexingLocalConfig, IndexingMasterConfig, IndexingWorkerConfig,
- LocalConfig, Result, WarcSource,
+ HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig, Result, WarcSource,
};
-pub struct Indexer {
- config: IndexingConfig,
- worker_addr: Option<String>,
-}
+pub struct Indexer {}
#[derive(Debug, Serialize, Deserialize, Clone)]
enum JobConfig {
@@ -85,6 +81,9 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
trace!("inserting webpage: {:?}", html.url());
+ trace!("title = {:?}", html.title());
+ trace!("text = {:?}", html.text());
+
let webpage = Webpage {
html,
backlinks,
@@ -118,13 +117,7 @@ impl Reduce<Index> for Index {
}
impl Indexer {
- pub fn new(config: IndexingConfig, worker_addr: Option<String>) -> Self {
- Self {
- config,
- worker_addr,
- }
- }
- fn run_master(config: &IndexingMasterConfig) -> Result<()> {
+ pub fn run_master(config: &IndexingMasterConfig) -> Result<()> {
info!("Running master for index construction");
let warc_paths = config.warc_source.paths()?;
@@ -147,9 +140,9 @@ impl Indexer {
config: job_config.clone(),
warc_path,
base_path: config
- .graph_base_path
+ .index_base_path
.clone()
- .unwrap_or_else(|| "index".to_string()),
+ .unwrap_or_else(|| "data/index".to_string()),
}
}));
@@ -164,8 +157,8 @@ impl Indexer {
Ok(())
}
- fn run_worker(worker_addr: String, config: &IndexingWorkerConfig) -> Result<()> {
- IndexingWorker::new(config.centrality_store_path.clone()).run::<Job, FrozenIndex>(
+ pub fn run_worker(worker_addr: String, centrality_store_path: String) -> Result<()> {
+ IndexingWorker::new(centrality_store_path).run::<Job, FrozenIndex>(
worker_addr
.parse::<SocketAddr>()
.expect("Could not parse worker address"),
@@ -173,7 +166,7 @@ impl Indexer {
Ok(())
}
- fn run_locally(config: &IndexingLocalConfig) -> Result<()> {
+ pub fn run_locally(config: &IndexingLocalConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
let job_config = match config.warc_source.clone() {
@@ -189,7 +182,7 @@ impl Indexer {
.map(|path| Job {
config: job_config.clone(),
warc_path: path,
- base_path: "index".to_string(),
+ base_path: "data/index".to_string(),
})
.map(|job| job.map(&worker))
.fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {
@@ -199,17 +192,4 @@ impl Indexer {
Ok(())
}
-
- pub fn run(&self) -> Result<()> {
- match &self.config {
- IndexingConfig::Master(config) => Self::run_master(config),
- IndexingConfig::Worker(config) => Self::run_worker(
- self.worker_addr
- .clone()
- .expect("Worker address not specified"),
- config,
- ),
- IndexingConfig::Local(config) => Self::run_locally(config),
- }
- }
}
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index 88fd1614..220fc6a6 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -14,11 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod centrality;
+pub mod frontend;
mod indexer;
-mod server;
mod webgraph;
pub use centrality::CentralityEntrypoint;
pub use indexer::Indexer;
-pub use server::ServerEntrypoint;
pub use webgraph::WebgraphEntrypoint;
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 3140f606..7493c3f9 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -18,8 +18,7 @@ use crate::{
warc::WarcFile,
webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},
webpage::{self, Html},
- HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,
- WebgraphMasterConfig,
+ HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,
};
use serde::{Deserialize, Serialize};
use std::{net::SocketAddr, path::Path};
@@ -111,19 +110,10 @@ impl Reduce<Webgraph> for Webgraph {
}
}
-pub struct WebgraphEntrypoint {
- config: WebgraphConfig,
- worker_addr: Option<String>,
-}
+pub struct WebgraphEntrypoint {}
impl WebgraphEntrypoint {
- pub fn new(config: WebgraphConfig, worker_addr: Option<String>) -> Self {
- Self {
- config,
- worker_addr,
- }
- }
- fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
+ pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
info!("Running master for webgraph construction");
let warc_paths = config.warc_source.paths()?;
@@ -148,7 +138,7 @@ impl WebgraphEntrypoint {
graph_base_path: config
.graph_base_path
.clone()
- .unwrap_or_else(|| "webgraph".to_string()),
+ .unwrap_or_else(|| "data/webgraph".to_string()),
}
}));
@@ -163,7 +153,7 @@ impl WebgraphEntrypoint {
Ok(())
}
- fn run_worker(worker_addr: String) -> Result<()> {
+ pub fn run_worker(worker_addr: String) -> Result<()> {
StatelessWorker::default().run::<Job, FrozenWebgraph>(
worker_addr
.parse::<SocketAddr>()
@@ -172,7 +162,7 @@ impl WebgraphEntrypoint {
Ok(())
}
- fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {
+ pub fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
let job_config = match config.warc_source.clone() {
@@ -187,7 +177,7 @@ impl WebgraphEntrypoint {
.map(|path| Job {
config: job_config.clone(),
warc_path: path,
- graph_base_path: "webgraph".to_string(),
+ graph_base_path: "data/webgraph".to_string(),
})
.map(|job| job.map(&worker))
.fold(
@@ -200,16 +190,4 @@ impl WebgraphEntrypoint {
Ok(())
}
-
- pub fn run(&self) -> Result<()> {
- match &self.config {
- WebgraphConfig::Master(config) => WebgraphEntrypoint::run_master(config),
- WebgraphConfig::Worker => WebgraphEntrypoint::run_worker(
- self.worker_addr
- .clone()
- .expect("Worker address not specified"),
- ),
- WebgraphConfig::Local(config) => WebgraphEntrypoint::run_locally(config),
- }
- }
}
diff --git a/src/entrypoint/server.rs b/src/frontend/mod.rs
similarity index 68%
rename from src/entrypoint/server.rs
rename to src/frontend/mod.rs
index 61220a72..4811fdb5 100644
--- a/src/entrypoint/server.rs
+++ b/src/frontend/mod.rs
@@ -14,15 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::net::SocketAddr;
-
use axum::{Extension, Router};
-use crate::{
- index::{Index, RetrievedWebpage},
- server::Server,
- Result, ServerConfig,
-};
+use crate::index::{Index, RetrievedWebpage};
+use anyhow::Result;
use std::collections::HashMap;
use std::sync::Arc;
@@ -36,38 +31,9 @@ use axum::{
routing::get,
};
-pub struct ServerEntrypoint {
- config: ServerConfig,
-}
-
-impl From<ServerConfig> for ServerEntrypoint {
- fn from(config: ServerConfig) -> Self {
- Self { config }
- }
-}
-
-impl ServerEntrypoint {
- pub async fn run(self) -> Result<()> {
- let search_index = Index::open(&self.config.index_path)?;
- let state = Arc::new(Server {
- index: search_index,
- });
-
- let app = Router::new().route("/", get(index)).layer(Extension(state));
- let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
- tracing::debug!("listening on {}", addr);
- axum::Server::bind(&addr)
- .serve(app.into_make_service())
- .await
- .unwrap();
-
- Ok(())
- }
-}
-
async fn index(
extract::Query(params): extract::Query<HashMap<String, String>>,
- Extension(state): Extension<Arc<Server>>,
+ Extension(state): Extension<Arc<State>>,
) -> impl IntoResponse {
let mut search_result = Vec::new();
@@ -80,7 +46,7 @@ async fn index(
.search(&query, ranker.collector())
.expect("Search failed");
- search_result = result.documents;
+ search_result = dbg!(result.documents);
}
let template = IndexTemplate { search_result };
@@ -88,13 +54,17 @@ async fn index(
}
#[derive(Template)]
-#[template(path = "index.html")]
+#[template(path = "index.html", escape = "none")]
struct IndexTemplate {
search_result: Vec<RetrievedWebpage>,
}
struct HtmlTemplate<T>(T);
+pub struct State {
+ pub index: Index,
+}
+
impl<T> IntoResponse for HtmlTemplate<T>
where
T: Template,
@@ -110,3 +80,12 @@ where
}
}
}
+
+pub fn router(index_path: &str) -> Result<Router> {
+ let search_index = Index::open(index_path)?;
+ let state = Arc::new(State {
+ index: search_index,
+ });
+
+ Ok(Router::new().route("/", get(index)).layer(Extension(state)))
+}
diff --git a/src/index.rs b/src/index.rs
index b5ea269f..6442f38f 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -146,13 +146,16 @@ impl Index {
meta.segments.push(segment);
}
+ meta.segments
+ .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));
+
fs::remove_dir_all(other_path).unwrap();
let self_path = Path::new(&path);
std::fs::write(
self_path.join("meta.json"),
- serde_json::to_string(&meta).unwrap(),
+ serde_json::to_string_pretty(&meta).unwrap(),
)
.unwrap();
@@ -262,6 +265,9 @@ mod tests {
<head>
<title>Test website</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -292,6 +298,9 @@ mod tests {
<head>
<title>Test website</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -321,6 +330,9 @@ mod tests {
<head>
<title>Website for runners</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -350,6 +362,9 @@ mod tests {
<head>
<title>Fast runner</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -394,6 +409,9 @@ mod tests {
<head>
<title>Website B</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.b.com",
@@ -435,6 +453,9 @@ mod tests {
<head>
<title>Website for runners</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -465,6 +486,9 @@ mod tests {
<head>
<title>News website</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.dr.dk",
@@ -492,6 +516,9 @@ mod tests {
<head>
<title>Test website</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -530,6 +557,9 @@ mod tests {
<head>
<title>Test website</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -547,6 +577,9 @@ mod tests {
<head>
<title>Test website</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.example.com",
@@ -555,7 +588,8 @@ mod tests {
))
.expect("failed to parse webpage");
- let index = index1.merge(index2);
+ let mut index = index1.merge(index2);
+ index.commit().unwrap();
let query = Query::parse("website").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
diff --git a/src/lib.rs b/src/lib.rs
index de3f3f2c..e1e46740 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -28,56 +28,23 @@ pub mod mapreduce;
mod directory;
mod exponential_backoff;
-mod query;
+mod frontend;
+pub mod query;
pub mod ranking;
mod schema;
pub mod searcher;
-pub mod server;
mod snippet;
mod tokenizer;
mod warc;
pub mod webgraph;
pub mod webpage;
-#[derive(Debug, Deserialize, Clone)]
-#[serde(tag = "type")]
-pub enum Config {
- Indexer(IndexingConfig),
- Webgraph(WebgraphConfig),
- Centrality(CentralityConfig),
- Server(ServerConfig),
-}
-
-#[derive(Debug, Deserialize, Clone)]
-pub struct ServerConfig {
- pub index_path: String,
-}
-
-#[derive(Debug, Deserialize, Clone)]
-pub struct CentralityConfig {
- pub webgraph_path: String,
- pub output_path: String,
-}
-
-#[derive(Debug, Deserialize, Clone)]
-#[serde(tag = "mode")]
-pub enum IndexingConfig {
- Master(IndexingMasterConfig),
- Worker(IndexingWorkerConfig),
- Local(IndexingLocalConfig),
-}
-
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingMasterConfig {
limit_warc_files: Option<usize>,
warc_source: WarcSource,
workers: Vec<String>,
- graph_base_path: Option<String>,
-}
-
-#[derive(Debug, Deserialize, Clone)]
-pub struct IndexingWorkerConfig {
- centrality_store_path: String,
+ index_base_path: Option<String>,
}
#[derive(Debug, Deserialize, Clone)]
@@ -86,14 +53,6 @@ pub struct IndexingLocalConfig {
centrality_store_path: String,
}
-#[derive(Debug, Deserialize, Clone)]
-#[serde(tag = "mode")]
-pub enum WebgraphConfig {
- Master(WebgraphMasterConfig),
- Worker,
- Local(WebgraphLocalConfig),
-}
-
#[derive(Debug, Deserialize, Clone)]
pub struct WebgraphMasterConfig {
limit_warc_files: Option<usize>,
diff --git a/src/main.rs b/src/main.rs
new file mode 100644
index 00000000..00913c4e
--- /dev/null
+++ b/src/main.rs
@@ -0,0 +1,127 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+use cuely::entrypoint::{frontend, CentralityEntrypoint, Indexer, WebgraphEntrypoint};
+use serde::de::DeserializeOwned;
+use std::fs;
+use std::path::Path;
+use tracing::Level;
+use tracing_subscriber::FmtSubscriber;
+
+#[derive(Parser)]
+#[clap(author, version, about, long_about = None)]
+#[clap(propagate_version = true)]
+struct Args {
+ #[clap(subcommand)]
+ command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+ Indexer {
+ #[clap(subcommand)]
+ options: IndexingOptions,
+ },
+ Centrality {
+ webgraph_path: String,
+ output_path: String,
+ },
+ Webgraph {
+ #[clap(subcommand)]
+ options: WebgraphOptions,
+ },
+ Frontend {
+ index_path: String,
+ #[clap(default_value = "0.0.0.0:3000")]
+ host: String,
+ },
+}
+
+#[derive(Subcommand)]
+enum WebgraphOptions {
+ Master { config_path: String },
+ Worker { address: String },
+ Local { config_path: String },
+}
+
+#[derive(Subcommand)]
+enum IndexingOptions {
+ Master {
+ config_path: String,
+ },
+ Worker {
+ address: String,
+ centrality_store_path: String,
+ },
+ Local {
+ config_path: String,
+ },
+}
+
+fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {
+ let raw_config = fs::read_to_string(path).expect("Failed to read config file");
+ toml::from_str(&raw_config).expect("Failed to parse config")
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let subscriber = FmtSubscriber::builder()
+ .with_max_level(Level::INFO)
+ .finish();
+ tracing::subscriber::set_global_default(subscriber).unwrap();
+
+ let args = Args::parse();
+
+ match args.command {
+ Commands::Indexer { options } => match options {
+ IndexingOptions::Master { config_path } => {
+ let config = load_toml_config(&config_path);
+ Indexer::run_master(&config)?;
+ }
+ IndexingOptions::Worker {
+ address,
+ centrality_store_path,
+ } => {
+ Indexer::run_worker(address, centrality_store_path)?;
+ }
+ IndexingOptions::Local { config_path } => {
+ let config = load_toml_config(&config_path);
+ Indexer::run_locally(&config)?;
+ }
+ },
+ Commands::Centrality {
+ webgraph_path,
+ output_path,
+ } => CentralityEntrypoint::run(webgraph_path, output_path),
+ Commands::Webgraph { options } => match options {
+ WebgraphOptions::Master { config_path } => {
+ let config = load_toml_config(config_path);
+ WebgraphEntrypoint::run_master(&config)?;
+ }
+ WebgraphOptions::Worker { address } => {
+ WebgraphEntrypoint::run_worker(address)?;
+ }
+ WebgraphOptions::Local { config_path } => {
+ let config = load_toml_config(config_path);
+ WebgraphEntrypoint::run_locally(&config)?;
+ }
+ },
+ Commands::Frontend { index_path, host } => frontend::run(&index_path, &host).await?,
+ }
+
+ Ok(())
+}
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index d5e82e30..c55f919c 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -38,9 +38,6 @@ pub enum Error {
#[error("could not get a working worker")]
NoAvailableWorker,
- #[error("failed to get value from channel")]
- ChannelRecv(#[from] async_channel::RecvError),
-
#[error("did not get a reponse")]
NoResponse,
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index f1f50334..8bdef605 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -34,7 +34,7 @@ impl Ranker {
}
}
- pub(crate) fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
+ pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
let score_tweaker = InitialScoreTweaker::new(self.query.clone());
TopDocs::with_limit(20).tweak_score(score_tweaker)
}
@@ -75,6 +75,9 @@ mod tests {
<head>
<title>Website B</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.b.com",
@@ -107,6 +110,9 @@ mod tests {
<head>
<title>DR Homepage</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.dr.dk",
@@ -121,6 +127,9 @@ mod tests {
<head>
<title>Subsite dr</title>
</head>
+ <body>
+ body
+ </body>
</html>
"#,
"https://www.dr.dk/whatever",
diff --git a/src/webpage.rs b/src/webpage.rs
index 3fd4295a..a0ed0e6b 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -210,7 +210,7 @@ impl<'a> Html<'a> {
.collect::<Vec<String>>()
}
- pub fn text(&self) -> String {
+ pub fn text(&self) -> Option<String> {
let strings: Vec<String> = match &self.dom {
Dom::ScraperHtml(dom) => {
let selectors = vec![
@@ -249,10 +249,19 @@ impl<'a> Html<'a> {
.unwrap_or_default(),
};
- Itertools::intersperse(strings.into_iter(), "\n".to_string())
- .collect::<String>()
- .trim()
- .to_string()
+ let s = Itertools::intersperse(
+ strings.into_iter().filter(|s| !s.is_empty()),
+ "\n".to_string(),
+ )
+ .collect::<String>()
+ .trim()
+ .to_string();
+
+ if s.is_empty() {
+ None
+ } else {
+ Some(s)
+ }
}
fn children_text(children: Children, parser: &tl::Parser) -> Vec<String> {
@@ -371,7 +380,13 @@ impl<'a> Html<'a> {
doc.add_text(tantivy_field, self.title().unwrap())
}
- Field::Body => doc.add_text(tantivy_field, self.text()),
+ Field::Body => {
+ let text = self.text();
+ if text.is_none() {
+ return Err(Error::EmptyField("body"));
+ }
+ doc.add_text(tantivy_field, text.unwrap())
+ }
Field::Url => doc.add_text(tantivy_field, self.url()),
Field::FastUrl => doc.add_bytes(
tantivy_field,
@@ -424,7 +439,10 @@ mod tests {
text: "Best example website ever".to_string()
}]
);
- assert_eq!(&webpage.text(), "Best example website ever");
+ assert_eq!(
+ webpage.text(),
+ Some("Best example website ever".to_string())
+ );
assert_eq!(webpage.title(), Some("Best website".to_string()));
let mut expected_meta = BTreeMap::new();
@@ -449,7 +467,7 @@ mod tests {
let webpage = Html::parse(raw, "https://www.example.com/whatever");
- assert_eq!(&webpage.text(), "test website");
+ assert_eq!(webpage.text(), Some("test website".to_string()));
}
#[test]
@@ -475,7 +493,7 @@ mod tests {
let webpage = Html::parse(raw, "https://www.example.com");
- assert!(!webpage.text().contains("not"));
+ assert!(!webpage.text().unwrap().contains("not"));
}
#[test]
@@ -501,7 +519,7 @@ mod tests {
let webpage = Html::parse(raw, "https://www.example.com");
- assert!(!webpage.text().contains("not"));
+ assert!(!webpage.text().unwrap().contains("not"));
}
#[test]