web search done right
Way better CLI
Mikkel Denkercb46c3417abd
diff --git a/.gitignore b/.gitignoreindex 129ac8b8..e9230917 100644--- a/.gitignore+++ b/.gitignore@@ -1,7 +1,5 @@/target/Cargo.lock+.DS_Storeflamegraph.svg-warc_files/-webgraph-centrality-index+datadiff --git a/Cargo.toml b/Cargo.tomlindex 962635ee..ce791c63 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,10 +11,6 @@ authors = ['Cuely ApS. <[email protected]>']name = "cuely"[dependencies]-askama = "0.11.1"-async-channel = "1.6.1"-async-trait = "0.1.53"-axum = "0.5.9"bincode = "1.3.3"clap = { version = "3.1.18", features = ["derive"] }encoding_rs = "0.8.31"@@ -36,11 +32,14 @@ sled = { version = "0.34.7", features = ["compression"] }tantivy = "0.18.0"thiserror = "1.0.31"tl = "0.7.5"-tokio = { version = "1.19.2", features = ["full"] }toml = "0.5.9"tracing = "0.1.34"tracing-subscriber = "0.3.11"whatlang = "0.16.0"+tokio = { version = "1.19.2", features = ["full"] }+askama = "0.11.1"+axum = "0.5.9"+anyhow = "1.0.58"[dev-dependencies]maplit = "1.0.2"diff --git a/configs/centrality.toml b/configs/centrality.tomldeleted file mode 100644index a7300b3b..00000000--- a/configs/centrality.toml+++ /dev/null@@ -1,3 +0,0 @@-type = "Centrality"-webgraph_path = "webgraph"-output_path = "centrality"\ No newline at end of filediff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex e65aee09..4b4335d8 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,10 +1,7 @@-type = "Indexer"-mode = "Local"-warc_paths_file = "warc.paths"centrality_store_path = "centrality"[warc_source]type = "Local"folder = "./warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]\ No newline at end of file+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]\ No newline at end of filediff --git a/configs/indexer/master.toml b/configs/indexer/master.tomlindex 5fae3cff..0122d89b 100644--- a/configs/indexer/master.toml+++ b/configs/indexer/master.toml@@ -1,6 +1,3 @@-type = "Indexer"-mode = "Master"-limit_warc_files = 100workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]diff --git a/configs/indexer/worker.toml b/configs/indexer/worker.tomldeleted file mode 100644index 46172815..00000000--- a/configs/indexer/worker.toml+++ /dev/null@@ -1,3 +0,0 @@-type = "Indexer"-mode = "Worker"-centrality_store_path = "centrality"\ No newline at end of filediff --git a/configs/server.toml b/configs/server.tomldeleted file mode 100644index 7e5b1d47..00000000--- a/configs/server.toml+++ /dev/null@@ -1,2 +0,0 @@-type = "Server"-index_path= "index/CC-MAIN-20220116093137-20220116123137-00001.warc.gz/"\ No newline at end of filediff --git a/src/bin/cuely.rs b/src/bin/cuely.rsdeleted file mode 100644index 8d97ff25..00000000--- a/src/bin/cuely.rs+++ /dev/null@@ -1,55 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use clap::Parser;-use cuely::entrypoint::{CentralityEntrypoint, Indexer, ServerEntrypoint, WebgraphEntrypoint};-use cuely::Config;-use std::fs;-use tracing::Level;-use tracing_subscriber::FmtSubscriber;--#[derive(Parser, Debug)]-struct Args {- config_file: String,- mapreduce_worker_addr: Option<String>,-}--#[tokio::main]-async fn main() {- let subscriber = FmtSubscriber::builder()- .with_max_level(Level::INFO)- .finish();- tracing::subscriber::set_global_default(subscriber).unwrap();-- let args = Args::parse();-- let raw_config = fs::read_to_string(args.config_file).expect("Failed to read config file");-- let config: Config = toml::from_str(&raw_config).expect("Failed to parse config");-- match config {- Config::Indexer(config) => {- Indexer::new(config, args.mapreduce_worker_addr)- .run()- .expect("Failed to index documents");- }- Config::Webgraph(config) => WebgraphEntrypoint::new(config, args.mapreduce_worker_addr)- .run()- .expect("Failed to build webgraph"),-- Config::Centrality(config) => CentralityEntrypoint::from(config).run(),- Config::Server(config) => ServerEntrypoint::from(config).run().await.unwrap(),- }-}diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rsindex 0d85a9ef..f7a40fbb 100644--- a/src/entrypoint/centrality.rs+++ b/src/entrypoint/centrality.rs@@ -14,27 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{- ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder, CentralityConfig,-};+use std::path::Path;-pub struct CentralityEntrypoint {- config: CentralityConfig,-}+use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};-impl From<CentralityConfig> for CentralityEntrypoint {- fn from(config: CentralityConfig) -> Self {- Self { config }- }-}+pub struct CentralityEntrypoint {}impl CentralityEntrypoint {- pub fn run(self) {- let graph = WebgraphBuilder::new(&self.config.webgraph_path)- .with_host_graph()- .open();+ pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {+ let graph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();- let mut centrality_store = CentralityStore::new(&self.config.output_path);+ let mut centrality_store = CentralityStore::new(output_path);centrality_store.append(graphdiff --git a/src/server.rs b/src/entrypoint/frontend.rssimilarity index 70%rename from src/server.rsrename to src/entrypoint/frontend.rsindex d0ae6e6e..4a3eda82 100644--- a/src/server.rs+++ b/src/entrypoint/frontend.rs@@ -14,8 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::index::Index;+use anyhow::Result;-pub struct Server {- pub index: Index,+use crate::frontend::router;++pub async fn run(index_path: &str, host: &str) -> Result<()> {+ let app = router(index_path)?;+ let addr = host.parse()?;+ tracing::info!("listening on {}", addr);+ axum::Server::bind(&addr)+ .serve(app.into_make_service())+ .await?;++ Ok(())}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 52ba8ca2..f8ee8558 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -25,14 +25,10 @@ use crate::ranking::centrality_store::CentralityStore;use crate::warc::WarcFile;use crate::webpage::{Html, Link, Webpage};use crate::{- HttpConfig, IndexingConfig, IndexingLocalConfig, IndexingMasterConfig, IndexingWorkerConfig,- LocalConfig, Result, WarcSource,+ HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig, Result, WarcSource,};-pub struct Indexer {- config: IndexingConfig,- worker_addr: Option<String>,-}+pub struct Indexer {}#[derive(Debug, Serialize, Deserialize, Clone)]enum JobConfig {@@ -85,6 +81,9 @@ impl Map<IndexingWorker, FrozenIndex> for Job {trace!("inserting webpage: {:?}", html.url());+ trace!("title = {:?}", html.title());+ trace!("text = {:?}", html.text());+let webpage = Webpage {html,backlinks,@@ -118,13 +117,7 @@ impl Reduce<Index> for Index {}impl Indexer {- pub fn new(config: IndexingConfig, worker_addr: Option<String>) -> Self {- Self {- config,- worker_addr,- }- }- fn run_master(config: &IndexingMasterConfig) -> Result<()> {+ pub fn run_master(config: &IndexingMasterConfig) -> Result<()> {info!("Running master for index construction");let warc_paths = config.warc_source.paths()?;@@ -147,9 +140,9 @@ impl Indexer {config: job_config.clone(),warc_path,base_path: config- .graph_base_path+ .index_base_path.clone()- .unwrap_or_else(|| "index".to_string()),+ .unwrap_or_else(|| "data/index".to_string()),}}));@@ -164,8 +157,8 @@ impl Indexer {Ok(())}- fn run_worker(worker_addr: String, config: &IndexingWorkerConfig) -> Result<()> {- IndexingWorker::new(config.centrality_store_path.clone()).run::<Job, FrozenIndex>(+ pub fn run_worker(worker_addr: String, centrality_store_path: String) -> Result<()> {+ IndexingWorker::new(centrality_store_path).run::<Job, FrozenIndex>(worker_addr.parse::<SocketAddr>().expect("Could not parse worker address"),@@ -173,7 +166,7 @@ impl Indexer {Ok(())}- fn run_locally(config: &IndexingLocalConfig) -> Result<()> {+ pub fn run_locally(config: &IndexingLocalConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;let job_config = match config.warc_source.clone() {@@ -189,7 +182,7 @@ impl Indexer {.map(|path| Job {config: job_config.clone(),warc_path: path,- base_path: "index".to_string(),+ base_path: "data/index".to_string(),}).map(|job| job.map(&worker)).fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {@@ -199,17 +192,4 @@ impl Indexer {Ok(())}-- pub fn run(&self) -> Result<()> {- match &self.config {- IndexingConfig::Master(config) => Self::run_master(config),- IndexingConfig::Worker(config) => Self::run_worker(- self.worker_addr- .clone()- .expect("Worker address not specified"),- config,- ),- IndexingConfig::Local(config) => Self::run_locally(config),- }- }}diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex 88fd1614..220fc6a6 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -14,11 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.mod centrality;+pub mod frontend;mod indexer;-mod server;mod webgraph;pub use centrality::CentralityEntrypoint;pub use indexer::Indexer;-pub use server::ServerEntrypoint;pub use webgraph::WebgraphEntrypoint;diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 3140f606..7493c3f9 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -18,8 +18,7 @@ use crate::{warc::WarcFile,webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},webpage::{self, Html},- HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,- WebgraphMasterConfig,+ HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,};use serde::{Deserialize, Serialize};use std::{net::SocketAddr, path::Path};@@ -111,19 +110,10 @@ impl Reduce<Webgraph> for Webgraph {}}-pub struct WebgraphEntrypoint {- config: WebgraphConfig,- worker_addr: Option<String>,-}+pub struct WebgraphEntrypoint {}impl WebgraphEntrypoint {- pub fn new(config: WebgraphConfig, worker_addr: Option<String>) -> Self {- Self {- config,- worker_addr,- }- }- fn run_master(config: &WebgraphMasterConfig) -> Result<()> {+ pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {info!("Running master for webgraph construction");let warc_paths = config.warc_source.paths()?;@@ -148,7 +138,7 @@ impl WebgraphEntrypoint {graph_base_path: config.graph_base_path.clone()- .unwrap_or_else(|| "webgraph".to_string()),+ .unwrap_or_else(|| "data/webgraph".to_string()),}}));@@ -163,7 +153,7 @@ impl WebgraphEntrypoint {Ok(())}- fn run_worker(worker_addr: String) -> Result<()> {+ pub fn run_worker(worker_addr: String) -> Result<()> {StatelessWorker::default().run::<Job, FrozenWebgraph>(worker_addr.parse::<SocketAddr>()@@ -172,7 +162,7 @@ impl WebgraphEntrypoint {Ok(())}- fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {+ pub fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;let job_config = match config.warc_source.clone() {@@ -187,7 +177,7 @@ impl WebgraphEntrypoint {.map(|path| Job {config: job_config.clone(),warc_path: path,- graph_base_path: "webgraph".to_string(),+ graph_base_path: "data/webgraph".to_string(),}).map(|job| job.map(&worker)).fold(@@ -200,16 +190,4 @@ impl WebgraphEntrypoint {Ok(())}-- pub fn run(&self) -> Result<()> {- match &self.config {- WebgraphConfig::Master(config) => WebgraphEntrypoint::run_master(config),- WebgraphConfig::Worker => WebgraphEntrypoint::run_worker(- self.worker_addr- .clone()- .expect("Worker address not specified"),- ),- WebgraphConfig::Local(config) => WebgraphEntrypoint::run_locally(config),- }- }}diff --git a/src/entrypoint/server.rs b/src/frontend/mod.rssimilarity index 68%rename from src/entrypoint/server.rsrename to src/frontend/mod.rsindex 61220a72..4811fdb5 100644--- a/src/entrypoint/server.rs+++ b/src/frontend/mod.rs@@ -14,15 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::net::SocketAddr;-use axum::{Extension, Router};-use crate::{- index::{Index, RetrievedWebpage},- server::Server,- Result, ServerConfig,-};+use crate::index::{Index, RetrievedWebpage};+use anyhow::Result;use std::collections::HashMap;use std::sync::Arc;@@ -36,38 +31,9 @@ use axum::{routing::get,};-pub struct ServerEntrypoint {- config: ServerConfig,-}--impl From<ServerConfig> for ServerEntrypoint {- fn from(config: ServerConfig) -> Self {- Self { config }- }-}--impl ServerEntrypoint {- pub async fn run(self) -> Result<()> {- let search_index = Index::open(&self.config.index_path)?;- let state = Arc::new(Server {- index: search_index,- });-- let app = Router::new().route("/", get(index)).layer(Extension(state));- let addr = SocketAddr::from(([127, 0, 0, 1], 3000));- tracing::debug!("listening on {}", addr);- axum::Server::bind(&addr)- .serve(app.into_make_service())- .await- .unwrap();-- Ok(())- }-}-async fn index(extract::Query(params): extract::Query<HashMap<String, String>>,- Extension(state): Extension<Arc<Server>>,+ Extension(state): Extension<Arc<State>>,) -> impl IntoResponse {let mut search_result = Vec::new();@@ -80,7 +46,7 @@ async fn index(.search(&query, ranker.collector()).expect("Search failed");- search_result = result.documents;+ search_result = dbg!(result.documents);}let template = IndexTemplate { search_result };@@ -88,13 +54,17 @@ async fn index(}#[derive(Template)]-#[template(path = "index.html")]+#[template(path = "index.html", escape = "none")]struct IndexTemplate {search_result: Vec<RetrievedWebpage>,}struct HtmlTemplate<T>(T);+pub struct State {+ pub index: Index,+}+impl<T> IntoResponse for HtmlTemplate<T>whereT: Template,@@ -110,3 +80,12 @@ where}}}++pub fn router(index_path: &str) -> Result<Router> {+ let search_index = Index::open(index_path)?;+ let state = Arc::new(State {+ index: search_index,+ });++ Ok(Router::new().route("/", get(index)).layer(Extension(state)))+}diff --git a/src/index.rs b/src/index.rsindex b5ea269f..6442f38f 100644--- a/src/index.rs+++ b/src/index.rs@@ -146,13 +146,16 @@ impl Index {meta.segments.push(segment);}+ meta.segments+ .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));+fs::remove_dir_all(other_path).unwrap();let self_path = Path::new(&path);std::fs::write(self_path.join("meta.json"),- serde_json::to_string(&meta).unwrap(),+ serde_json::to_string_pretty(&meta).unwrap(),).unwrap();@@ -262,6 +265,9 @@ mod tests {<head><title>Test website</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -292,6 +298,9 @@ mod tests {<head><title>Test website</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -321,6 +330,9 @@ mod tests {<head><title>Website for runners</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -350,6 +362,9 @@ mod tests {<head><title>Fast runner</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -394,6 +409,9 @@ mod tests {<head><title>Website B</title></head>+ <body>+ body+ </body></html>"#,"https://www.b.com",@@ -435,6 +453,9 @@ mod tests {<head><title>Website for runners</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -465,6 +486,9 @@ mod tests {<head><title>News website</title></head>+ <body>+ body+ </body></html>"#,"https://www.dr.dk",@@ -492,6 +516,9 @@ mod tests {<head><title>Test website</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -530,6 +557,9 @@ mod tests {<head><title>Test website</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -547,6 +577,9 @@ mod tests {<head><title>Test website</title></head>+ <body>+ body+ </body></html>"#,"https://www.example.com",@@ -555,7 +588,8 @@ mod tests {)).expect("failed to parse webpage");- let index = index1.merge(index2);+ let mut index = index1.merge(index2);+ index.commit().unwrap();let query = Query::parse("website").expect("Failed to parse query");let ranker = Ranker::new(query.clone());diff --git a/src/lib.rs b/src/lib.rsindex de3f3f2c..e1e46740 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -28,56 +28,23 @@ pub mod mapreduce;mod directory;mod exponential_backoff;-mod query;+mod frontend;+pub mod query;pub mod ranking;mod schema;pub mod searcher;-pub mod server;mod snippet;mod tokenizer;mod warc;pub mod webgraph;pub mod webpage;-#[derive(Debug, Deserialize, Clone)]-#[serde(tag = "type")]-pub enum Config {- Indexer(IndexingConfig),- Webgraph(WebgraphConfig),- Centrality(CentralityConfig),- Server(ServerConfig),-}--#[derive(Debug, Deserialize, Clone)]-pub struct ServerConfig {- pub index_path: String,-}--#[derive(Debug, Deserialize, Clone)]-pub struct CentralityConfig {- pub webgraph_path: String,- pub output_path: String,-}--#[derive(Debug, Deserialize, Clone)]-#[serde(tag = "mode")]-pub enum IndexingConfig {- Master(IndexingMasterConfig),- Worker(IndexingWorkerConfig),- Local(IndexingLocalConfig),-}-#[derive(Debug, Deserialize, Clone)]pub struct IndexingMasterConfig {limit_warc_files: Option<usize>,warc_source: WarcSource,workers: Vec<String>,- graph_base_path: Option<String>,-}--#[derive(Debug, Deserialize, Clone)]-pub struct IndexingWorkerConfig {- centrality_store_path: String,+ index_base_path: Option<String>,}#[derive(Debug, Deserialize, Clone)]@@ -86,14 +53,6 @@ pub struct IndexingLocalConfig {centrality_store_path: String,}-#[derive(Debug, Deserialize, Clone)]-#[serde(tag = "mode")]-pub enum WebgraphConfig {- Master(WebgraphMasterConfig),- Worker,- Local(WebgraphLocalConfig),-}-#[derive(Debug, Deserialize, Clone)]pub struct WebgraphMasterConfig {limit_warc_files: Option<usize>,diff --git a/src/main.rs b/src/main.rsnew file mode 100644index 00000000..00913c4e--- /dev/null+++ b/src/main.rs@@ -0,0 +1,127 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use anyhow::Result;+use clap::{Parser, Subcommand};+use cuely::entrypoint::{frontend, CentralityEntrypoint, Indexer, WebgraphEntrypoint};+use serde::de::DeserializeOwned;+use std::fs;+use std::path::Path;+use tracing::Level;+use tracing_subscriber::FmtSubscriber;++#[derive(Parser)]+#[clap(author, version, about, long_about = None)]+#[clap(propagate_version = true)]+struct Args {+ #[clap(subcommand)]+ command: Commands,+}++#[derive(Subcommand)]+enum Commands {+ Indexer {+ #[clap(subcommand)]+ options: IndexingOptions,+ },+ Centrality {+ webgraph_path: String,+ output_path: String,+ },+ Webgraph {+ #[clap(subcommand)]+ options: WebgraphOptions,+ },+ Frontend {+ index_path: String,+ #[clap(default_value = "0.0.0.0:3000")]+ host: String,+ },+}++#[derive(Subcommand)]+enum WebgraphOptions {+ Master { config_path: String },+ Worker { address: String },+ Local { config_path: String },+}++#[derive(Subcommand)]+enum IndexingOptions {+ Master {+ config_path: String,+ },+ Worker {+ address: String,+ centrality_store_path: String,+ },+ Local {+ config_path: String,+ },+}++fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {+ let raw_config = fs::read_to_string(path).expect("Failed to read config file");+ toml::from_str(&raw_config).expect("Failed to parse config")+}++#[tokio::main]+async fn main() -> Result<()> {+ let subscriber = FmtSubscriber::builder()+ .with_max_level(Level::INFO)+ .finish();+ tracing::subscriber::set_global_default(subscriber).unwrap();++ let args = Args::parse();++ match args.command {+ Commands::Indexer { options } => match options {+ IndexingOptions::Master { config_path } => {+ let config = load_toml_config(&config_path);+ Indexer::run_master(&config)?;+ }+ IndexingOptions::Worker {+ address,+ centrality_store_path,+ } => {+ Indexer::run_worker(address, centrality_store_path)?;+ }+ IndexingOptions::Local { config_path } => {+ let config = load_toml_config(&config_path);+ Indexer::run_locally(&config)?;+ }+ },+ Commands::Centrality {+ webgraph_path,+ output_path,+ } => CentralityEntrypoint::run(webgraph_path, output_path),+ Commands::Webgraph { options } => match options {+ WebgraphOptions::Master { config_path } => {+ let config = load_toml_config(config_path);+ WebgraphEntrypoint::run_master(&config)?;+ }+ WebgraphOptions::Worker { address } => {+ WebgraphEntrypoint::run_worker(address)?;+ }+ WebgraphOptions::Local { config_path } => {+ let config = load_toml_config(config_path);+ WebgraphEntrypoint::run_locally(&config)?;+ }+ },+ Commands::Frontend { index_path, host } => frontend::run(&index_path, &host).await?,+ }++ Ok(())+}diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex d5e82e30..c55f919c 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -38,9 +38,6 @@ pub enum Error {#[error("could not get a working worker")]NoAvailableWorker,- #[error("failed to get value from channel")]- ChannelRecv(#[from] async_channel::RecvError),-#[error("did not get a reponse")]NoResponse,}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex f1f50334..8bdef605 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -34,7 +34,7 @@ impl Ranker {}}- pub(crate) fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {+ pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {let score_tweaker = InitialScoreTweaker::new(self.query.clone());TopDocs::with_limit(20).tweak_score(score_tweaker)}@@ -75,6 +75,9 @@ mod tests {<head><title>Website B</title></head>+ <body>+ body+ </body></html>"#,"https://www.b.com",@@ -107,6 +110,9 @@ mod tests {<head><title>DR Homepage</title></head>+ <body>+ body+ </body></html>"#,"https://www.dr.dk",@@ -121,6 +127,9 @@ mod tests {<head><title>Subsite dr</title></head>+ <body>+ body+ </body></html>"#,"https://www.dr.dk/whatever",diff --git a/src/webpage.rs b/src/webpage.rsindex 3fd4295a..a0ed0e6b 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -210,7 +210,7 @@ impl<'a> Html<'a> {.collect::<Vec<String>>()}- pub fn text(&self) -> String {+ pub fn text(&self) -> Option<String> {let strings: Vec<String> = match &self.dom {Dom::ScraperHtml(dom) => {let selectors = vec![@@ -249,10 +249,19 @@ impl<'a> Html<'a> {.unwrap_or_default(),};- Itertools::intersperse(strings.into_iter(), "\n".to_string())- .collect::<String>()- .trim()- .to_string()+ let s = Itertools::intersperse(+ strings.into_iter().filter(|s| !s.is_empty()),+ "\n".to_string(),+ )+ .collect::<String>()+ .trim()+ .to_string();++ if s.is_empty() {+ None+ } else {+ Some(s)+ }}fn children_text(children: Children, parser: &tl::Parser) -> Vec<String> {@@ -371,7 +380,13 @@ impl<'a> Html<'a> {doc.add_text(tantivy_field, self.title().unwrap())}- Field::Body => doc.add_text(tantivy_field, self.text()),+ Field::Body => {+ let text = self.text();+ if text.is_none() {+ return Err(Error::EmptyField("body"));+ }+ doc.add_text(tantivy_field, text.unwrap())+ }Field::Url => doc.add_text(tantivy_field, self.url()),Field::FastUrl => doc.add_bytes(tantivy_field,@@ -424,7 +439,10 @@ mod tests {text: "Best example website ever".to_string()}]);- assert_eq!(&webpage.text(), "Best example website ever");+ assert_eq!(+ webpage.text(),+ Some("Best example website ever".to_string())+ );assert_eq!(webpage.title(), Some("Best website".to_string()));let mut expected_meta = BTreeMap::new();@@ -449,7 +467,7 @@ mod tests {let webpage = Html::parse(raw, "https://www.example.com/whatever");- assert_eq!(&webpage.text(), "test website");+ assert_eq!(webpage.text(), Some("test website".to_string()));}#[test]@@ -475,7 +493,7 @@ mod tests {let webpage = Html::parse(raw, "https://www.example.com");- assert!(!webpage.text().contains("not"));+ assert!(!webpage.text().unwrap().contains("not"));}#[test]@@ -501,7 +519,7 @@ mod tests {let webpage = Html::parse(raw, "https://www.example.com");- assert!(!webpage.text().contains("not"));+ assert!(!webpage.text().unwrap().contains("not"));}#[test]