web search done right
new mode: webgraph builder
Mikkel Denker718034fb9d65
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex 59b932f7..a579857a 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -13,7 +13,8 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use cuely::{Config, Indexer, Mode};+use cuely::entrypoint::{Indexer, WebgraphBuilder};+use cuely::{Config, Mode};use std::fs;#[tokio::main]@@ -25,10 +26,14 @@ async fn main() {match config.mode {Mode::Indexer => {- Indexer::from_config(config)+ Indexer::from(config).run().await.expect("Failed to index documents");}+ Mode::Webgraph => WebgraphBuilder::from(config)+ .run()+ .await+ .expect("Failed to build webgraph"),}}diff --git a/src/indexer.rs b/src/entrypoint/indexer.rssimilarity index 97%rename from src/indexer.rsrename to src/entrypoint/indexer.rsindex 37c84dd1..e0ff6e68 100644--- a/src/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -23,15 +23,15 @@ use tokio::io::AsyncReadExt;use crate::warc::WarcFile;use crate::webpage::Html;-use crate::{Config, Error, Result, WarcSource};+use crate::{Config, Error, Result};pub struct Indexer {warc_paths: Vec<String>,config: Config,}-impl Indexer {- pub fn from_config(config: Config) -> Self {+impl From<Config> for Indexer {+ fn from(config: Config) -> Self {let file = File::open(&config.warc_paths_file).unwrap();let mut warc_paths = Vec::new();@@ -41,7 +41,9 @@ impl Indexer {Self { warc_paths, config }}+}+impl Indexer {pub async fn run(self) -> Result<()> {let pb = ProgressBar::new(self.warc_paths.len() as u64);pb.set_style(diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsnew file mode 100644index 00000000..608da539--- /dev/null+++ b/src/entrypoint/mod.rs@@ -0,0 +1,20 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+mod indexer;+mod webgraph;++pub use indexer::Indexer;+pub use webgraph::WebgraphBuilder;diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsnew file mode 100644index 00000000..d11b60b6--- /dev/null+++ b/src/entrypoint/webgraph.rs@@ -0,0 +1,30 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::{Config, Result};++pub struct WebgraphBuilder {}++impl From<Config> for WebgraphBuilder {+ fn from(config: Config) -> Self {+ Self {}+ }+}++impl WebgraphBuilder {+ pub async fn run(&self) -> Result<()> {+ todo!();+ }+}diff --git a/src/lib.rs b/src/lib.rsindex d7cf790a..5dc0326e 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -19,8 +19,8 @@ use std::num::ParseIntError;use tantivy::TantivyError;use thiserror::Error;+pub mod entrypoint;mod index;-mod indexer;mod query;mod ranking;mod schema;@@ -31,8 +31,6 @@ mod warc;mod webgraph;mod webpage;-pub use indexer::Indexer;-#[derive(Debug, Deserialize, Clone)]pub struct Config {pub mode: Mode,@@ -49,7 +47,10 @@ pub enum WarcSource {#[derive(Debug, Deserialize, Clone)]pub enum Mode {+ /// Index warc documents into indexIndexer,+ /// Create webgraph from warc documents+ Webgraph,}#[derive(Debug, Deserialize, Clone)]