Skip to content

web search done right

new mode: webgraph builder

Mikkel Denker718034fb9d65
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index 59b932f7..a579857a 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -13,7 +13,8 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use cuely::{Config, Indexer, Mode};
+use cuely::entrypoint::{Indexer, WebgraphBuilder};
+use cuely::{Config, Mode};
use std::fs;
#[tokio::main]
@@ -25,10 +26,14 @@ async fn main() {
match config.mode {
Mode::Indexer => {
- Indexer::from_config(config)
+ Indexer::from(config)
.run()
.await
.expect("Failed to index documents");
}
+ Mode::Webgraph => WebgraphBuilder::from(config)
+ .run()
+ .await
+ .expect("Failed to build webgraph"),
}
}
diff --git a/src/indexer.rs b/src/entrypoint/indexer.rs
similarity index 97%
rename from src/indexer.rs
rename to src/entrypoint/indexer.rs
index 37c84dd1..e0ff6e68 100644
--- a/src/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -23,15 +23,15 @@ use tokio::io::AsyncReadExt;
use crate::warc::WarcFile;
use crate::webpage::Html;
-use crate::{Config, Error, Result, WarcSource};
+use crate::{Config, Error, Result};
pub struct Indexer {
warc_paths: Vec<String>,
config: Config,
}
-impl Indexer {
- pub fn from_config(config: Config) -> Self {
+impl From<Config> for Indexer {
+ fn from(config: Config) -> Self {
let file = File::open(&config.warc_paths_file).unwrap();
let mut warc_paths = Vec::new();
@@ -41,7 +41,9 @@ impl Indexer {
Self { warc_paths, config }
}
+}
+impl Indexer {
pub async fn run(self) -> Result<()> {
let pb = ProgressBar::new(self.warc_paths.len() as u64);
pb.set_style(
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
new file mode 100644
index 00000000..608da539
--- /dev/null
+++ b/src/entrypoint/mod.rs
@@ -0,0 +1,20 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod indexer;
+mod webgraph;
+
+pub use indexer::Indexer;
+pub use webgraph::WebgraphBuilder;
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
new file mode 100644
index 00000000..d11b60b6
--- /dev/null
+++ b/src/entrypoint/webgraph.rs
@@ -0,0 +1,30 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::{Config, Result};
+
+pub struct WebgraphBuilder {}
+
+impl From<Config> for WebgraphBuilder {
+ fn from(config: Config) -> Self {
+ Self {}
+ }
+}
+
+impl WebgraphBuilder {
+ pub async fn run(&self) -> Result<()> {
+ todo!();
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index d7cf790a..5dc0326e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -19,8 +19,8 @@ use std::num::ParseIntError;
use tantivy::TantivyError;
use thiserror::Error;
+pub mod entrypoint;
mod index;
-mod indexer;
mod query;
mod ranking;
mod schema;
@@ -31,8 +31,6 @@ mod warc;
mod webgraph;
mod webpage;
-pub use indexer::Indexer;
-
#[derive(Debug, Deserialize, Clone)]
pub struct Config {
pub mode: Mode,
@@ -49,7 +47,10 @@ pub enum WarcSource {
#[derive(Debug, Deserialize, Clone)]
pub enum Mode {
+ /// Index warc documents into index
Indexer,
+ /// Create webgraph from warc documents
+ Webgraph,
}
#[derive(Debug, Deserialize, Clone)]