web search done right
serialize/deserialize webgraph
Mikkel Denker40a2c2c0f888
diff --git a/Cargo.toml b/Cargo.tomlindex f1b839de..748a0047 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -14,6 +14,7 @@ name = "cuely"async-channel = "1.6.1"async-trait = "0.1.53"bincode = "1.3.3"+clap = { version = "3.1.18", features = ["derive"] }encoding_rs = "0.8.31"flate2 = "1.0.23"futures = "0.3.21"diff --git a/configs/indexer.toml b/configs/indexer.tomlindex 108b529c..4efb218b 100644--- a/configs/indexer.toml+++ b/configs/indexer.toml@@ -1,16 +1,12 @@-mode = "Indexer"+type = "Indexer"warc_paths_file = "warc.paths"# [warc_source]# type = "S3"-#-# [warc_source.args]# name = "us-east-1"# endpoint = "https://s3.us-east-1.amazonaws.com"# bucket = "commoncrawl"[warc_source]type = "HTTP"--[warc_source.args]base_url = "https://data.commoncrawl.org/"diff --git a/configs/webgraph_master.toml b/configs/webgraph_master.tomlnew file mode 100644index 00000000..0573ccf2--- /dev/null+++ b/configs/webgraph_master.toml@@ -0,0 +1,17 @@+type = "Webgraph"+mode = "Master"++warc_paths_file = "warc.paths"+addr = "0.0.0.0:1337"+workers = ["0.0.0.0:1338"]+++# [warc_source]+# type = "S3"+# name = "us-east-1"+# endpoint = "https://s3.us-east-1.amazonaws.com"+# bucket = "commoncrawl"++[warc_source]+type = "HTTP"+base_url = "https://data.commoncrawl.org/"diff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex a579857a..d4deef0f 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -13,27 +13,33 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use clap::Parser;use cuely::entrypoint::{Indexer, WebgraphBuilder};-use cuely::{Config, Mode};+use cuely::Config;use std::fs;+#[derive(Parser, Debug)]+struct Args {+ config_file: String,+}+#[tokio::main]async fn main() {- let raw_config =- fs::read_to_string("configs/indexer.toml").expect("Failed to read config file");+ let args = Args::parse();++ let raw_config = fs::read_to_string(args.config_file).expect("Failed to read config file");let config: Config = toml::from_str(&raw_config).expect("Failed to parse config");- match config.mode {- Mode::Indexer => {+ match config {+ Config::Indexer(config) => {Indexer::from(config).run().await.expect("Failed to index documents");}- Mode::Webgraph => WebgraphBuilder::from(config)+ Config::Webgraph(config) => WebgraphBuilder::from(config).run()- .await.expect("Failed to build webgraph"),}}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex e0ff6e68..d84f1b29 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -23,15 +23,15 @@ use tokio::io::AsyncReadExt;use crate::warc::WarcFile;use crate::webpage::Html;-use crate::{Config, Error, Result};+use crate::{Error, IndexingConfig, Result};pub struct Indexer {warc_paths: Vec<String>,- config: Config,+ config: IndexingConfig,}-impl From<Config> for Indexer {- fn from(config: Config) -> Self {+impl From<IndexingConfig> for Indexer {+ fn from(config: IndexingConfig) -> Self {let file = File::open(&config.warc_paths_file).unwrap();let mut warc_paths = Vec::new();@@ -56,12 +56,7 @@ impl Indexer {stream::iter(self.warc_paths.into_iter().progress_with(pb)).map(|warc_path| {- let source = self- .config- .warc_source- .as_ref()- .expect("Indexing needs a warc source")- .clone();+ let source = self.config.warc_source.clone();tokio::spawn(async move { WarcFile::download(source, &warc_path).await })})diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex d11b60b6..0d311df9 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -13,18 +13,58 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{Config, Result};+use crate::{+ mapreduce::{Map, Reduce},+ webgraph::{SledStore, Webgraph},+ Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,+};+use serde::{Deserialize, Serialize};-pub struct WebgraphBuilder {}+pub struct WebgraphBuilder {+ config: WebgraphConfig,+}++impl From<WebgraphConfig> for WebgraphBuilder {+ fn from(config: WebgraphConfig) -> Self {+ Self { config }+ }+}-impl From<Config> for WebgraphBuilder {- fn from(config: Config) -> Self {- Self {}+#[derive(Serialize, Deserialize)]+struct SingleJob {+ warc_source: WarcSource,+ warc_path: String,+}++#[derive(Serialize, Deserialize)]+struct Job(Vec<SingleJob>);++impl Map<Webgraph<SledStore>> for Job {+ fn map(self) -> Webgraph<SledStore> {+ todo!()+ }+}++impl Reduce<Webgraph<SledStore>> for Webgraph<SledStore> {+ fn reduce(mut self, other: Webgraph<SledStore>) -> Webgraph<SledStore> {+ self.merge(other);+ self}}impl WebgraphBuilder {- pub async fn run(&self) -> Result<()> {+ fn run_master(config: &WebgraphMasterConfig) -> Result<()> {todo!();}++ fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {+ todo!();+ }++ pub fn run(&self) -> Result<()> {+ match &self.config {+ WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),+ WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),+ }+ }}diff --git a/src/lib.rs b/src/lib.rsindex 4483457b..bd79d89c 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -13,7 +13,7 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::Deserialize;+use serde::{Deserialize, Serialize};use std::io;use std::num::ParseIntError;use tantivy::TantivyError;@@ -35,35 +35,53 @@ mod webgraph;mod webpage;#[derive(Debug, Deserialize, Clone)]-pub struct Config {- pub mode: Mode,- warc_source: Option<WarcSource>,+#[serde(tag = "type")]+pub enum Config {+ Indexer(IndexingConfig),+ Webgraph(WebgraphConfig),+}++#[derive(Debug, Deserialize, Clone)]+pub struct IndexingConfig {+ warc_source: WarcSource,warc_paths_file: String,}#[derive(Debug, Deserialize, Clone)]-#[serde(tag = "type", content = "args")]-pub enum WarcSource {- S3(S3Config),- HTTP(HttpConfig),+#[serde(tag = "mode")]+pub enum WebgraphConfig {+ Master(WebgraphMasterConfig),+ Worker(WebgraphWorkerConfig),}#[derive(Debug, Deserialize, Clone)]-pub enum Mode {- /// Index warc documents into index- Indexer,- /// Create webgraph from warc documents- Webgraph,+pub struct WebgraphMasterConfig {+ warc_source: WarcSource,+ warc_paths_file: String,+ addr: String,+ workers: Vec<String>,}#[derive(Debug, Deserialize, Clone)]+pub struct WebgraphWorkerConfig {+ addr: String,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+#[serde(tag = "type")]+pub enum WarcSource {+ S3(S3Config),+ HTTP(HttpConfig),+}++#[derive(Debug, Serialize, Deserialize, Clone)]pub struct S3Config {name: String,endpoint: String,bucket: String,}-#[derive(Debug, Deserialize, Clone)]+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct HttpConfig {base_url: String,}