Skip to content

web search done right

serialize/deserialize webgraph

Mikkel Denker40a2c2c0f888
diff --git a/Cargo.toml b/Cargo.toml
index f1b839de..748a0047 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,6 +14,7 @@ name = "cuely"
async-channel = "1.6.1"
async-trait = "0.1.53"
bincode = "1.3.3"
+clap = { version = "3.1.18", features = ["derive"] }
encoding_rs = "0.8.31"
flate2 = "1.0.23"
futures = "0.3.21"
diff --git a/configs/indexer.toml b/configs/indexer.toml
index 108b529c..4efb218b 100644
--- a/configs/indexer.toml
+++ b/configs/indexer.toml
@@ -1,16 +1,12 @@
-mode = "Indexer"
+type = "Indexer"
warc_paths_file = "warc.paths"
# [warc_source]
# type = "S3"
-#
-# [warc_source.args]
# name = "us-east-1"
# endpoint = "https://s3.us-east-1.amazonaws.com"
# bucket = "commoncrawl"
[warc_source]
type = "HTTP"
-
-[warc_source.args]
base_url = "https://data.commoncrawl.org/"
diff --git a/configs/webgraph_master.toml b/configs/webgraph_master.toml
new file mode 100644
index 00000000..0573ccf2
--- /dev/null
+++ b/configs/webgraph_master.toml
@@ -0,0 +1,17 @@
+type = "Webgraph"
+mode = "Master"
+
+warc_paths_file = "warc.paths"
+addr = "0.0.0.0:1337"
+workers = ["0.0.0.0:1338"]
+
+
+# [warc_source]
+# type = "S3"
+# name = "us-east-1"
+# endpoint = "https://s3.us-east-1.amazonaws.com"
+# bucket = "commoncrawl"
+
+[warc_source]
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index a579857a..d4deef0f 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -13,27 +13,33 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use clap::Parser;
use cuely::entrypoint::{Indexer, WebgraphBuilder};
-use cuely::{Config, Mode};
+use cuely::Config;
use std::fs;
+#[derive(Parser, Debug)]
+struct Args {
+ config_file: String,
+}
+
#[tokio::main]
async fn main() {
- let raw_config =
- fs::read_to_string("configs/indexer.toml").expect("Failed to read config file");
+ let args = Args::parse();
+
+ let raw_config = fs::read_to_string(args.config_file).expect("Failed to read config file");
let config: Config = toml::from_str(&raw_config).expect("Failed to parse config");
- match config.mode {
- Mode::Indexer => {
+ match config {
+ Config::Indexer(config) => {
Indexer::from(config)
.run()
.await
.expect("Failed to index documents");
}
- Mode::Webgraph => WebgraphBuilder::from(config)
+ Config::Webgraph(config) => WebgraphBuilder::from(config)
.run()
- .await
.expect("Failed to build webgraph"),
}
}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index e0ff6e68..d84f1b29 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -23,15 +23,15 @@ use tokio::io::AsyncReadExt;
use crate::warc::WarcFile;
use crate::webpage::Html;
-use crate::{Config, Error, Result};
+use crate::{Error, IndexingConfig, Result};
pub struct Indexer {
warc_paths: Vec<String>,
- config: Config,
+ config: IndexingConfig,
}
-impl From<Config> for Indexer {
- fn from(config: Config) -> Self {
+impl From<IndexingConfig> for Indexer {
+ fn from(config: IndexingConfig) -> Self {
let file = File::open(&config.warc_paths_file).unwrap();
let mut warc_paths = Vec::new();
@@ -56,12 +56,7 @@ impl Indexer {
stream::iter(self.warc_paths.into_iter().progress_with(pb))
.map(|warc_path| {
- let source = self
- .config
- .warc_source
- .as_ref()
- .expect("Indexing needs a warc source")
- .clone();
+ let source = self.config.warc_source.clone();
tokio::spawn(async move { WarcFile::download(source, &warc_path).await })
})
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index d11b60b6..0d311df9 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -13,18 +13,58 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{Config, Result};
+use crate::{
+ mapreduce::{Map, Reduce},
+ webgraph::{SledStore, Webgraph},
+ Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,
+};
+use serde::{Deserialize, Serialize};
-pub struct WebgraphBuilder {}
+pub struct WebgraphBuilder {
+ config: WebgraphConfig,
+}
+
+impl From<WebgraphConfig> for WebgraphBuilder {
+ fn from(config: WebgraphConfig) -> Self {
+ Self { config }
+ }
+}
-impl From<Config> for WebgraphBuilder {
- fn from(config: Config) -> Self {
- Self {}
+#[derive(Serialize, Deserialize)]
+struct SingleJob {
+ warc_source: WarcSource,
+ warc_path: String,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Job(Vec<SingleJob>);
+
+impl Map<Webgraph<SledStore>> for Job {
+ fn map(self) -> Webgraph<SledStore> {
+ todo!()
+ }
+}
+
+impl Reduce<Webgraph<SledStore>> for Webgraph<SledStore> {
+ fn reduce(mut self, other: Webgraph<SledStore>) -> Webgraph<SledStore> {
+ self.merge(other);
+ self
}
}
impl WebgraphBuilder {
- pub async fn run(&self) -> Result<()> {
+ fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
todo!();
}
+
+ fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {
+ todo!();
+ }
+
+ pub fn run(&self) -> Result<()> {
+ match &self.config {
+ WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),
+ WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),
+ }
+ }
}
diff --git a/src/lib.rs b/src/lib.rs
index 4483457b..bd79d89c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,7 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use std::io;
use std::num::ParseIntError;
use tantivy::TantivyError;
@@ -35,35 +35,53 @@ mod webgraph;
mod webpage;
#[derive(Debug, Deserialize, Clone)]
-pub struct Config {
- pub mode: Mode,
- warc_source: Option<WarcSource>,
+#[serde(tag = "type")]
+pub enum Config {
+ Indexer(IndexingConfig),
+ Webgraph(WebgraphConfig),
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct IndexingConfig {
+ warc_source: WarcSource,
warc_paths_file: String,
}
#[derive(Debug, Deserialize, Clone)]
-#[serde(tag = "type", content = "args")]
-pub enum WarcSource {
- S3(S3Config),
- HTTP(HttpConfig),
+#[serde(tag = "mode")]
+pub enum WebgraphConfig {
+ Master(WebgraphMasterConfig),
+ Worker(WebgraphWorkerConfig),
}
#[derive(Debug, Deserialize, Clone)]
-pub enum Mode {
- /// Index warc documents into index
- Indexer,
- /// Create webgraph from warc documents
- Webgraph,
+pub struct WebgraphMasterConfig {
+ warc_source: WarcSource,
+ warc_paths_file: String,
+ addr: String,
+ workers: Vec<String>,
}
#[derive(Debug, Deserialize, Clone)]
+pub struct WebgraphWorkerConfig {
+ addr: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(tag = "type")]
+pub enum WarcSource {
+ S3(S3Config),
+ HTTP(HttpConfig),
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct S3Config {
name: String,
endpoint: String,
bucket: String,
}
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct HttpConfig {
base_url: String,
}