Skip to content

web search done right

significant speedup in webgraph by inserting into memory and only flushing to disk when needed

Mikkel Denkerabcada0245fa
diff --git a/configs/webgraph_master.toml b/configs/webgraph_master.toml
index 0573ccf2..6fe78ffa 100644
--- a/configs/webgraph_master.toml
+++ b/configs/webgraph_master.toml
@@ -2,16 +2,8 @@ type = "Webgraph"
mode = "Master"
warc_paths_file = "warc.paths"
-addr = "0.0.0.0:1337"
workers = ["0.0.0.0:1338"]
-
-# [warc_source]
-# type = "S3"
-# name = "us-east-1"
-# endpoint = "https://s3.us-east-1.amazonaws.com"
-# bucket = "commoncrawl"
-
[warc_source]
type = "HTTP"
base_url = "https://data.commoncrawl.org/"
diff --git a/configs/webgraph_worker.toml b/configs/webgraph_worker.toml
new file mode 100644
index 00000000..69d289b3
--- /dev/null
+++ b/configs/webgraph_worker.toml
@@ -0,0 +1,4 @@
+type = "Webgraph"
+mode = "Worker"
+
+addr = "0.0.0.0:1338"
\ No newline at end of file
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index d4deef0f..ff84f3a6 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -17,6 +17,8 @@ use clap::Parser;
use cuely::entrypoint::{Indexer, WebgraphBuilder};
use cuely::Config;
use std::fs;
+use tracing::Level;
+use tracing_subscriber::FmtSubscriber;
#[derive(Parser, Debug)]
struct Args {
@@ -25,6 +27,11 @@ struct Args {
#[tokio::main]
async fn main() {
+ let subscriber = FmtSubscriber::builder()
+ .with_max_level(Level::DEBUG)
+ .finish();
+ tracing::subscriber::set_global_default(subscriber).unwrap();
+
let args = Args::parse();
let raw_config = fs::read_to_string(args.config_file).expect("Failed to read config file");
@@ -40,6 +47,7 @@ async fn main() {
}
Config::Webgraph(config) => WebgraphBuilder::from(config)
.run()
+ .await
.expect("Failed to build webgraph"),
}
}
diff --git a/src/directory.rs b/src/directory.rs
new file mode 100644
index 00000000..ac1b24ed
--- /dev/null
+++ b/src/directory.rs
@@ -0,0 +1,91 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::Result;
+use serde::{Deserialize, Serialize};
+use std::fs;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum DirEntry {
+ Folder {
+ name: String,
+ entries: Vec<DirEntry>,
+ },
+ File {
+ name: String,
+ content: Vec<u8>,
+ },
+}
+
+fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {
+ let mut res = Vec::new();
+
+ for f in fs::read_dir(path)? {
+ let entry = f?;
+ let metadata = entry.metadata()?;
+
+ if metadata.is_dir() {
+ res.push(DirEntry::Folder {
+ name: entry.path().as_os_str().to_str().unwrap().to_string(),
+ entries: iterate_children(entry.path().as_os_str().to_str().unwrap())?,
+ })
+ } else if metadata.is_file() {
+ res.push(DirEntry::File {
+ name: entry.path().as_os_str().to_str().unwrap().to_string(),
+ content: fs::read(entry.path())?,
+ })
+ }
+ }
+
+ Ok(res)
+}
+
+fn recreate_folder(entry: &DirEntry) -> Result<()> {
+ match entry {
+ DirEntry::Folder { name, entries } => {
+ fs::create_dir(name)?;
+
+ for entry in entries {
+ recreate_folder(entry)?;
+ }
+
+ Ok(())
+ }
+ DirEntry::File { name, content } => Ok(fs::write(name, content)?),
+ }
+}
+
+pub fn scan_folder(path: String) -> Result<DirEntry> {
+ Ok(DirEntry::Folder {
+ entries: iterate_children(&path)?,
+ name: path,
+ })
+}
+
+pub fn serialize(path: String) -> Result<Vec<u8>> {
+ let folder = scan_folder(path)?;
+ Ok(bincode::serialize(&folder)?)
+}
+
+pub fn deserialize(bytes: &[u8]) -> Result<String> {
+ let entry = bincode::deserialize(bytes)?;
+ recreate_folder(&entry)?;
+
+ match entry {
+ DirEntry::Folder { name, entries: _ } => Ok(name),
+ DirEntry::File { name, content: _ } => Ok(name),
+ }
+}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index d84f1b29..8e50f8f1 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -87,48 +87,4 @@ impl Indexer {
Ok(())
}
-
- async fn download_from_http(warc_path: String, base_url: String) -> Result<Vec<u8>> {
- let mut url = base_url;
- if !url.ends_with('/') {
- url += "/";
- }
- url += &warc_path;
-
- let client = reqwest::Client::new();
- let res = client.get(url).send().await?;
-
- Ok(Vec::from(&res.bytes().await?[..]))
- }
-
- async fn download_from_s3(
- key: String,
- region_name: String,
- region_endpoint: String,
- bucket: String,
- ) -> Result<Vec<u8>> {
- let region = Region::Custom {
- name: region_name,
- endpoint: region_endpoint,
- };
-
- let client = S3Client::new(region);
-
- let obj = client
- .get_object(GetObjectRequest {
- bucket,
- key,
- ..Default::default()
- })
- .await?;
-
- let mut res = Vec::new();
- obj.body
- .ok_or(Error::S3DownloadError)?
- .into_async_read()
- .read_to_end(&mut res)
- .await?;
-
- Ok(res)
- }
}
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 0d311df9..0738cdfe 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -14,11 +14,20 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- mapreduce::{Map, Reduce},
- webgraph::{SledStore, Webgraph},
- Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,
+ mapreduce::{Map, MapReduce, Reduce, Worker},
+ warc::WarcFile,
+ webgraph::{Node, SledStore, Webgraph},
+ webpage::{self, Html},
+ HttpConfig, Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,
};
use serde::{Deserialize, Serialize};
+use std::{
+ fs::File,
+ io::{self, BufRead},
+ net::SocketAddr,
+ path::Path,
+};
+use tracing::{debug, info};
pub struct WebgraphBuilder {
config: WebgraphConfig,
@@ -31,17 +40,52 @@ impl From<WebgraphConfig> for WebgraphBuilder {
}
#[derive(Serialize, Deserialize)]
-struct SingleJob {
- warc_source: WarcSource,
+struct Job {
+ http_config: HttpConfig,
warc_path: String,
}
-#[derive(Serialize, Deserialize)]
-struct Job(Vec<SingleJob>);
-
impl Map<Webgraph<SledStore>> for Job {
fn map(self) -> Webgraph<SledStore> {
- todo!()
+ let name = self.warc_path.split("/").last().unwrap();
+
+ info!("processing {}", name);
+
+ let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));
+
+ let file = futures::executor::block_on(WarcFile::download(
+ WarcSource::HTTP(self.http_config),
+ &self.warc_path,
+ ))
+ .unwrap();
+
+ for record in file.records() {
+ if let Ok(record) = record {
+ let webpage = Html::parse(&record.response.body, &record.request.url);
+ for link in webpage
+ .links()
+ .into_iter()
+ .filter(|link| {
+ link.destination.starts_with("http://")
+ || link.destination.starts_with("https://")
+ })
+ .filter(|link| {
+ webpage::domain(&link.source) != webpage::domain(&link.destination)
+ })
+ {
+ debug!("inserting link {:?}", link);
+ graph.insert(
+ Node::from(link.source),
+ Node::from(link.destination),
+ link.text,
+ );
+ }
+ }
+ }
+
+ info!("{} done", name);
+
+ graph
}
}
@@ -53,18 +97,52 @@ impl Reduce<Webgraph<SledStore>> for Webgraph<SledStore> {
}
impl WebgraphBuilder {
- fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
- todo!();
+ async fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
+ info!("Running master for webgraph construction");
+
+ let file = File::open(&config.warc_paths_file)?;
+ let mut warc_paths = Vec::new();
+
+ for line in io::BufReader::new(file).lines() {
+ warc_paths.push(line?);
+ }
+
+ let workers: Vec<SocketAddr> = config
+ .workers
+ .iter()
+ .map(|worker| worker.parse().unwrap())
+ .collect();
+
+ let http_config = if let WarcSource::HTTP(http_config) = config.warc_source.clone() {
+ Some(http_config)
+ } else {
+ None
+ };
+
+ let http_config = http_config.unwrap();
+
+ warc_paths
+ .into_iter()
+ .map(|warc_path| Job {
+ http_config: http_config.clone(),
+ warc_path,
+ })
+ .map_reduce(&workers)
+ .await
+ .expect("failed to build webgraph");
+
+ Ok(())
}
- fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {
- todo!();
+ async fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {
+ Worker::run::<Job, Webgraph<SledStore>>(config.addr.parse::<SocketAddr>().unwrap()).await?;
+ Ok(())
}
- pub fn run(&self) -> Result<()> {
+ pub async fn run(&self) -> Result<()> {
match &self.config {
- WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),
- WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),
+ WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config).await,
+ WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config).await,
}
}
}
diff --git a/src/lib.rs b/src/lib.rs
index bd79d89c..226bc926 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -24,6 +24,7 @@ pub mod index;
pub mod mapreduce;
+mod directory;
mod query;
mod ranking;
mod schema;
@@ -58,7 +59,6 @@ pub enum WebgraphConfig {
pub struct WebgraphMasterConfig {
warc_source: WarcSource,
warc_paths_file: String,
- addr: String,
workers: Vec<String>,
}
@@ -120,6 +120,9 @@ pub enum Error {
#[error("Error while serializing/deserializing to/from bytes")]
Serialization(#[from] bincode::Error),
+
+ #[error("Error executing distributed jobs")]
+ MapReduce(#[from] mapreduce::Error),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 134a8e45..1336e166 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -26,7 +26,7 @@ struct RemoteWorker {
impl RemoteWorker {
fn retry_strategy() -> impl Iterator<Item = Duration> {
- ExponentialBackoff::from_millis(10).take(4)
+ ExponentialBackoff::from_millis(10).take(5)
}
async fn connect(&self) -> Result<TcpStream> {