web search done right
significant speedup in webgraph by inserting into memory and only flushing to disk when needed
Mikkel Denkerabcada0245fa
diff --git a/configs/webgraph_master.toml b/configs/webgraph_master.tomlindex 0573ccf2..6fe78ffa 100644--- a/configs/webgraph_master.toml+++ b/configs/webgraph_master.toml@@ -2,16 +2,8 @@ type = "Webgraph"mode = "Master"warc_paths_file = "warc.paths"-addr = "0.0.0.0:1337"workers = ["0.0.0.0:1338"]--# [warc_source]-# type = "S3"-# name = "us-east-1"-# endpoint = "https://s3.us-east-1.amazonaws.com"-# bucket = "commoncrawl"-[warc_source]type = "HTTP"base_url = "https://data.commoncrawl.org/"diff --git a/configs/webgraph_worker.toml b/configs/webgraph_worker.tomlnew file mode 100644index 00000000..69d289b3--- /dev/null+++ b/configs/webgraph_worker.toml@@ -0,0 +1,4 @@+type = "Webgraph"+mode = "Worker"++addr = "0.0.0.0:1338"\ No newline at end of filediff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex d4deef0f..ff84f3a6 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -17,6 +17,8 @@ use clap::Parser;use cuely::entrypoint::{Indexer, WebgraphBuilder};use cuely::Config;use std::fs;+use tracing::Level;+use tracing_subscriber::FmtSubscriber;#[derive(Parser, Debug)]struct Args {@@ -25,6 +27,11 @@ struct Args {#[tokio::main]async fn main() {+ let subscriber = FmtSubscriber::builder()+ .with_max_level(Level::DEBUG)+ .finish();+ tracing::subscriber::set_global_default(subscriber).unwrap();+let args = Args::parse();let raw_config = fs::read_to_string(args.config_file).expect("Failed to read config file");@@ -40,6 +47,7 @@ async fn main() {}Config::Webgraph(config) => WebgraphBuilder::from(config).run()+ .await.expect("Failed to build webgraph"),}}diff --git a/src/directory.rs b/src/directory.rsnew file mode 100644index 00000000..ac1b24ed--- /dev/null+++ b/src/directory.rs@@ -0,0 +1,91 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::Result;+use serde::{Deserialize, Serialize};+use std::fs;++#[derive(Serialize, Deserialize, Debug)]+pub enum DirEntry {+ Folder {+ name: String,+ entries: Vec<DirEntry>,+ },+ File {+ name: String,+ content: Vec<u8>,+ },+}++fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {+ let mut res = Vec::new();++ for f in fs::read_dir(path)? {+ let entry = f?;+ let metadata = entry.metadata()?;++ if metadata.is_dir() {+ res.push(DirEntry::Folder {+ name: entry.path().as_os_str().to_str().unwrap().to_string(),+ entries: iterate_children(entry.path().as_os_str().to_str().unwrap())?,+ })+ } else if metadata.is_file() {+ res.push(DirEntry::File {+ name: entry.path().as_os_str().to_str().unwrap().to_string(),+ content: fs::read(entry.path())?,+ })+ }+ }++ Ok(res)+}++fn recreate_folder(entry: &DirEntry) -> Result<()> {+ match entry {+ DirEntry::Folder { name, entries } => {+ fs::create_dir(name)?;++ for entry in entries {+ recreate_folder(entry)?;+ }++ Ok(())+ }+ DirEntry::File { name, content } => Ok(fs::write(name, content)?),+ }+}++pub fn scan_folder(path: String) -> Result<DirEntry> {+ Ok(DirEntry::Folder {+ entries: iterate_children(&path)?,+ name: path,+ })+}++pub fn serialize(path: String) -> Result<Vec<u8>> {+ let folder = scan_folder(path)?;+ Ok(bincode::serialize(&folder)?)+}++pub fn deserialize(bytes: &[u8]) -> Result<String> {+ let entry = bincode::deserialize(bytes)?;+ recreate_folder(&entry)?;++ match entry {+ DirEntry::Folder { name, entries: _ } => Ok(name),+ DirEntry::File { name, content: _ } => Ok(name),+ }+}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex d84f1b29..8e50f8f1 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -87,48 +87,4 @@ impl Indexer {Ok(())}-- async fn download_from_http(warc_path: String, base_url: String) -> Result<Vec<u8>> {- let mut url = base_url;- if !url.ends_with('/') {- url += "/";- }- url += &warc_path;-- let client = reqwest::Client::new();- let res = client.get(url).send().await?;-- Ok(Vec::from(&res.bytes().await?[..]))- }-- async fn download_from_s3(- key: String,- region_name: String,- region_endpoint: String,- bucket: String,- ) -> Result<Vec<u8>> {- let region = Region::Custom {- name: region_name,- endpoint: region_endpoint,- };-- let client = S3Client::new(region);-- let obj = client- .get_object(GetObjectRequest {- bucket,- key,- ..Default::default()- })- .await?;-- let mut res = Vec::new();- obj.body- .ok_or(Error::S3DownloadError)?- .into_async_read()- .read_to_end(&mut res)- .await?;-- Ok(res)- }}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 0d311df9..0738cdfe 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -14,11 +14,20 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- mapreduce::{Map, Reduce},- webgraph::{SledStore, Webgraph},- Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,+ mapreduce::{Map, MapReduce, Reduce, Worker},+ warc::WarcFile,+ webgraph::{Node, SledStore, Webgraph},+ webpage::{self, Html},+ HttpConfig, Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,};use serde::{Deserialize, Serialize};+use std::{+ fs::File,+ io::{self, BufRead},+ net::SocketAddr,+ path::Path,+};+use tracing::{debug, info};pub struct WebgraphBuilder {config: WebgraphConfig,@@ -31,17 +40,52 @@ impl From<WebgraphConfig> for WebgraphBuilder {}#[derive(Serialize, Deserialize)]-struct SingleJob {- warc_source: WarcSource,+struct Job {+ http_config: HttpConfig,warc_path: String,}-#[derive(Serialize, Deserialize)]-struct Job(Vec<SingleJob>);-impl Map<Webgraph<SledStore>> for Job {fn map(self) -> Webgraph<SledStore> {- todo!()+ let name = self.warc_path.split("/").last().unwrap();++ info!("processing {}", name);++ let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));++ let file = futures::executor::block_on(WarcFile::download(+ WarcSource::HTTP(self.http_config),+ &self.warc_path,+ ))+ .unwrap();++ for record in file.records() {+ if let Ok(record) = record {+ let webpage = Html::parse(&record.response.body, &record.request.url);+ for link in webpage+ .links()+ .into_iter()+ .filter(|link| {+ link.destination.starts_with("http://")+ || link.destination.starts_with("https://")+ })+ .filter(|link| {+ webpage::domain(&link.source) != webpage::domain(&link.destination)+ })+ {+ debug!("inserting link {:?}", link);+ graph.insert(+ Node::from(link.source),+ Node::from(link.destination),+ link.text,+ );+ }+ }+ }++ info!("{} done", name);++ graph}}@@ -53,18 +97,52 @@ impl Reduce<Webgraph<SledStore>> for Webgraph<SledStore> {}impl WebgraphBuilder {- fn run_master(config: &WebgraphMasterConfig) -> Result<()> {- todo!();+ async fn run_master(config: &WebgraphMasterConfig) -> Result<()> {+ info!("Running master for webgraph construction");++ let file = File::open(&config.warc_paths_file)?;+ let mut warc_paths = Vec::new();++ for line in io::BufReader::new(file).lines() {+ warc_paths.push(line?);+ }++ let workers: Vec<SocketAddr> = config+ .workers+ .iter()+ .map(|worker| worker.parse().unwrap())+ .collect();++ let http_config = if let WarcSource::HTTP(http_config) = config.warc_source.clone() {+ Some(http_config)+ } else {+ None+ };++ let http_config = http_config.unwrap();++ warc_paths+ .into_iter()+ .map(|warc_path| Job {+ http_config: http_config.clone(),+ warc_path,+ })+ .map_reduce(&workers)+ .await+ .expect("failed to build webgraph");++ Ok(())}- fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {- todo!();+ async fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {+ Worker::run::<Job, Webgraph<SledStore>>(config.addr.parse::<SocketAddr>().unwrap()).await?;+ Ok(())}- pub fn run(&self) -> Result<()> {+ pub async fn run(&self) -> Result<()> {match &self.config {- WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),- WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),+ WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config).await,+ WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config).await,}}}diff --git a/src/lib.rs b/src/lib.rsindex bd79d89c..226bc926 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -24,6 +24,7 @@ pub mod index;pub mod mapreduce;+mod directory;mod query;mod ranking;mod schema;@@ -58,7 +59,6 @@ pub enum WebgraphConfig {pub struct WebgraphMasterConfig {warc_source: WarcSource,warc_paths_file: String,- addr: String,workers: Vec<String>,}@@ -120,6 +120,9 @@ pub enum Error {#[error("Error while serializing/deserializing to/from bytes")]Serialization(#[from] bincode::Error),++ #[error("Error executing distributed jobs")]+ MapReduce(#[from] mapreduce::Error),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 134a8e45..1336e166 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -26,7 +26,7 @@ struct RemoteWorker {impl RemoteWorker {fn retry_strategy() -> impl Iterator<Item = Duration> {- ExponentialBackoff::from_millis(10).take(4)+ ExponentialBackoff::from_millis(10).take(5)}async fn connect(&self) -> Result<TcpStream> {