web search done right
less locking+cloning in webgraph. Default trait for webgraph store
Mikkel Denker2db70f23544a
diff --git a/.gitignore b/.gitignoreindex 4fffb2f8..79b16f44 100644--- a/.gitignore+++ b/.gitignore@@ -1,2 +1,5 @@/target/Cargo.lock+flamegraph.svg+warc_files/+webgraphdiff --git a/Cargo.toml b/Cargo.tomlindex 748a0047..4dc65dde 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -41,3 +41,6 @@ whatlang = "0.16.0"[dev-dependencies]maplit = "1.0.2"++[profile.release]+debug = true\ No newline at end of filediff --git a/configs/webgraph_master.toml b/configs/webgraph_master.tomldeleted file mode 100644index 6fe78ffa..00000000--- a/configs/webgraph_master.toml+++ /dev/null@@ -1,9 +0,0 @@-type = "Webgraph"-mode = "Master"--warc_paths_file = "warc.paths"-workers = ["0.0.0.0:1338"]--[warc_source]-type = "HTTP"-base_url = "https://data.commoncrawl.org/"diff --git a/configs/webgraph_worker.toml b/configs/webgraph_worker.tomldeleted file mode 100644index 69d289b3..00000000--- a/configs/webgraph_worker.toml+++ /dev/null@@ -1,4 +0,0 @@-type = "Webgraph"-mode = "Worker"--addr = "0.0.0.0:1338"\ No newline at end of filediff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex ff84f3a6..931e15ea 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -28,7 +28,7 @@ struct Args {#[tokio::main]async fn main() {let subscriber = FmtSubscriber::builder()- .with_max_level(Level::DEBUG)+ .with_max_level(Level::INFO).finish();tracing::subscriber::set_global_default(subscriber).unwrap();diff --git a/src/directory.rs b/src/directory.rsindex ac1b24ed..acc6d3a1 100644--- a/src/directory.rs+++ b/src/directory.rs@@ -16,7 +16,7 @@use crate::Result;use serde::{Deserialize, Serialize};-use std::fs;+use std::{fs, path::Path};#[derive(Serialize, Deserialize, Debug)]pub enum DirEntry {@@ -31,10 +31,11 @@ pub enum DirEntry {}fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {+ dbg!(&path);let mut res = Vec::new();for f in fs::read_dir(path)? {- let entry = f?;+ let entry = dbg!(f?);let metadata = entry.metadata()?;if metadata.is_dir() {@@ -56,6 +57,9 @@ fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {fn recreate_folder(entry: &DirEntry) -> Result<()> {match entry {DirEntry::Folder { name, entries } => {+ if Path::new(name).exists() {+ fs::remove_dir_all(name)?;+ }fs::create_dir(name)?;for entry in entries {diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 8e50f8f1..838ca17b 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -15,15 +15,12 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use futures::prelude::*;use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};-use rusoto_core::Region;-use rusoto_s3::{GetObjectRequest, S3Client, S3};use std::fs::File;use std::io::{self, BufRead};-use tokio::io::AsyncReadExt;use crate::warc::WarcFile;use crate::webpage::Html;-use crate::{Error, IndexingConfig, Result};+use crate::{IndexingConfig, Result};pub struct Indexer {warc_paths: Vec<String>,diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 0738cdfe..fc44727b 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -18,15 +18,11 @@ use crate::{warc::WarcFile,webgraph::{Node, SledStore, Webgraph},webpage::{self, Html},- HttpConfig, Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,+ HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,+ WebgraphMasterConfig, WebgraphWorkerConfig,};use serde::{Deserialize, Serialize};-use std::{- fs::File,- io::{self, BufRead},- net::SocketAddr,- path::Path,-};+use std::{net::SocketAddr, path::Path};use tracing::{debug, info};pub struct WebgraphBuilder {@@ -39,47 +35,51 @@ impl From<WebgraphConfig> for WebgraphBuilder {}}+#[derive(Serialize, Deserialize, Clone)]+enum JobConfig {+ HTTP(HttpConfig),+ Local(LocalConfig),+}+#[derive(Serialize, Deserialize)]struct Job {- http_config: HttpConfig,+ config: JobConfig,warc_path: String,}-impl Map<Webgraph<SledStore>> for Job {- fn map(self) -> Webgraph<SledStore> {- let name = self.warc_path.split("/").last().unwrap();+impl Map<Webgraph> for Job {+ fn map(self) -> Webgraph {+ let name = self.warc_path.split('/').last().unwrap();info!("processing {}", name);let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));- let file = futures::executor::block_on(WarcFile::download(- WarcSource::HTTP(self.http_config),- &self.warc_path,- ))- .unwrap();-- for record in file.records() {- if let Ok(record) = record {- let webpage = Html::parse(&record.response.body, &record.request.url);- for link in webpage- .links()- .into_iter()- .filter(|link| {- link.destination.starts_with("http://")- || link.destination.starts_with("https://")- })- .filter(|link| {- webpage::domain(&link.source) != webpage::domain(&link.destination)- })- {- debug!("inserting link {:?}", link);- graph.insert(- Node::from(link.source),- Node::from(link.destination),- link.text,- );- }+ let source = match self.config {+ JobConfig::HTTP(config) => WarcSource::HTTP(config),+ JobConfig::Local(config) => WarcSource::Local(config),+ };++ let file =+ futures::executor::block_on(WarcFile::download(source, &self.warc_path)).unwrap();++ for record in file.records().flatten() {+ let webpage = Html::parse(&record.response.body, &record.request.url);+ for link in webpage+ .links()+ .into_iter()+ .filter(|link| {+ link.destination.starts_with("http://")+ || link.destination.starts_with("https://")+ })+ .filter(|link| webpage::domain(&link.source) != webpage::domain(&link.destination))+ {+ debug!("inserting link {:?}", link);+ graph.insert(+ Node::from(link.source),+ Node::from(link.destination),+ link.text,+ );}}@@ -89,8 +89,8 @@ impl Map<Webgraph<SledStore>> for Job {}}-impl Reduce<Webgraph<SledStore>> for Webgraph<SledStore> {- fn reduce(mut self, other: Webgraph<SledStore>) -> Webgraph<SledStore> {+impl Reduce<Webgraph> for Webgraph {+ fn reduce(mut self, other: Webgraph) -> Webgraph {self.merge(other);self}@@ -100,12 +100,7 @@ impl WebgraphBuilder {async fn run_master(config: &WebgraphMasterConfig) -> Result<()> {info!("Running master for webgraph construction");- let file = File::open(&config.warc_paths_file)?;- let mut warc_paths = Vec::new();-- for line in io::BufReader::new(file).lines() {- warc_paths.push(line?);- }+ let warc_paths = config.warc_source.paths()?;let workers: Vec<SocketAddr> = config.workers@@ -113,18 +108,16 @@ impl WebgraphBuilder {.map(|worker| worker.parse().unwrap()).collect();- let http_config = if let WarcSource::HTTP(http_config) = config.warc_source.clone() {- Some(http_config)- } else {- None+ let job_config = match config.warc_source.clone() {+ WarcSource::S3(_) => todo!("s3 not supported yet"),+ WarcSource::HTTP(config) => JobConfig::HTTP(config),+ WarcSource::Local(config) => JobConfig::Local(config),};- let http_config = http_config.unwrap();-warc_paths.into_iter().map(|warc_path| Job {- http_config: http_config.clone(),+ config: job_config.clone(),warc_path,}).map_reduce(&workers)@@ -135,7 +128,31 @@ impl WebgraphBuilder {}async fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {- Worker::run::<Job, Webgraph<SledStore>>(config.addr.parse::<SocketAddr>().unwrap()).await?;+ Worker::run::<Job, Webgraph>(config.addr.parse::<SocketAddr>().unwrap()).await?;+ Ok(())+ }++ fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {+ let warc_paths = config.warc_source.paths()?;++ let job_config = match config.warc_source.clone() {+ WarcSource::S3(_) => todo!("s3 not supported yet"),+ WarcSource::HTTP(config) => JobConfig::HTTP(config),+ WarcSource::Local(config) => JobConfig::Local(config),+ };++ warc_paths+ .into_iter()+ .map(|path| Job {+ config: job_config.clone(),+ warc_path: path,+ })+ .map(|job| job.map())+ .fold(None, |acc: Option<Webgraph>, elem| match acc {+ Some(acc) => Some(acc.reduce(elem)),+ None => Some(elem),+ });+Ok(())}@@ -143,6 +160,7 @@ impl WebgraphBuilder {match &self.config {WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config).await,WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config).await,+ WebgraphConfig::Local(config) => WebgraphBuilder::run_locally(config),}}}diff --git a/src/lib.rs b/src/lib.rsindex 226bc926..0ce2fa79 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -14,7 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use serde::{Deserialize, Serialize};-use std::io;+use std::fs::File;+use std::io::{self, BufRead};use std::num::ParseIntError;use tantivy::TantivyError;use thiserror::Error;@@ -53,12 +54,12 @@ pub struct IndexingConfig {pub enum WebgraphConfig {Master(WebgraphMasterConfig),Worker(WebgraphWorkerConfig),+ Local(WebgraphLocalConfig),}#[derive(Debug, Deserialize, Clone)]pub struct WebgraphMasterConfig {warc_source: WarcSource,- warc_paths_file: String,workers: Vec<String>,}@@ -67,23 +68,62 @@ pub struct WebgraphWorkerConfig {addr: String,}+#[derive(Debug, Deserialize, Clone)]+pub struct WebgraphLocalConfig {+ warc_source: WarcSource,+}+#[derive(Debug, Serialize, Deserialize, Clone)]#[serde(tag = "type")]pub enum WarcSource {S3(S3Config),HTTP(HttpConfig),+ Local(LocalConfig),+}++impl WarcSource {+ pub fn paths(&self) -> Result<Vec<String>> {+ let mut warc_paths = Vec::new();+ match &self {+ WarcSource::S3(config) => {+ let file = File::open(&config.warc_paths_file)?;+ for line in io::BufReader::new(file).lines() {+ warc_paths.push(line?);+ }+ }+ WarcSource::HTTP(config) => {+ let file = File::open(&config.warc_paths_file)?;+ for line in io::BufReader::new(file).lines() {+ warc_paths.push(line?);+ }+ }+ WarcSource::Local(config) => {+ warc_paths = config.names.clone();+ }+ }++ Ok(warc_paths)+ }+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct LocalConfig {+ folder: String,+ names: Vec<String>,}#[derive(Debug, Serialize, Deserialize, Clone)]pub struct S3Config {name: String,endpoint: String,+ warc_paths_file: String,bucket: String,}#[derive(Debug, Serialize, Deserialize, Clone)]pub struct HttpConfig {base_url: String,+ warc_paths_file: String,}#[derive(Error, Debug)]diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rsindex f89c2364..4ba105c1 100644--- a/src/mapreduce/worker.rs+++ b/src/mapreduce/worker.rs@@ -41,6 +41,7 @@ impl Worker {debug!("received job");let res = job.map();let bytes = bincode::serialize(&res)?;+ debug!("serialized result into {} bytes", bytes.len());stream.write_all(&bytes[..]).await?;}Task::AllFinished => {diff --git a/src/warc.rs b/src/warc.rsindex cacf1c8c..5e044683 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -15,7 +15,9 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{Error, Result, WarcSource};use std::collections::BTreeMap;+use std::fs::File;use std::io::{BufRead, BufReader, Read};+use std::path::Path;use tokio::io::AsyncReadExt;use flate2::read::MultiGzDecoder;@@ -80,9 +82,20 @@ impl WarcFile {WarcSource::HTTP(config) => {WarcFile::download_from_http(path, config.base_url.clone()).await}+ WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),}}+ fn load_from_folder(name: &str, folder: &str) -> Result<Self> {+ let mut bytes = Vec::new();+ let f = File::open(dbg!(Path::new(folder).join(name)))?;++ let mut reader = BufReader::new(f);+ reader.read_to_end(&mut bytes)?;++ Ok(WarcFile::new(bytes))+ }+async fn download_from_http(path: &str, base_url: String) -> Result<Self> {let mut url = base_url;if !url.ends_with('/') {