Skip to content

web search done right

less locking+cloning in webgraph. Default trait for webgraph store

Mikkel Denker2db70f23544a
diff --git a/.gitignore b/.gitignore
index 4fffb2f8..79b16f44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,5 @@
/target
/Cargo.lock
+flamegraph.svg
+warc_files/
+webgraph
diff --git a/Cargo.toml b/Cargo.toml
index 748a0047..4dc65dde 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -41,3 +41,6 @@ whatlang = "0.16.0"
[dev-dependencies]
maplit = "1.0.2"
+
+[profile.release]
+debug = true
\ No newline at end of file
diff --git a/configs/webgraph_master.toml b/configs/webgraph_master.toml
deleted file mode 100644
index 6fe78ffa..00000000
--- a/configs/webgraph_master.toml
+++ /dev/null
@@ -1,9 +0,0 @@
-type = "Webgraph"
-mode = "Master"
-
-warc_paths_file = "warc.paths"
-workers = ["0.0.0.0:1338"]
-
-[warc_source]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
diff --git a/configs/webgraph_worker.toml b/configs/webgraph_worker.toml
deleted file mode 100644
index 69d289b3..00000000
--- a/configs/webgraph_worker.toml
+++ /dev/null
@@ -1,4 +0,0 @@
-type = "Webgraph"
-mode = "Worker"
-
-addr = "0.0.0.0:1338"
\ No newline at end of file
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index ff84f3a6..931e15ea 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -28,7 +28,7 @@ struct Args {
#[tokio::main]
async fn main() {
let subscriber = FmtSubscriber::builder()
- .with_max_level(Level::DEBUG)
+ .with_max_level(Level::INFO)
.finish();
tracing::subscriber::set_global_default(subscriber).unwrap();
diff --git a/src/directory.rs b/src/directory.rs
index ac1b24ed..acc6d3a1 100644
--- a/src/directory.rs
+++ b/src/directory.rs
@@ -16,7 +16,7 @@
use crate::Result;
use serde::{Deserialize, Serialize};
-use std::fs;
+use std::{fs, path::Path};
#[derive(Serialize, Deserialize, Debug)]
pub enum DirEntry {
@@ -31,10 +31,11 @@ pub enum DirEntry {
}
fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {
+ dbg!(&path);
let mut res = Vec::new();
for f in fs::read_dir(path)? {
- let entry = f?;
+ let entry = dbg!(f?);
let metadata = entry.metadata()?;
if metadata.is_dir() {
@@ -56,6 +57,9 @@ fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {
fn recreate_folder(entry: &DirEntry) -> Result<()> {
match entry {
DirEntry::Folder { name, entries } => {
+ if Path::new(name).exists() {
+ fs::remove_dir_all(name)?;
+ }
fs::create_dir(name)?;
for entry in entries {
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 8e50f8f1..838ca17b 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -15,15 +15,12 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use futures::prelude::*;
use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
-use rusoto_core::Region;
-use rusoto_s3::{GetObjectRequest, S3Client, S3};
use std::fs::File;
use std::io::{self, BufRead};
-use tokio::io::AsyncReadExt;
use crate::warc::WarcFile;
use crate::webpage::Html;
-use crate::{Error, IndexingConfig, Result};
+use crate::{IndexingConfig, Result};
pub struct Indexer {
warc_paths: Vec<String>,
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 0738cdfe..fc44727b 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -18,15 +18,11 @@ use crate::{
warc::WarcFile,
webgraph::{Node, SledStore, Webgraph},
webpage::{self, Html},
- HttpConfig, Result, WarcSource, WebgraphConfig, WebgraphMasterConfig, WebgraphWorkerConfig,
+ HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,
+ WebgraphMasterConfig, WebgraphWorkerConfig,
};
use serde::{Deserialize, Serialize};
-use std::{
- fs::File,
- io::{self, BufRead},
- net::SocketAddr,
- path::Path,
-};
+use std::{net::SocketAddr, path::Path};
use tracing::{debug, info};
pub struct WebgraphBuilder {
@@ -39,47 +35,51 @@ impl From<WebgraphConfig> for WebgraphBuilder {
}
}
+#[derive(Serialize, Deserialize, Clone)]
+enum JobConfig {
+ HTTP(HttpConfig),
+ Local(LocalConfig),
+}
+
#[derive(Serialize, Deserialize)]
struct Job {
- http_config: HttpConfig,
+ config: JobConfig,
warc_path: String,
}
-impl Map<Webgraph<SledStore>> for Job {
- fn map(self) -> Webgraph<SledStore> {
- let name = self.warc_path.split("/").last().unwrap();
+impl Map<Webgraph> for Job {
+ fn map(self) -> Webgraph {
+ let name = self.warc_path.split('/').last().unwrap();
info!("processing {}", name);
let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));
- let file = futures::executor::block_on(WarcFile::download(
- WarcSource::HTTP(self.http_config),
- &self.warc_path,
- ))
- .unwrap();
-
- for record in file.records() {
- if let Ok(record) = record {
- let webpage = Html::parse(&record.response.body, &record.request.url);
- for link in webpage
- .links()
- .into_iter()
- .filter(|link| {
- link.destination.starts_with("http://")
- || link.destination.starts_with("https://")
- })
- .filter(|link| {
- webpage::domain(&link.source) != webpage::domain(&link.destination)
- })
- {
- debug!("inserting link {:?}", link);
- graph.insert(
- Node::from(link.source),
- Node::from(link.destination),
- link.text,
- );
- }
+ let source = match self.config {
+ JobConfig::HTTP(config) => WarcSource::HTTP(config),
+ JobConfig::Local(config) => WarcSource::Local(config),
+ };
+
+ let file =
+ futures::executor::block_on(WarcFile::download(source, &self.warc_path)).unwrap();
+
+ for record in file.records().flatten() {
+ let webpage = Html::parse(&record.response.body, &record.request.url);
+ for link in webpage
+ .links()
+ .into_iter()
+ .filter(|link| {
+ link.destination.starts_with("http://")
+ || link.destination.starts_with("https://")
+ })
+ .filter(|link| webpage::domain(&link.source) != webpage::domain(&link.destination))
+ {
+ debug!("inserting link {:?}", link);
+ graph.insert(
+ Node::from(link.source),
+ Node::from(link.destination),
+ link.text,
+ );
}
}
@@ -89,8 +89,8 @@ impl Map<Webgraph<SledStore>> for Job {
}
}
-impl Reduce<Webgraph<SledStore>> for Webgraph<SledStore> {
- fn reduce(mut self, other: Webgraph<SledStore>) -> Webgraph<SledStore> {
+impl Reduce<Webgraph> for Webgraph {
+ fn reduce(mut self, other: Webgraph) -> Webgraph {
self.merge(other);
self
}
@@ -100,12 +100,7 @@ impl WebgraphBuilder {
async fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
info!("Running master for webgraph construction");
- let file = File::open(&config.warc_paths_file)?;
- let mut warc_paths = Vec::new();
-
- for line in io::BufReader::new(file).lines() {
- warc_paths.push(line?);
- }
+ let warc_paths = config.warc_source.paths()?;
let workers: Vec<SocketAddr> = config
.workers
@@ -113,18 +108,16 @@ impl WebgraphBuilder {
.map(|worker| worker.parse().unwrap())
.collect();
- let http_config = if let WarcSource::HTTP(http_config) = config.warc_source.clone() {
- Some(http_config)
- } else {
- None
+ let job_config = match config.warc_source.clone() {
+ WarcSource::S3(_) => todo!("s3 not supported yet"),
+ WarcSource::HTTP(config) => JobConfig::HTTP(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
};
- let http_config = http_config.unwrap();
-
warc_paths
.into_iter()
.map(|warc_path| Job {
- http_config: http_config.clone(),
+ config: job_config.clone(),
warc_path,
})
.map_reduce(&workers)
@@ -135,7 +128,31 @@ impl WebgraphBuilder {
}
async fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {
- Worker::run::<Job, Webgraph<SledStore>>(config.addr.parse::<SocketAddr>().unwrap()).await?;
+ Worker::run::<Job, Webgraph>(config.addr.parse::<SocketAddr>().unwrap()).await?;
+ Ok(())
+ }
+
+ fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {
+ let warc_paths = config.warc_source.paths()?;
+
+ let job_config = match config.warc_source.clone() {
+ WarcSource::S3(_) => todo!("s3 not supported yet"),
+ WarcSource::HTTP(config) => JobConfig::HTTP(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
+ };
+
+ warc_paths
+ .into_iter()
+ .map(|path| Job {
+ config: job_config.clone(),
+ warc_path: path,
+ })
+ .map(|job| job.map())
+ .fold(None, |acc: Option<Webgraph>, elem| match acc {
+ Some(acc) => Some(acc.reduce(elem)),
+ None => Some(elem),
+ });
+
Ok(())
}
@@ -143,6 +160,7 @@ impl WebgraphBuilder {
match &self.config {
WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config).await,
WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config).await,
+ WebgraphConfig::Local(config) => WebgraphBuilder::run_locally(config),
}
}
}
diff --git a/src/lib.rs b/src/lib.rs
index 226bc926..0ce2fa79 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -14,7 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use serde::{Deserialize, Serialize};
-use std::io;
+use std::fs::File;
+use std::io::{self, BufRead};
use std::num::ParseIntError;
use tantivy::TantivyError;
use thiserror::Error;
@@ -53,12 +54,12 @@ pub struct IndexingConfig {
pub enum WebgraphConfig {
Master(WebgraphMasterConfig),
Worker(WebgraphWorkerConfig),
+ Local(WebgraphLocalConfig),
}
#[derive(Debug, Deserialize, Clone)]
pub struct WebgraphMasterConfig {
warc_source: WarcSource,
- warc_paths_file: String,
workers: Vec<String>,
}
@@ -67,23 +68,62 @@ pub struct WebgraphWorkerConfig {
addr: String,
}
+#[derive(Debug, Deserialize, Clone)]
+pub struct WebgraphLocalConfig {
+ warc_source: WarcSource,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(tag = "type")]
pub enum WarcSource {
S3(S3Config),
HTTP(HttpConfig),
+ Local(LocalConfig),
+}
+
+impl WarcSource {
+ pub fn paths(&self) -> Result<Vec<String>> {
+ let mut warc_paths = Vec::new();
+ match &self {
+ WarcSource::S3(config) => {
+ let file = File::open(&config.warc_paths_file)?;
+ for line in io::BufReader::new(file).lines() {
+ warc_paths.push(line?);
+ }
+ }
+ WarcSource::HTTP(config) => {
+ let file = File::open(&config.warc_paths_file)?;
+ for line in io::BufReader::new(file).lines() {
+ warc_paths.push(line?);
+ }
+ }
+ WarcSource::Local(config) => {
+ warc_paths = config.names.clone();
+ }
+ }
+
+ Ok(warc_paths)
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct LocalConfig {
+ folder: String,
+ names: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct S3Config {
name: String,
endpoint: String,
+ warc_paths_file: String,
bucket: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct HttpConfig {
base_url: String,
+ warc_paths_file: String,
}
#[derive(Error, Debug)]
diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rs
index f89c2364..4ba105c1 100644
--- a/src/mapreduce/worker.rs
+++ b/src/mapreduce/worker.rs
@@ -41,6 +41,7 @@ impl Worker {
debug!("received job");
let res = job.map();
let bytes = bincode::serialize(&res)?;
+ debug!("serialized result into {} bytes", bytes.len());
stream.write_all(&bytes[..]).await?;
}
Task::AllFinished => {
diff --git a/src/warc.rs b/src/warc.rs
index cacf1c8c..5e044683 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -15,7 +15,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{Error, Result, WarcSource};
use std::collections::BTreeMap;
+use std::fs::File;
use std::io::{BufRead, BufReader, Read};
+use std::path::Path;
use tokio::io::AsyncReadExt;
use flate2::read::MultiGzDecoder;
@@ -80,9 +82,20 @@ impl WarcFile {
WarcSource::HTTP(config) => {
WarcFile::download_from_http(path, config.base_url.clone()).await
}
+ WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),
}
}
+ fn load_from_folder(name: &str, folder: &str) -> Result<Self> {
+ let mut bytes = Vec::new();
+ let f = File::open(dbg!(Path::new(folder).join(name)))?;
+
+ let mut reader = BufReader::new(f);
+ reader.read_to_end(&mut bytes)?;
+
+ Ok(WarcFile::new(bytes))
+ }
+
async fn download_from_http(path: &str, base_url: String) -> Result<Self> {
let mut url = base_url;
if !url.ends_with('/') {