Skip to content

web search done right

Normalize redirects in web graph

Mikkel Denkeraf1e43206b28
diff --git a/configs/indexer/master.toml b/configs/indexer/master.toml
deleted file mode 100644
index 0122d89b..00000000
--- a/configs/indexer/master.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-limit_warc_files = 100
-workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]
-
-[warc_source]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "warc.paths"
\ No newline at end of file
diff --git a/configs/indexer/profile.toml b/configs/indexer/profile.toml
deleted file mode 100644
index b45b7607..00000000
--- a/configs/indexer/profile.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-warc_paths_file = "warc.paths"
-centrality_store_path = "./data/centrality"
-
-[warc_source]
-type = "Local"
-folder = "./data/warc_files"
-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
diff --git a/configs/webgraph/master.toml b/configs/webgraph/master.toml
deleted file mode 100644
index 8cd58012..00000000
--- a/configs/webgraph/master.toml
+++ /dev/null
@@ -1,9 +0,0 @@
-level = "host"
-limit_warc_files = 10000
-workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]
-graph_base_path = "/mnt/webgraph"
-
-[warc_source]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "warc.paths"
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index b197312e..6e87ae5d 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -282,13 +282,20 @@ impl From<u64> for UrlId {
}
}
-struct RedirectDb {
+pub struct RedirectDb {
inner: rocksdb::DB,
}
impl RedirectDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let inner = rocksdb::DB::open_default(path.as_ref())?;
+ let mut options = rocksdb::Options::default();
+ options.create_if_missing(true);
+
+ let mut block_options = BlockBasedOptions::default();
+ block_options.set_bloom_filter(64.0, true);
+ options.set_block_based_table_factory(&block_options);
+
+ let inner = rocksdb::DB::open(&options, path.as_ref())?;
Ok(Self { inner })
}
@@ -305,6 +312,18 @@ impl RedirectDb {
Ok(())
}
+
+ pub fn get(&self, from: &Url) -> Result<Option<Url>> {
+ let url_bytes = bincode::serialize(from)?;
+ let redirect_bytes = self.inner.get(url_bytes)?;
+
+ if let Some(redirect_bytes) = redirect_bytes {
+ let redirect: Url = bincode::deserialize(&redirect_bytes)?;
+ return Ok(Some(redirect));
+ }
+
+ Ok(None)
+ }
}
struct UrlToInsert {
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index a6715f79..cafddf8e 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -30,7 +30,7 @@ use self::{
};
pub mod coordinator;
-mod crawl_db;
+pub mod crawl_db;
mod robots_txt;
mod warc_writer;
mod worker;
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index d6ea0514..afa53ed1 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -83,7 +83,8 @@ fn create_webgraph() -> Result<()> {
level: crate::WebgraphLevel::Host,
};
- let graph = webgraph::process_job(&job);
+ let worker = webgraph::WebgraphWorker { redirect: None };
+ let graph = worker.process_job(&job);
std::fs::rename(graph.path, out_path)?;
std::fs::remove_dir_all(&out_path_tmp)?;
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 4f303d1f..ea9f1fb9 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -14,7 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use chrono::Utc;
-use std::net::SocketAddr;
use std::path::Path;
use std::thread;
@@ -26,15 +25,15 @@ use tracing::{debug, info, trace};
use crate::entrypoint::download_all_warc_files;
use crate::executor::Executor;
use crate::index::{FrozenIndex, Index};
-use crate::mapreduce::{Manager, Map, Reduce, Worker};
+use crate::mapreduce::{Map, Reduce, Worker};
use crate::ranking::centrality_store::IndexerCentralityStore;
use crate::ranking::SignalAggregator;
use crate::warc::WarcFile;
use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
use crate::webpage::{Html, Link, Webpage};
use crate::{
- human_website_annotations, HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig,
- Result, S3Config, WarcSource,
+ human_website_annotations, HttpConfig, IndexingLocalConfig, LocalConfig, Result, S3Config,
+ WarcSource,
};
pub struct Indexer {}
@@ -292,90 +291,7 @@ impl Reduce<Index> for Index {
}
impl Indexer {
- pub fn run_master(config: &IndexingMasterConfig) -> Result<()> {
- tokio::runtime::Builder::new_current_thread()
- .enable_all()
- .build()
- .unwrap()
- .block_on(async {
- info!("Running master for index construction");
-
- let warc_paths = config.warc_source.paths().unwrap();
-
- let workers: Vec<SocketAddr> = config
- .workers
- .iter()
- .map(|worker| worker.parse().unwrap())
- .collect();
-
- let job_config: JobConfig = config.warc_source.clone().into();
-
- let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
- warc_paths
- .into_iter()
- .skip(config.skip_warc_files.unwrap_or(0))
- .take(config.limit_warc_files.unwrap_or(usize::MAX))
- .chunks(config.batch_size.unwrap_or(1))
- .into_iter()
- .map(|warc_paths| Job {
- source_config: job_config.clone(),
- warc_paths: warc_paths.collect_vec(),
- host_centrality_threshold: config.host_centrality_threshold,
- base_path: config
- .index_base_path
- .clone()
- .unwrap_or_else(|| "data/index".to_string()),
- minimum_clean_words: config.minimum_clean_words,
- })
- .collect_vec()
- .into_iter(),
- );
-
- let manager = Manager::new(&workers);
- let mut index: Index = manager
- .run::<IndexingWorker, Job, FrozenIndex, Index>(warc_paths)
- .await
- .unwrap();
-
- index
- .inverted_index
- .merge_into_max_segments(num_cpus::get() as u32)
- .unwrap();
- });
-
- Ok(())
- }
-
- pub fn run_worker(
- worker_addr: String,
- host_centrality_store_path: String,
- page_centrality_store_path: Option<String>,
- webgraph_path: Option<String>,
- topics_path: Option<String>,
- ) -> Result<()> {
- tokio::runtime::Builder::new_current_thread()
- .enable_all()
- .build()
- .unwrap()
- .block_on(async {
- IndexingWorker::new(
- host_centrality_store_path,
- page_centrality_store_path,
- webgraph_path,
- topics_path,
- )
- .run::<Job, FrozenIndex>(
- worker_addr
- .parse::<SocketAddr>()
- .expect("Could not parse worker address"),
- )
- .await
- .unwrap();
- });
- Ok(())
- }
-
- pub fn run_locally(config: &IndexingLocalConfig) -> Result<()> {
+ pub fn run(config: &IndexingLocalConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
let job_config: JobConfig = config.warc_source.clone().into();
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 68b0fb4c..ce1ce6e7 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -14,18 +14,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ crawler::crawl_db::RedirectDb,
entrypoint::download_all_warc_files,
- mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},
+ mapreduce::{Map, Reduce, Worker},
warc::WarcFile,
webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
- HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLevel, WebgraphLocalConfig,
- WebgraphMasterConfig,
+ HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphConstructConfig, WebgraphLevel,
};
use itertools::Itertools;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
-use std::{net::SocketAddr, path::Path, thread};
+use std::{path::Path, thread};
use tokio::pin;
use tracing::{info, trace};
@@ -73,76 +73,95 @@ fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
WebgraphBuilder::new(path).open()
}
-pub fn process_job(job: &Job) -> webgraph::Webgraph {
- let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
-
- info!("processing {}", name);
-
- let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));
+pub struct WebgraphWorker {
+ pub redirect: Option<RedirectDb>,
+}
- let source = WarcSource::from(job.config.clone());
+impl WebgraphWorker {
+ pub fn process_job(&self, job: &Job) -> webgraph::Webgraph {
+ let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
- let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);
- pin!(warc_files);
+ info!("processing {}", name);
- for warc_path in warc_files.by_ref() {
- let name = warc_path.split('/').last().unwrap();
- let path = Path::new(&job.graph_base_path)
- .join("warc_files")
- .join(name);
+ let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));
- if let Ok(file) = WarcFile::open(&path) {
- for record in file.records().flatten() {
- let webpage = Html::parse_without_text(&record.response.body, &record.request.url);
- for link in webpage
- .anchor_links()
- .into_iter()
- .filter(|link| matches!(link.destination.protocol(), "http" | "https"))
- .filter(|link| link.source.domain() != link.destination.domain())
- .filter(|link| link.matches_url_regex())
- {
- if link.source.domain() == link.destination.domain() {
- continue;
- }
+ let source = WarcSource::from(job.config.clone());
- trace!("inserting link {:?}", link);
- let mut source = Node::from(link.source);
- source.remove_protocol();
+ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);
+ pin!(warc_files);
- let mut destination = Node::from(link.destination);
- destination.remove_protocol();
+ for warc_path in warc_files.by_ref() {
+ let name = warc_path.split('/').last().unwrap();
+ let path = Path::new(&job.graph_base_path)
+ .join("warc_files")
+ .join(name);
- if let WebgraphLevel::Host = job.level {
- source = source.into_host();
- destination = destination.into_host();
+ if let Ok(file) = WarcFile::open(&path) {
+ for record in file.records().flatten() {
+ let webpage =
+ Html::parse_without_text(&record.response.body, &record.request.url);
+ for link in webpage
+ .anchor_links()
+ .into_iter()
+ .filter(|link| matches!(link.destination.protocol(), "http" | "https"))
+ .filter(|link| link.source.domain() != link.destination.domain())
+ .filter(|link| link.matches_url_regex())
+ {
+ let source = link.source.clone();
+ let mut destination = link.destination.clone();
+
+ if let Some(redirect) = &self.redirect {
+ if let Some(new_destination) = redirect.get(&destination).unwrap() {
+ trace!("redirecting {:?} to {:?}", destination, new_destination);
+ destination = new_destination;
+ }
+ }
+
+ if source.domain() == destination.domain() {
+ continue;
+ }
+
+ trace!("inserting link {:?}", link);
+ let mut source = Node::from(source);
+ source.remove_protocol();
+
+ let mut destination = Node::from(destination);
+ destination.remove_protocol();
+
+ if let WebgraphLevel::Host = job.level {
+ source = source.into_host();
+ destination = destination.into_host();
+ }
+
+ graph.insert(source, destination, link.text);
}
-
- graph.insert(source, destination, link.text);
}
}
- }
- graph.commit();
+ graph.commit();
- std::fs::remove_file(path).unwrap();
- }
- graph.merge_segments(1);
+ std::fs::remove_file(path).unwrap();
+ }
+ graph.merge_segments(1);
- info!("{} done", name);
+ info!("{} done", name);
- graph
+ graph
+ }
}
-impl Map<StatelessWorker, FrozenWebgraph> for Job {
- fn map(&self, _worker: &StatelessWorker) -> FrozenWebgraph {
- let graph = process_job(self);
+impl Worker for WebgraphWorker {}
+
+impl Map<WebgraphWorker, FrozenWebgraph> for Job {
+ fn map(&self, worker: &WebgraphWorker) -> FrozenWebgraph {
+ let graph = worker.process_job(self);
graph.into()
}
}
-impl Map<StatelessWorker, GraphPointer> for Job {
- fn map(&self, _worker: &StatelessWorker) -> GraphPointer {
- let graph = process_job(self);
+impl Map<WebgraphWorker, GraphPointer> for Job {
+ fn map(&self, worker: &WebgraphWorker) -> GraphPointer {
+ let graph = worker.process_job(self);
GraphPointer { path: graph.path }
}
}
@@ -190,81 +209,17 @@ impl Reduce<GraphPointer> for webgraph::Webgraph {
pub struct Webgraph {}
impl Webgraph {
- pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
- tokio::runtime::Builder::new_current_thread()
- .enable_all()
- .build()
- .unwrap()
- .block_on(async {
- info!("Running master for webgraph construction");
-
- let warc_paths = config.warc_source.paths().unwrap();
-
- let workers: Vec<SocketAddr> = config
- .workers
- .iter()
- .map(|worker| worker.parse().unwrap())
- .collect();
-
- let job_config = JobConfig::from(config.warc_source.clone());
-
- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
- warc_paths
- .into_iter()
- .chunks(config.batch_size.unwrap_or(1))
- .into_iter()
- .map(|warc_paths| Job {
- config: job_config.clone(),
- level: config.level.clone(),
- warc_paths: warc_paths.into_iter().collect(),
- graph_base_path: config
- .graph_base_path
- .clone()
- .unwrap_or_else(|| "data/webgraph".to_string()),
- })
- .collect::<Vec<_>>()
- .into_iter(),
- );
-
- if let Some(limit) = config.limit_warc_files {
- warc_paths = Box::new(warc_paths.take(limit));
- }
-
- let manager = Manager::new(&workers);
- let _graph: webgraph::Webgraph = manager
- .run::<StatelessWorker, Job, webgraph::FrozenWebgraph, webgraph::Webgraph>(
- warc_paths,
- )
- .await
- .unwrap();
- });
-
- Ok(())
- }
-
- pub fn run_worker(worker_addr: String) -> Result<()> {
- tokio::runtime::Builder::new_current_thread()
- .enable_all()
- .build()
- .unwrap()
- .block_on(async {
- StatelessWorker::default()
- .run::<Job, FrozenWebgraph>(
- worker_addr
- .parse::<SocketAddr>()
- .expect("Could not parse worker address"),
- )
- .await
- .unwrap();
- });
- Ok(())
- }
-
- pub fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {
+ pub fn run(config: &WebgraphConstructConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
let job_config = JobConfig::from(config.warc_source.clone());
- let worker = StatelessWorker::default();
+
+ let redirect = match &config.redirect_db_path {
+ Some(path) => Some(RedirectDb::open(path)?),
+ None => None,
+ };
+
+ let worker = WebgraphWorker { redirect };
let graphs: Vec<_> = warc_paths
.into_iter()
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 4622ae5d..f746c6c2 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -84,18 +84,6 @@ pub mod webgraph;
pub mod webpage;
mod widgets;
-#[derive(Debug, Deserialize, Clone)]
-pub struct IndexingMasterConfig {
- limit_warc_files: Option<usize>,
- skip_warc_files: Option<usize>,
- warc_source: WarcSource,
- workers: Vec<String>,
- batch_size: Option<usize>,
- host_centrality_threshold: Option<f64>,
- index_base_path: Option<String>,
- minimum_clean_words: Option<usize>,
-}
-
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
limit_warc_files: Option<usize>,
@@ -119,18 +107,9 @@ pub enum WebgraphLevel {
}
#[derive(Debug, Deserialize, Clone)]
-pub struct WebgraphMasterConfig {
- level: WebgraphLevel,
- limit_warc_files: Option<usize>,
- warc_source: WarcSource,
- workers: Vec<String>,
- graph_base_path: Option<String>,
- batch_size: Option<usize>,
-}
-
-#[derive(Debug, Deserialize, Clone)]
-pub struct WebgraphLocalConfig {
+pub struct WebgraphConstructConfig {
level: WebgraphLevel,
+ redirect_db_path: Option<String>,
limit_warc_files: Option<usize>,
warc_source: WarcSource,
graph_base_path: Option<String>,
diff --git a/core/src/main.rs b/core/src/main.rs
index 0b612f3e..c93edd28 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -126,13 +126,7 @@ enum CentralityType {
#[derive(Subcommand)]
enum WebgraphOptions {
- Master {
- config_path: String,
- },
- Worker {
- address: String,
- },
- Local {
+ Create {
config_path: String,
},
Merge {
@@ -148,17 +142,7 @@ enum WebgraphOptions {
#[derive(Subcommand)]
enum IndexingOptions {
- Master {
- config_path: String,
- },
- Worker {
- address: String,
- host_centrality_store_path: String,
- page_centrality_store_path: Option<String>,
- webgraph_path: Option<String>,
- topics_path: Option<String>,
- },
- Local {
+ Search {
config_path: String,
},
Entity {
@@ -185,28 +169,9 @@ fn main() -> Result<()> {
match args.command {
Commands::Indexer { options } => match options {
- IndexingOptions::Master { config_path } => {
+ IndexingOptions::Search { config_path } => {
let config = load_toml_config(config_path);
- entrypoint::Indexer::run_master(&config)?;
- }
- IndexingOptions::Worker {
- address,
- host_centrality_store_path,
- page_centrality_store_path,
- webgraph_path,
- topics_path,
- } => {
- entrypoint::Indexer::run_worker(
- address,
- host_centrality_store_path,
- page_centrality_store_path,
- webgraph_path,
- topics_path,
- )?;
- }
- IndexingOptions::Local { config_path } => {
- let config = load_toml_config(config_path);
- entrypoint::Indexer::run_locally(&config)?;
+ entrypoint::Indexer::run(&config)?;
}
IndexingOptions::Entity {
wikipedia_dump_path,
@@ -236,16 +201,9 @@ fn main() -> Result<()> {
tracing::info!("Done");
}
Commands::Webgraph { options } => match options {
- WebgraphOptions::Master { config_path } => {
- let config = load_toml_config(config_path);
- entrypoint::Webgraph::run_master(&config)?;
- }
- WebgraphOptions::Worker { address } => {
- entrypoint::Webgraph::run_worker(address)?;
- }
- WebgraphOptions::Local { config_path } => {
+ WebgraphOptions::Create { config_path } => {
let config = load_toml_config(config_path);
- entrypoint::Webgraph::run_locally(&config)?;
+ entrypoint::Webgraph::run(&config)?;
}
WebgraphOptions::Merge {
mut paths,
diff --git a/justfile b/justfile
index 7957f9c3..7a9dd550 100644
--- a/justfile
+++ b/justfile
@@ -1,11 +1,3 @@
-@profile-indexer:
- sudo rm -rf data/index
- just cargo flamegraph --root -- indexer local configs/indexer/profile.toml
-
-@webgraph:
- rm -rf data/webgraph
- just cargo run --release -- webgraph local configs/webgraph/local.toml
-
@frontend-rerun *ARGS:
cd frontend; npm run build
./scripts/run_frontend.py {{ARGS}}
@@ -31,13 +23,6 @@
just setup {{ARGS}}
just cargo run --release --all-features -- configure {{ARGS}}
-@centrality webgraph output:
- ./scripts/build_harmonic {{webgraph}} {{output}}
-
-@entity:
- rm -rf data/entity
- just cargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entity
-
@setup_python_env:
rm -rf .venv
python3 -m venv .venv
diff --git a/scripts/build_harmonic b/scripts/build_harmonic
deleted file mode 100755
index 0de84d75..00000000
--- a/scripts/build_harmonic
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-cargo run --release centrality $1 $2 harmonic
-cargo run --release centrality $1 $2 online-harmonic
-cargo run --release centrality $1 $2 similarity
\ No newline at end of file