web search done right
Normalize redirects in web graph
Mikkel Denkeraf1e43206b28
diff --git a/configs/indexer/master.toml b/configs/indexer/master.tomldeleted file mode 100644index 0122d89b..00000000--- a/configs/indexer/master.toml+++ /dev/null@@ -1,7 +0,0 @@-limit_warc_files = 100-workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]--[warc_source]-type = "HTTP"-base_url = "https://data.commoncrawl.org/"-warc_paths_file = "warc.paths"\ No newline at end of filediff --git a/configs/indexer/profile.toml b/configs/indexer/profile.tomldeleted file mode 100644index b45b7607..00000000--- a/configs/indexer/profile.toml+++ /dev/null@@ -1,7 +0,0 @@-warc_paths_file = "warc.paths"-centrality_store_path = "./data/centrality"--[warc_source]-type = "Local"-folder = "./data/warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]diff --git a/configs/webgraph/master.toml b/configs/webgraph/master.tomldeleted file mode 100644index 8cd58012..00000000--- a/configs/webgraph/master.toml+++ /dev/null@@ -1,9 +0,0 @@-level = "host"-limit_warc_files = 10000-workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]-graph_base_path = "/mnt/webgraph"--[warc_source]-type = "HTTP"-base_url = "https://data.commoncrawl.org/"-warc_paths_file = "warc.paths"diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex b197312e..6e87ae5d 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -282,13 +282,20 @@ impl From<u64> for UrlId {}}-struct RedirectDb {+pub struct RedirectDb {inner: rocksdb::DB,}impl RedirectDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let inner = rocksdb::DB::open_default(path.as_ref())?;+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);++ let mut block_options = BlockBasedOptions::default();+ block_options.set_bloom_filter(64.0, true);+ options.set_block_based_table_factory(&block_options);++ let inner = rocksdb::DB::open(&options, path.as_ref())?;Ok(Self { inner })}@@ -305,6 +312,18 @@ impl RedirectDb {Ok(())}++ pub fn get(&self, from: &Url) -> Result<Option<Url>> {+ let url_bytes = bincode::serialize(from)?;+ let redirect_bytes = self.inner.get(url_bytes)?;++ if let Some(redirect_bytes) = redirect_bytes {+ let redirect: Url = bincode::deserialize(&redirect_bytes)?;+ return Ok(Some(redirect));+ }++ Ok(None)+ }}struct UrlToInsert {diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex a6715f79..cafddf8e 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -30,7 +30,7 @@ use self::{};pub mod coordinator;-mod crawl_db;+pub mod crawl_db;mod robots_txt;mod warc_writer;mod worker;diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex d6ea0514..afa53ed1 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -83,7 +83,8 @@ fn create_webgraph() -> Result<()> {level: crate::WebgraphLevel::Host,};- let graph = webgraph::process_job(&job);+ let worker = webgraph::WebgraphWorker { redirect: None };+ let graph = worker.process_job(&job);std::fs::rename(graph.path, out_path)?;std::fs::remove_dir_all(&out_path_tmp)?;diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 4f303d1f..ea9f1fb9 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use chrono::Utc;-use std::net::SocketAddr;use std::path::Path;use std::thread;@@ -26,15 +25,15 @@ use tracing::{debug, info, trace};use crate::entrypoint::download_all_warc_files;use crate::executor::Executor;use crate::index::{FrozenIndex, Index};-use crate::mapreduce::{Manager, Map, Reduce, Worker};+use crate::mapreduce::{Map, Reduce, Worker};use crate::ranking::centrality_store::IndexerCentralityStore;use crate::ranking::SignalAggregator;use crate::warc::WarcFile;use crate::webgraph::{Node, Webgraph, WebgraphBuilder};use crate::webpage::{Html, Link, Webpage};use crate::{- human_website_annotations, HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig,- Result, S3Config, WarcSource,+ human_website_annotations, HttpConfig, IndexingLocalConfig, LocalConfig, Result, S3Config,+ WarcSource,};pub struct Indexer {}@@ -292,90 +291,7 @@ impl Reduce<Index> for Index {}impl Indexer {- pub fn run_master(config: &IndexingMasterConfig) -> Result<()> {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- .block_on(async {- info!("Running master for index construction");-- let warc_paths = config.warc_source.paths().unwrap();-- let workers: Vec<SocketAddr> = config- .workers- .iter()- .map(|worker| worker.parse().unwrap())- .collect();-- let job_config: JobConfig = config.warc_source.clone().into();-- let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(- warc_paths- .into_iter()- .skip(config.skip_warc_files.unwrap_or(0))- .take(config.limit_warc_files.unwrap_or(usize::MAX))- .chunks(config.batch_size.unwrap_or(1))- .into_iter()- .map(|warc_paths| Job {- source_config: job_config.clone(),- warc_paths: warc_paths.collect_vec(),- host_centrality_threshold: config.host_centrality_threshold,- base_path: config- .index_base_path- .clone()- .unwrap_or_else(|| "data/index".to_string()),- minimum_clean_words: config.minimum_clean_words,- })- .collect_vec()- .into_iter(),- );-- let manager = Manager::new(&workers);- let mut index: Index = manager- .run::<IndexingWorker, Job, FrozenIndex, Index>(warc_paths)- .await- .unwrap();-- index- .inverted_index- .merge_into_max_segments(num_cpus::get() as u32)- .unwrap();- });-- Ok(())- }-- pub fn run_worker(- worker_addr: String,- host_centrality_store_path: String,- page_centrality_store_path: Option<String>,- webgraph_path: Option<String>,- topics_path: Option<String>,- ) -> Result<()> {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- .block_on(async {- IndexingWorker::new(- host_centrality_store_path,- page_centrality_store_path,- webgraph_path,- topics_path,- )- .run::<Job, FrozenIndex>(- worker_addr- .parse::<SocketAddr>()- .expect("Could not parse worker address"),- )- .await- .unwrap();- });- Ok(())- }-- pub fn run_locally(config: &IndexingLocalConfig) -> Result<()> {+ pub fn run(config: &IndexingLocalConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;let job_config: JobConfig = config.warc_source.clone().into();diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 68b0fb4c..ce1ce6e7 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -14,18 +14,18 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ crawler::crawl_db::RedirectDb,entrypoint::download_all_warc_files,- mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},+ mapreduce::{Map, Reduce, Worker},warc::WarcFile,webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,- HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLevel, WebgraphLocalConfig,- WebgraphMasterConfig,+ HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphConstructConfig, WebgraphLevel,};use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};-use std::{net::SocketAddr, path::Path, thread};+use std::{path::Path, thread};use tokio::pin;use tracing::{info, trace};@@ -73,76 +73,95 @@ fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {WebgraphBuilder::new(path).open()}-pub fn process_job(job: &Job) -> webgraph::Webgraph {- let name = job.warc_paths.first().unwrap().split('/').last().unwrap();-- info!("processing {}", name);-- let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));+pub struct WebgraphWorker {+ pub redirect: Option<RedirectDb>,+}- let source = WarcSource::from(job.config.clone());+impl WebgraphWorker {+ pub fn process_job(&self, job: &Job) -> webgraph::Webgraph {+ let name = job.warc_paths.first().unwrap().split('/').last().unwrap();- let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);- pin!(warc_files);+ info!("processing {}", name);- for warc_path in warc_files.by_ref() {- let name = warc_path.split('/').last().unwrap();- let path = Path::new(&job.graph_base_path)- .join("warc_files")- .join(name);+ let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));- if let Ok(file) = WarcFile::open(&path) {- for record in file.records().flatten() {- let webpage = Html::parse_without_text(&record.response.body, &record.request.url);- for link in webpage- .anchor_links()- .into_iter()- .filter(|link| matches!(link.destination.protocol(), "http" | "https"))- .filter(|link| link.source.domain() != link.destination.domain())- .filter(|link| link.matches_url_regex())- {- if link.source.domain() == link.destination.domain() {- continue;- }+ let source = WarcSource::from(job.config.clone());- trace!("inserting link {:?}", link);- let mut source = Node::from(link.source);- source.remove_protocol();+ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);+ pin!(warc_files);- let mut destination = Node::from(link.destination);- destination.remove_protocol();+ for warc_path in warc_files.by_ref() {+ let name = warc_path.split('/').last().unwrap();+ let path = Path::new(&job.graph_base_path)+ .join("warc_files")+ .join(name);- if let WebgraphLevel::Host = job.level {- source = source.into_host();- destination = destination.into_host();+ if let Ok(file) = WarcFile::open(&path) {+ for record in file.records().flatten() {+ let webpage =+ Html::parse_without_text(&record.response.body, &record.request.url);+ for link in webpage+ .anchor_links()+ .into_iter()+ .filter(|link| matches!(link.destination.protocol(), "http" | "https"))+ .filter(|link| link.source.domain() != link.destination.domain())+ .filter(|link| link.matches_url_regex())+ {+ let source = link.source.clone();+ let mut destination = link.destination.clone();++ if let Some(redirect) = &self.redirect {+ if let Some(new_destination) = redirect.get(&destination).unwrap() {+ trace!("redirecting {:?} to {:?}", destination, new_destination);+ destination = new_destination;+ }+ }++ if source.domain() == destination.domain() {+ continue;+ }++ trace!("inserting link {:?}", link);+ let mut source = Node::from(source);+ source.remove_protocol();++ let mut destination = Node::from(destination);+ destination.remove_protocol();++ if let WebgraphLevel::Host = job.level {+ source = source.into_host();+ destination = destination.into_host();+ }++ graph.insert(source, destination, link.text);}-- graph.insert(source, destination, link.text);}}- }- graph.commit();+ graph.commit();- std::fs::remove_file(path).unwrap();- }- graph.merge_segments(1);+ std::fs::remove_file(path).unwrap();+ }+ graph.merge_segments(1);- info!("{} done", name);+ info!("{} done", name);- graph+ graph+ }}-impl Map<StatelessWorker, FrozenWebgraph> for Job {- fn map(&self, _worker: &StatelessWorker) -> FrozenWebgraph {- let graph = process_job(self);+impl Worker for WebgraphWorker {}++impl Map<WebgraphWorker, FrozenWebgraph> for Job {+ fn map(&self, worker: &WebgraphWorker) -> FrozenWebgraph {+ let graph = worker.process_job(self);graph.into()}}-impl Map<StatelessWorker, GraphPointer> for Job {- fn map(&self, _worker: &StatelessWorker) -> GraphPointer {- let graph = process_job(self);+impl Map<WebgraphWorker, GraphPointer> for Job {+ fn map(&self, worker: &WebgraphWorker) -> GraphPointer {+ let graph = worker.process_job(self);GraphPointer { path: graph.path }}}@@ -190,81 +209,17 @@ impl Reduce<GraphPointer> for webgraph::Webgraph {pub struct Webgraph {}impl Webgraph {- pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- .block_on(async {- info!("Running master for webgraph construction");-- let warc_paths = config.warc_source.paths().unwrap();-- let workers: Vec<SocketAddr> = config- .workers- .iter()- .map(|worker| worker.parse().unwrap())- .collect();-- let job_config = JobConfig::from(config.warc_source.clone());-- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(- warc_paths- .into_iter()- .chunks(config.batch_size.unwrap_or(1))- .into_iter()- .map(|warc_paths| Job {- config: job_config.clone(),- level: config.level.clone(),- warc_paths: warc_paths.into_iter().collect(),- graph_base_path: config- .graph_base_path- .clone()- .unwrap_or_else(|| "data/webgraph".to_string()),- })- .collect::<Vec<_>>()- .into_iter(),- );-- if let Some(limit) = config.limit_warc_files {- warc_paths = Box::new(warc_paths.take(limit));- }-- let manager = Manager::new(&workers);- let _graph: webgraph::Webgraph = manager- .run::<StatelessWorker, Job, webgraph::FrozenWebgraph, webgraph::Webgraph>(- warc_paths,- )- .await- .unwrap();- });-- Ok(())- }-- pub fn run_worker(worker_addr: String) -> Result<()> {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- .block_on(async {- StatelessWorker::default()- .run::<Job, FrozenWebgraph>(- worker_addr- .parse::<SocketAddr>()- .expect("Could not parse worker address"),- )- .await- .unwrap();- });- Ok(())- }-- pub fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {+ pub fn run(config: &WebgraphConstructConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;let job_config = JobConfig::from(config.warc_source.clone());- let worker = StatelessWorker::default();++ let redirect = match &config.redirect_db_path {+ Some(path) => Some(RedirectDb::open(path)?),+ None => None,+ };++ let worker = WebgraphWorker { redirect };let graphs: Vec<_> = warc_paths.into_iter()diff --git a/core/src/lib.rs b/core/src/lib.rsindex 4622ae5d..f746c6c2 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -84,18 +84,6 @@ pub mod webgraph;pub mod webpage;mod widgets;-#[derive(Debug, Deserialize, Clone)]-pub struct IndexingMasterConfig {- limit_warc_files: Option<usize>,- skip_warc_files: Option<usize>,- warc_source: WarcSource,- workers: Vec<String>,- batch_size: Option<usize>,- host_centrality_threshold: Option<f64>,- index_base_path: Option<String>,- minimum_clean_words: Option<usize>,-}-#[derive(Debug, Deserialize, Clone)]pub struct IndexingLocalConfig {limit_warc_files: Option<usize>,@@ -119,18 +107,9 @@ pub enum WebgraphLevel {}#[derive(Debug, Deserialize, Clone)]-pub struct WebgraphMasterConfig {- level: WebgraphLevel,- limit_warc_files: Option<usize>,- warc_source: WarcSource,- workers: Vec<String>,- graph_base_path: Option<String>,- batch_size: Option<usize>,-}--#[derive(Debug, Deserialize, Clone)]-pub struct WebgraphLocalConfig {+pub struct WebgraphConstructConfig {level: WebgraphLevel,+ redirect_db_path: Option<String>,limit_warc_files: Option<usize>,warc_source: WarcSource,graph_base_path: Option<String>,diff --git a/core/src/main.rs b/core/src/main.rsindex 0b612f3e..c93edd28 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -126,13 +126,7 @@ enum CentralityType {#[derive(Subcommand)]enum WebgraphOptions {- Master {- config_path: String,- },- Worker {- address: String,- },- Local {+ Create {config_path: String,},Merge {@@ -148,17 +142,7 @@ enum WebgraphOptions {#[derive(Subcommand)]enum IndexingOptions {- Master {- config_path: String,- },- Worker {- address: String,- host_centrality_store_path: String,- page_centrality_store_path: Option<String>,- webgraph_path: Option<String>,- topics_path: Option<String>,- },- Local {+ Search {config_path: String,},Entity {@@ -185,28 +169,9 @@ fn main() -> Result<()> {match args.command {Commands::Indexer { options } => match options {- IndexingOptions::Master { config_path } => {+ IndexingOptions::Search { config_path } => {let config = load_toml_config(config_path);- entrypoint::Indexer::run_master(&config)?;- }- IndexingOptions::Worker {- address,- host_centrality_store_path,- page_centrality_store_path,- webgraph_path,- topics_path,- } => {- entrypoint::Indexer::run_worker(- address,- host_centrality_store_path,- page_centrality_store_path,- webgraph_path,- topics_path,- )?;- }- IndexingOptions::Local { config_path } => {- let config = load_toml_config(config_path);- entrypoint::Indexer::run_locally(&config)?;+ entrypoint::Indexer::run(&config)?;}IndexingOptions::Entity {wikipedia_dump_path,@@ -236,16 +201,9 @@ fn main() -> Result<()> {tracing::info!("Done");}Commands::Webgraph { options } => match options {- WebgraphOptions::Master { config_path } => {- let config = load_toml_config(config_path);- entrypoint::Webgraph::run_master(&config)?;- }- WebgraphOptions::Worker { address } => {- entrypoint::Webgraph::run_worker(address)?;- }- WebgraphOptions::Local { config_path } => {+ WebgraphOptions::Create { config_path } => {let config = load_toml_config(config_path);- entrypoint::Webgraph::run_locally(&config)?;+ entrypoint::Webgraph::run(&config)?;}WebgraphOptions::Merge {mut paths,diff --git a/justfile b/justfileindex 7957f9c3..7a9dd550 100644--- a/justfile+++ b/justfile@@ -1,11 +1,3 @@-@profile-indexer:- sudo rm -rf data/index- just cargo flamegraph --root -- indexer local configs/indexer/profile.toml--@webgraph:- rm -rf data/webgraph- just cargo run --release -- webgraph local configs/webgraph/local.toml-@frontend-rerun *ARGS:cd frontend; npm run build./scripts/run_frontend.py {{ARGS}}@@ -31,13 +23,6 @@just setup {{ARGS}}just cargo run --release --all-features -- configure {{ARGS}}-@centrality webgraph output:- ./scripts/build_harmonic {{webgraph}} {{output}}--@entity:- rm -rf data/entity- just cargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entity-@setup_python_env:rm -rf .venvpython3 -m venv .venvdiff --git a/scripts/build_harmonic b/scripts/build_harmonicdeleted file mode 100755index 0de84d75..00000000--- a/scripts/build_harmonic+++ /dev/null@@ -1,4 +0,0 @@-#!/bin/bash-cargo run --release centrality $1 $2 harmonic-cargo run --release centrality $1 $2 online-harmonic-cargo run --release centrality $1 $2 similarity\ No newline at end of file