web search done right
host centrality threshold during indexing
Mikkel Denker473c2325e0ca
diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rsindex 54eba9f4..7d85dc5d 100644--- a/benches/search-preindexed-goggle.rs+++ b/benches/search-preindexed-goggle.rs@@ -20,6 +20,7 @@ macro_rules! bench {selected_region: None,goggle_program: Some($goggle.to_string()),skip_pages: None,+ site_rankings: None,}).unwrap()})diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rsindex 87dfa85e..bfd70f3f 100644--- a/benches/search-preindexed.rs+++ b/benches/search-preindexed.rs@@ -19,6 +19,7 @@ macro_rules! bench {selected_region: None,goggle_program: None,skip_pages: None,+ site_rankings: None,}).unwrap()})diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex 7c3fa1e9..4e7c19e9 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,6 +1,7 @@warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"limit_warc_files = 3+host_centrality_threshold = 0.0000000001download_images = false[warc_source]diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex c4532da2..38f60af2 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -47,6 +47,7 @@ struct Job {download_images: bool,warc_paths: Vec<String>,base_path: String,+ host_centrality_threshold: Option<f64>,}struct IndexingWorker {@@ -99,7 +100,21 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {None => true,}){- let html = Html::parse(&record.response.body, &record.request.url);+ let mut html = Html::parse_without_text(&record.response.body, &record.request.url);++ let host_centrality = worker+ .host_centrality_store+ .get(html.url().host_without_specific_subdomains())+ .unwrap_or_default();++ if let Some(host_centrality_threshold) = job.host_centrality_threshold {+ if host_centrality < host_centrality_threshold {+ continue;+ }+ }++ html.parse_text();+let backlinks: Vec<Link> = worker.webgraph.as_ref()@@ -116,11 +131,6 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}).unwrap_or_else(Vec::new);- let host_centrality = worker- .host_centrality_store- .get(html.url().host_without_specific_subdomains())- .unwrap_or_default();-let page_centrality = worker.page_centrality_store.get(html.url().raw())@@ -257,6 +267,7 @@ impl Indexer {source_config: job_config.clone(),warc_paths: warc_paths.collect_vec(),download_images: config.download_images.unwrap_or(true),+ host_centrality_threshold: config.host_centrality_threshold,base_path: config.index_base_path.clone()@@ -329,6 +340,7 @@ impl Indexer {source_config: job_config.clone(),warc_paths: warc_paths.collect_vec(),download_images: config.download_images.unwrap_or(true),+ host_centrality_threshold: config.host_centrality_threshold,base_path: config.output_path.clone()diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 10eee39b..fa7b1edb 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -68,8 +68,7 @@ fn process_job(job: &Job) -> webgraph::Webgraph {for warc_path in &warc_files {if let Ok(file) = WarcFile::open(warc_path) {for record in file.records().flatten() {- let webpage =- Html::parse_including_text(&record.response.body, &record.request.url, false);+ let webpage = Html::parse_without_text(&record.response.body, &record.request.url);for link in webpage.links().into_iter()diff --git a/src/lib.rs b/src/lib.rsindex 8c12a1b6..70b572ff 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -69,6 +69,7 @@ pub struct IndexingMasterConfig {workers: Vec<String>,batch_size: Option<usize>,download_images: Option<bool>,+ host_centrality_threshold: Option<f64>,index_base_path: Option<String>,}@@ -81,6 +82,7 @@ pub struct IndexingLocalConfig {webgraph_path: Option<String>,output_path: Option<String>,download_images: Option<bool>,+ host_centrality_threshold: Option<f64>,centrality_store_path: String,}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 222d69f0..aed42b6c 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -261,45 +261,45 @@ pub struct Html {impl Html {pub fn parse(html: &str, url: &str) -> Self {- Self::parse_including_text(html, url, true)- }-- pub fn parse_including_text(html: &str, url: &str, include_text: bool) -> Self {- let root = kuchiki::parse_html().one(html);+ let mut html = Self::parse_without_text(html, url);- let mut all_text = None;- let mut clean_text = None;- let mut lang = None;+ html.parse_text();- if include_text {- let paragraphs = JustText::paragraphs(root.clone());-- lang = paragraphs- .iter()- .max_by_key(|paragraph| paragraph.text.len())- .and_then(|paragraph| {- whatlang::detect(¶graph.text).and_then(|info| {- if info.is_reliable() && info.confidence() > 0.95 {- Some(info.lang())- } else {- None- }- })- });+ html+ }- all_text = Html::calculate_all_text(¶graphs, &lang.unwrap_or(Lang::Eng));- clean_text = Html::calculate_clean_text(¶graphs, &lang.unwrap_or(Lang::Eng));- }+ pub fn parse_without_text(html: &str, url: &str) -> Self {+ let root = kuchiki::parse_html().one(html);Self {root,- all_text,- clean_text,- lang,+ all_text: None,+ clean_text: None,+ lang: None,url: url.to_string().into(),}}+ pub fn parse_text(&mut self) {+ let paragraphs = JustText::paragraphs(self.root.clone());++ self.lang = paragraphs+ .iter()+ .max_by_key(|paragraph| paragraph.text.len())+ .and_then(|paragraph| {+ whatlang::detect(¶graph.text).and_then(|info| {+ if info.is_reliable() && info.confidence() > 0.95 {+ Some(info.lang())+ } else {+ None+ }+ })+ });++ self.all_text = Html::calculate_all_text(¶graphs, &self.lang.unwrap_or(Lang::Eng));+ self.clean_text = Html::calculate_clean_text(¶graphs, &self.lang.unwrap_or(Lang::Eng));+ }+pub fn links(&self) -> Vec<Link> {let mut links = Vec::new();let mut open_links = Vec::new();