Skip to content

web search done right

host centrality threshold during indexing

Mikkel Denker473c2325e0ca
diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rs
index 54eba9f4..7d85dc5d 100644
--- a/benches/search-preindexed-goggle.rs
+++ b/benches/search-preindexed-goggle.rs
@@ -20,6 +20,7 @@ macro_rules! bench {
selected_region: None,
goggle_program: Some($goggle.to_string()),
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
})
diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rs
index 87dfa85e..bfd70f3f 100644
--- a/benches/search-preindexed.rs
+++ b/benches/search-preindexed.rs
@@ -19,6 +19,7 @@ macro_rules! bench {
selected_region: None,
goggle_program: None,
skip_pages: None,
+ site_rankings: None,
})
.unwrap()
})
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index 7c3fa1e9..4e7c19e9 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,6 +1,7 @@
warc_paths_file = "warc.paths"
centrality_store_path = "./data/centrality"
limit_warc_files = 3
+host_centrality_threshold = 0.0000000001
download_images = false
[warc_source]
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index c4532da2..38f60af2 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -47,6 +47,7 @@ struct Job {
download_images: bool,
warc_paths: Vec<String>,
base_path: String,
+ host_centrality_threshold: Option<f64>,
}
struct IndexingWorker {
@@ -99,7 +100,21 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
None => true,
})
{
- let html = Html::parse(&record.response.body, &record.request.url);
+ let mut html = Html::parse_without_text(&record.response.body, &record.request.url);
+
+ let host_centrality = worker
+ .host_centrality_store
+ .get(html.url().host_without_specific_subdomains())
+ .unwrap_or_default();
+
+ if let Some(host_centrality_threshold) = job.host_centrality_threshold {
+ if host_centrality < host_centrality_threshold {
+ continue;
+ }
+ }
+
+ html.parse_text();
+
let backlinks: Vec<Link> = worker
.webgraph
.as_ref()
@@ -116,11 +131,6 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
})
.unwrap_or_else(Vec::new);
- let host_centrality = worker
- .host_centrality_store
- .get(html.url().host_without_specific_subdomains())
- .unwrap_or_default();
-
let page_centrality = worker
.page_centrality_store
.get(html.url().raw())
@@ -257,6 +267,7 @@ impl Indexer {
source_config: job_config.clone(),
warc_paths: warc_paths.collect_vec(),
download_images: config.download_images.unwrap_or(true),
+ host_centrality_threshold: config.host_centrality_threshold,
base_path: config
.index_base_path
.clone()
@@ -329,6 +340,7 @@ impl Indexer {
source_config: job_config.clone(),
warc_paths: warc_paths.collect_vec(),
download_images: config.download_images.unwrap_or(true),
+ host_centrality_threshold: config.host_centrality_threshold,
base_path: config
.output_path
.clone()
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 10eee39b..fa7b1edb 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -68,8 +68,7 @@ fn process_job(job: &Job) -> webgraph::Webgraph {
for warc_path in &warc_files {
if let Ok(file) = WarcFile::open(warc_path) {
for record in file.records().flatten() {
- let webpage =
- Html::parse_including_text(&record.response.body, &record.request.url, false);
+ let webpage = Html::parse_without_text(&record.response.body, &record.request.url);
for link in webpage
.links()
.into_iter()
diff --git a/src/lib.rs b/src/lib.rs
index 8c12a1b6..70b572ff 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -69,6 +69,7 @@ pub struct IndexingMasterConfig {
workers: Vec<String>,
batch_size: Option<usize>,
download_images: Option<bool>,
+ host_centrality_threshold: Option<f64>,
index_base_path: Option<String>,
}
@@ -81,6 +82,7 @@ pub struct IndexingLocalConfig {
webgraph_path: Option<String>,
output_path: Option<String>,
download_images: Option<bool>,
+ host_centrality_threshold: Option<f64>,
centrality_store_path: String,
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 222d69f0..aed42b6c 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -261,45 +261,45 @@ pub struct Html {
impl Html {
pub fn parse(html: &str, url: &str) -> Self {
- Self::parse_including_text(html, url, true)
- }
-
- pub fn parse_including_text(html: &str, url: &str, include_text: bool) -> Self {
- let root = kuchiki::parse_html().one(html);
+ let mut html = Self::parse_without_text(html, url);
- let mut all_text = None;
- let mut clean_text = None;
- let mut lang = None;
+ html.parse_text();
- if include_text {
- let paragraphs = JustText::paragraphs(root.clone());
-
- lang = paragraphs
- .iter()
- .max_by_key(|paragraph| paragraph.text.len())
- .and_then(|paragraph| {
- whatlang::detect(&paragraph.text).and_then(|info| {
- if info.is_reliable() && info.confidence() > 0.95 {
- Some(info.lang())
- } else {
- None
- }
- })
- });
+ html
+ }
- all_text = Html::calculate_all_text(&paragraphs, &lang.unwrap_or(Lang::Eng));
- clean_text = Html::calculate_clean_text(&paragraphs, &lang.unwrap_or(Lang::Eng));
- }
+ pub fn parse_without_text(html: &str, url: &str) -> Self {
+ let root = kuchiki::parse_html().one(html);
Self {
root,
- all_text,
- clean_text,
- lang,
+ all_text: None,
+ clean_text: None,
+ lang: None,
url: url.to_string().into(),
}
}
+ pub fn parse_text(&mut self) {
+ let paragraphs = JustText::paragraphs(self.root.clone());
+
+ self.lang = paragraphs
+ .iter()
+ .max_by_key(|paragraph| paragraph.text.len())
+ .and_then(|paragraph| {
+ whatlang::detect(&paragraph.text).and_then(|info| {
+ if info.is_reliable() && info.confidence() > 0.95 {
+ Some(info.lang())
+ } else {
+ None
+ }
+ })
+ });
+
+ self.all_text = Html::calculate_all_text(&paragraphs, &self.lang.unwrap_or(Lang::Eng));
+ self.clean_text = Html::calculate_clean_text(&paragraphs, &self.lang.unwrap_or(Lang::Eng));
+ }
+
pub fn links(&self) -> Vec<Link> {
let mut links = Vec::new();
let mut open_links = Vec::new();