web search done right
Adjust just-text default parameters since we now have the option to only index webpages with clean text
Mikkel Denker4947fccdfee2
diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex 811b225a..a789af50 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,6 +1,7 @@centrality_store_path = "./data/centrality"limit_warc_files = 1download_images = false+minimum_clean_words = 40# crawl_stability_path = "data/crawl-stability"[warc_source]diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex ec50c64e..56d675d3 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -134,6 +134,7 @@ fn create_inverted_index() -> Result<()> {base_path: out_path_tmp.to_str().unwrap().to_string(),host_centrality_threshold: None,max_num_segments: 1,+ minimum_clean_words: Some(40),};let webgraph_path = Path::new(DATA_PATH).join("webgraph");@@ -182,49 +183,11 @@ fn parse_topics() -> Result<()> {topics.save(Path::new(DATA_PATH).join("human_annotations"))}-fn create_topic_centrality() {- let index_path = Path::new(DATA_PATH)- .join("index")- .to_str()- .unwrap()- .to_string();- let topics_path = Path::new(DATA_PATH)- .join("human_annotations")- .to_str()- .unwrap()- .to_string();- let webgraph_path = Path::new(DATA_PATH)- .join("webgraph")- .to_str()- .unwrap()- .to_string();- let online_harmonic_path = Path::new(DATA_PATH)- .join("centrality")- .join("online_harmonic")- .to_str()- .unwrap()- .to_string();- let output_path = Path::new(DATA_PATH)- .join("topic_centrality")- .to_str()- .unwrap()- .to_string();-- super::topic_centrality::run(- index_path,- topics_path,- webgraph_path,- online_harmonic_path,- output_path,- );-}-fn index_files() -> Result<()> {create_webgraph()?;calculate_centrality();parse_topics()?;create_inverted_index()?;- create_topic_centrality();create_entity_index()?;Ok(())diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 496b959d..0f866c3d 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -53,6 +53,7 @@ pub struct Job {pub warc_paths: Vec<String>,pub base_path: String,pub host_centrality_threshold: Option<f64>,+ pub minimum_clean_words: Option<usize>,pub max_num_segments: u32,}@@ -129,14 +130,24 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {}}+ html.parse_text();+ if let Some(minimum_clean_words) = job.minimum_clean_words {+ match html.clean_text() {+ Some(clean_text) => {+ if clean_text.split_whitespace().count() < minimum_clean_words {+ continue;+ }+ }+ None => continue,+ }+ }+let crawl_stability = worker.crawl_stabilty.as_ref().and_then(|stability| stability.get(&html.url().site().to_string())).unwrap_or_default();- html.parse_text();-let backlinks: Vec<Link> = worker.webgraph.as_ref()@@ -326,6 +337,7 @@ impl Indexer {.clone().unwrap_or_else(|| "data/index".to_string()),max_num_segments: config.final_num_segments.unwrap_or(20),+ minimum_clean_words: config.minimum_clean_words,}).collect_vec().into_iter(),@@ -411,6 +423,7 @@ impl Indexer {.clone().unwrap_or_else(|| "data/index".to_string()),max_num_segments: config.final_num_segments.unwrap_or(20),+ minimum_clean_words: config.minimum_clean_words,}),).unwrap_or_default();diff --git a/core/src/lib.rs b/core/src/lib.rsindex b03aee29..dcb5b49d 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -81,6 +81,7 @@ pub struct IndexingMasterConfig {download_images: Option<bool>,host_centrality_threshold: Option<f64>,index_base_path: Option<String>,+ minimum_clean_words: Option<usize>,}#[derive(Debug, Deserialize, Clone)]@@ -97,6 +98,7 @@ pub struct IndexingLocalConfig {host_centrality_threshold: Option<f64>,topics_path: Option<String>,centrality_store_path: String,+ minimum_clean_words: Option<usize>,}#[derive(Debug, Deserialize, Clone)]diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rsindex 6acd5551..73552243 100644--- a/core/src/spell/dictionary.rs+++ b/core/src/spell/dictionary.rs@@ -332,7 +332,7 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {}pub fn insert_page(&mut self, webpage: &Webpage) {- let text = webpage.html.clean_text().unwrap_or_default();+ let text = webpage.html.clean_text().cloned().unwrap_or_default();let mut stream = Normal::default().token_stream(text.as_str());diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rsindex a8b95db5..896dab59 100644--- a/core/src/webgraph/centrality/topic.rs+++ b/core/src/webgraph/centrality/topic.rs@@ -179,7 +179,9 @@ impl TopicCentrality {.filter_map(|node| webgraph.node2id(&node)).collect();- assert!(!top_sites.is_empty());+ if top_sites.is_empty() {+ continue;+ }let scorer = harmonic.scorer(&top_sites, &[]);diff --git a/core/src/webpage/just_text.rs b/core/src/webpage/just_text.rsindex 85bc8c7f..69fdecdd 100644--- a/core/src/webpage/just_text.rs+++ b/core/src/webpage/just_text.rs@@ -24,12 +24,30 @@ use super::Preprocessor;// implementation of the JustText algorithm described in this thesis: https://is.muni.cz/th/45523/fi_d/phdthesis.pdf// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py-const MAX_LINK_DENSITY_DEFAULT: f64 = 0.2;-const LENGTH_LOW_DEFAULT: usize = 70;-const LENGTH_HIGH_DEFAULT: usize = 200;-const STOPWORDS_LOW_DEFAULT: f64 = 0.30;-const STOPWORDS_HIGH_DEFAULT: f64 = 0.32;-const MAX_HEADING_DISTANCE_DEFAULT: usize = 200;+/// Paragraphs that have a length density above this threshold+/// will be classified as bad.+const MAX_LINK_DENSITY_DEFAULT: f64 = 0.2; // originally 0.2++/// paragraphs that has a length less than this is+/// either bad or short.+const LENGTH_LOW_DEFAULT: usize = 50; // originally 70++/// Paragraphs that has a length greater than this are classified+/// as good given they have a higher stopword density than `STOPWORDS_HIGH_DEFAULT`.+const LENGTH_HIGH_DEFAULT: usize = 100; // originally 200++/// If stopword density is greater than this threshold, the paragraph+/// is classified as neargood.+const STOPWORDS_LOW_DEFAULT: f64 = 0.15; // originaly 0.30++/// If stopword density is greater than this threshold, the paragraph+/// is either classified as good or neargood depending on the+/// `LENGTH_HIGH_DEFAULT`.+const STOPWORDS_HIGH_DEFAULT: f64 = 0.2; // originaly 0.32++/// Max number of heading characters that will be classified+/// as good or neargood.+const MAX_HEADING_DISTANCE_DEFAULT: usize = 200; // originally 200#[derive(Debug, Clone)]enum IntermediateClassification {diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex b9866086..aebe9758 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -568,8 +568,8 @@ impl Html {}}- pub fn clean_text(&self) -> Option<String> {- self.clean_text.clone()+ pub fn clean_text(&self) -> Option<&String> {+ self.clean_text.as_ref()}fn calculate_all_text(paragraphs: &[Paragraph], lang: &Lang) -> Option<String> {@@ -659,7 +659,7 @@ impl Html {}fn pretokenize_clean_text(&self) -> PreTokenizedString {- let clean_text = self.clean_text().unwrap_or_default();+ let clean_text = self.clean_text().cloned().unwrap_or_default();self.pretokenize_string(clean_text)}@@ -1217,7 +1217,7 @@ mod tests {text: "Link to example".to_string()}]);- assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));+ assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));let mut expected_meta = HashMap::new();expected_meta.insert("name".to_string(), "meta1".to_string());@@ -1266,7 +1266,7 @@ mod tests {let webpage = Html::parse(&raw, "https://www.example.com/whatever");- assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));+ assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));}#[test]@@ -1880,4 +1880,12 @@ mod tests {]);}++ #[test]+ fn stackoverflow_question_has_clean_text() {+ let stackoverflow = include_str!("../../testcases/schema_org/stackoverflow_with_code.html");+ let html = Html::parse(stackoverflow, "https://www.example.com");++ assert!(html.clean_text().is_some());+ }}