web search done right
reduce thresholds and minimum_clean_words to make it easier to trigger sidebars during development
Mikkel Denker47c84a0cca38
diff --git a/configs/frontend.toml b/configs/frontend.tomlindex 0930fcc0..0e6b6dfc 100644--- a/configs/frontend.toml+++ b/configs/frontend.toml@@ -9,4 +9,8 @@ fact_check_model_path = "data/fact_model"# lambda_model_path = "data/lambdamart.txt"# qa_model_path = "data/qa_model"bangs_path = "data/bangs.json"-summarizer_path = "data/summarizer"\ No newline at end of file+summarizer_path = "data/summarizer"++[thresholds]+stackoverflow = 0.0+entity_sidebar = 0.0\ No newline at end of filediff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 94940393..71abc93b 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -122,7 +122,7 @@ fn create_inverted_index() -> Result<()> {warc_paths: vec![warc_path.to_str().unwrap().to_string()],base_path: out_path_tmp.to_str().unwrap().to_string(),host_centrality_threshold: None,- minimum_clean_words: Some(40),+ minimum_clean_words: None,};let webgraph_path = Path::new(DATA_PATH).join("webgraph");diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 99eede9b..76ca799f 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -156,6 +156,11 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}html.parse_text();++ if html.empty_all_text() {+ continue;+ }+if let Some(minimum_clean_words) = job.minimum_clean_words {match html.clean_text() {Some(clean_text) => {diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 11c7b2f9..fde2c31c 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -764,6 +764,13 @@ impl Html {self.all_text.clone()}+ pub fn empty_all_text(&self) -> bool {+ match &self.all_text {+ Some(text) => text.is_empty(),+ None => true,+ }+ }+pub fn title(&self) -> Option<String> {if let Some(title) = self.root.select_first("title") {let title = title.text_contents().trim().to_string();