Skip to content

web search done right

reduce thresholds and minimum_clean_words to make it easier to trigger sidebars during development

Mikkel Denker47c84a0cca38
diff --git a/configs/frontend.toml b/configs/frontend.toml
index 0930fcc0..0e6b6dfc 100644
--- a/configs/frontend.toml
+++ b/configs/frontend.toml
@@ -9,4 +9,8 @@ fact_check_model_path = "data/fact_model"
# lambda_model_path = "data/lambdamart.txt"
# qa_model_path = "data/qa_model"
bangs_path = "data/bangs.json"
-summarizer_path = "data/summarizer"
\ No newline at end of file
+summarizer_path = "data/summarizer"
+
+[thresholds]
+stackoverflow = 0.0
+entity_sidebar = 0.0
\ No newline at end of file
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 94940393..71abc93b 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -122,7 +122,7 @@ fn create_inverted_index() -> Result<()> {
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
base_path: out_path_tmp.to_str().unwrap().to_string(),
host_centrality_threshold: None,
- minimum_clean_words: Some(40),
+ minimum_clean_words: None,
};
let webgraph_path = Path::new(DATA_PATH).join("webgraph");
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 99eede9b..76ca799f 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -156,6 +156,11 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
}
html.parse_text();
+
+ if html.empty_all_text() {
+ continue;
+ }
+
if let Some(minimum_clean_words) = job.minimum_clean_words {
match html.clean_text() {
Some(clean_text) => {
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 11c7b2f9..fde2c31c 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -764,6 +764,13 @@ impl Html {
self.all_text.clone()
}
+ pub fn empty_all_text(&self) -> bool {
+ match &self.all_text {
+ Some(text) => text.is_empty(),
+ None => true,
+ }
+ }
+
pub fn title(&self) -> Option<String> {
if let Some(title) = self.root.select_first("title") {
let title = title.text_contents().trim().to_string();