Skip to content

web search done right

Adjust just-text default parameters since we now have the option to only index webpages with clean text

Mikkel Denker4947fccdfee2
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index 811b225a..a789af50 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,6 +1,7 @@
centrality_store_path = "./data/centrality"
limit_warc_files = 1
download_images = false
+minimum_clean_words = 40
# crawl_stability_path = "data/crawl-stability"
[warc_source]
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index ec50c64e..56d675d3 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -134,6 +134,7 @@ fn create_inverted_index() -> Result<()> {
base_path: out_path_tmp.to_str().unwrap().to_string(),
host_centrality_threshold: None,
max_num_segments: 1,
+ minimum_clean_words: Some(40),
};
let webgraph_path = Path::new(DATA_PATH).join("webgraph");
@@ -182,49 +183,11 @@ fn parse_topics() -> Result<()> {
topics.save(Path::new(DATA_PATH).join("human_annotations"))
}
-fn create_topic_centrality() {
- let index_path = Path::new(DATA_PATH)
- .join("index")
- .to_str()
- .unwrap()
- .to_string();
- let topics_path = Path::new(DATA_PATH)
- .join("human_annotations")
- .to_str()
- .unwrap()
- .to_string();
- let webgraph_path = Path::new(DATA_PATH)
- .join("webgraph")
- .to_str()
- .unwrap()
- .to_string();
- let online_harmonic_path = Path::new(DATA_PATH)
- .join("centrality")
- .join("online_harmonic")
- .to_str()
- .unwrap()
- .to_string();
- let output_path = Path::new(DATA_PATH)
- .join("topic_centrality")
- .to_str()
- .unwrap()
- .to_string();
-
- super::topic_centrality::run(
- index_path,
- topics_path,
- webgraph_path,
- online_harmonic_path,
- output_path,
- );
-}
-
fn index_files() -> Result<()> {
create_webgraph()?;
calculate_centrality();
parse_topics()?;
create_inverted_index()?;
- create_topic_centrality();
create_entity_index()?;
Ok(())
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 496b959d..0f866c3d 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -53,6 +53,7 @@ pub struct Job {
pub warc_paths: Vec<String>,
pub base_path: String,
pub host_centrality_threshold: Option<f64>,
+ pub minimum_clean_words: Option<usize>,
pub max_num_segments: u32,
}
@@ -129,14 +130,24 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
}
}
+ html.parse_text();
+ if let Some(minimum_clean_words) = job.minimum_clean_words {
+ match html.clean_text() {
+ Some(clean_text) => {
+ if clean_text.split_whitespace().count() < minimum_clean_words {
+ continue;
+ }
+ }
+ None => continue,
+ }
+ }
+
let crawl_stability = worker
.crawl_stabilty
.as_ref()
.and_then(|stability| stability.get(&html.url().site().to_string()))
.unwrap_or_default();
- html.parse_text();
-
let backlinks: Vec<Link> = worker
.webgraph
.as_ref()
@@ -326,6 +337,7 @@ impl Indexer {
.clone()
.unwrap_or_else(|| "data/index".to_string()),
max_num_segments: config.final_num_segments.unwrap_or(20),
+ minimum_clean_words: config.minimum_clean_words,
})
.collect_vec()
.into_iter(),
@@ -411,6 +423,7 @@ impl Indexer {
.clone()
.unwrap_or_else(|| "data/index".to_string()),
max_num_segments: config.final_num_segments.unwrap_or(20),
+ minimum_clean_words: config.minimum_clean_words,
}),
)
.unwrap_or_default();
diff --git a/core/src/lib.rs b/core/src/lib.rs
index b03aee29..dcb5b49d 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -81,6 +81,7 @@ pub struct IndexingMasterConfig {
download_images: Option<bool>,
host_centrality_threshold: Option<f64>,
index_base_path: Option<String>,
+ minimum_clean_words: Option<usize>,
}
#[derive(Debug, Deserialize, Clone)]
@@ -97,6 +98,7 @@ pub struct IndexingLocalConfig {
host_centrality_threshold: Option<f64>,
topics_path: Option<String>,
centrality_store_path: String,
+ minimum_clean_words: Option<usize>,
}
#[derive(Debug, Deserialize, Clone)]
diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rs
index 6acd5551..73552243 100644
--- a/core/src/spell/dictionary.rs
+++ b/core/src/spell/dictionary.rs
@@ -332,7 +332,7 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {
}
pub fn insert_page(&mut self, webpage: &Webpage) {
- let text = webpage.html.clean_text().unwrap_or_default();
+ let text = webpage.html.clean_text().cloned().unwrap_or_default();
let mut stream = Normal::default().token_stream(text.as_str());
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index a8b95db5..896dab59 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -179,7 +179,9 @@ impl TopicCentrality {
.filter_map(|node| webgraph.node2id(&node))
.collect();
- assert!(!top_sites.is_empty());
+ if top_sites.is_empty() {
+ continue;
+ }
let scorer = harmonic.scorer(&top_sites, &[]);
diff --git a/core/src/webpage/just_text.rs b/core/src/webpage/just_text.rs
index 85bc8c7f..69fdecdd 100644
--- a/core/src/webpage/just_text.rs
+++ b/core/src/webpage/just_text.rs
@@ -24,12 +24,30 @@ use super::Preprocessor;
// implementation of the JustText algorithm described in this thesis: https://is.muni.cz/th/45523/fi_d/phdthesis.pdf
// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py
-const MAX_LINK_DENSITY_DEFAULT: f64 = 0.2;
-const LENGTH_LOW_DEFAULT: usize = 70;
-const LENGTH_HIGH_DEFAULT: usize = 200;
-const STOPWORDS_LOW_DEFAULT: f64 = 0.30;
-const STOPWORDS_HIGH_DEFAULT: f64 = 0.32;
-const MAX_HEADING_DISTANCE_DEFAULT: usize = 200;
+/// Paragraphs that have a length density above this threshold
+/// will be classified as bad.
+const MAX_LINK_DENSITY_DEFAULT: f64 = 0.2; // originally 0.2
+
+/// paragraphs that has a length less than this is
+/// either bad or short.
+const LENGTH_LOW_DEFAULT: usize = 50; // originally 70
+
+/// Paragraphs that has a length greater than this are classified
+/// as good given they have a higher stopword density than `STOPWORDS_HIGH_DEFAULT`.
+const LENGTH_HIGH_DEFAULT: usize = 100; // originally 200
+
+/// If stopword density is greater than this threshold, the paragraph
+/// is classified as neargood.
+const STOPWORDS_LOW_DEFAULT: f64 = 0.15; // originaly 0.30
+
+/// If stopword density is greater than this threshold, the paragraph
+/// is either classified as good or neargood depending on the
+/// `LENGTH_HIGH_DEFAULT`.
+const STOPWORDS_HIGH_DEFAULT: f64 = 0.2; // originaly 0.32
+
+/// Max number of heading characters that will be classified
+/// as good or neargood.
+const MAX_HEADING_DISTANCE_DEFAULT: usize = 200; // originally 200
#[derive(Debug, Clone)]
enum IntermediateClassification {
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index b9866086..aebe9758 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -568,8 +568,8 @@ impl Html {
}
}
- pub fn clean_text(&self) -> Option<String> {
- self.clean_text.clone()
+ pub fn clean_text(&self) -> Option<&String> {
+ self.clean_text.as_ref()
}
fn calculate_all_text(paragraphs: &[Paragraph], lang: &Lang) -> Option<String> {
@@ -659,7 +659,7 @@ impl Html {
}
fn pretokenize_clean_text(&self) -> PreTokenizedString {
- let clean_text = self.clean_text().unwrap_or_default();
+ let clean_text = self.clean_text().cloned().unwrap_or_default();
self.pretokenize_string(clean_text)
}
@@ -1217,7 +1217,7 @@ mod tests {
text: "Link to example".to_string()
}]
);
- assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));
+ assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
let mut expected_meta = HashMap::new();
expected_meta.insert("name".to_string(), "meta1".to_string());
@@ -1266,7 +1266,7 @@ mod tests {
let webpage = Html::parse(&raw, "https://www.example.com/whatever");
- assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));
+ assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
}
#[test]
@@ -1880,4 +1880,12 @@ mod tests {
]
);
}
+
+ #[test]
+ fn stackoverflow_question_has_clean_text() {
+ let stackoverflow = include_str!("../../testcases/schema_org/stackoverflow_with_code.html");
+ let html = Html::parse(stackoverflow, "https://www.example.com");
+
+ assert!(html.clean_text().is_some());
+ }
}