web search done right
Index (almost) all text on page. Only use clean text for snippets unless absolutely necesarry.
Mikkel Denker7f42601f4662
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex a89333fb..c807951d 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -83,7 +83,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {trace!("inserting webpage: {:?}", html.url());trace!("title = {:?}", html.title());- trace!("text = {:?}", html.text());+ trace!("text = {:?}", html.clean_text());let webpage = Webpage {html,diff --git a/src/index.rs b/src/index.rsindex 3d660d01..59f52a8a 100644--- a/src/index.rs+++ b/src/index.rs@@ -134,7 +134,13 @@ impl Index {.collect();for page in &mut webpages {- page.snippet = snippet::generate(query, &page.body, &page.description, &searcher)?;+ page.snippet = snippet::generate(+ query,+ &page.body,+ &page.dirty_body,+ &page.description,+ &searcher,+ )?;}Ok(SearchResult {@@ -256,6 +262,7 @@ pub struct RetrievedWebpage {pub url: String,pub snippet: String,pub body: String,+ pub dirty_body: String,pub description: Option<String>,pub favicon: Option<Image>,pub primary_image_uuid: Option<String>,@@ -275,7 +282,7 @@ impl From<Document> for RetrievedWebpage {.expect("Title field should be text").to_string()}- Field::StemmedBody => {+ Field::StemmedCleanBody => {webpage.body = value.value.as_text()@@ -323,14 +330,22 @@ impl From<Document> for RetrievedWebpage {}}}+ Field::StemmedAllBody => {+ webpage.dirty_body = value+ .value+ .as_text()+ .expect("Stemmed all body field should be text")+ .to_string()+ }Field::BacklinkText| Field::Centrality| Field::Host| Field::StemmedTitle- | Field::Body+ | Field::CleanBody| Field::Domain| Field::DomainIfHomepage| Field::IsHomepage+ | Field::AllBody| Field::FetchTimeMs => {}}}diff --git a/src/schema.rs b/src/schema.rsindex f4036516..646debec 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -25,9 +25,11 @@ pub const CENTRALITY_SCALING: u64 = 1_000_000_000;#[derive(Clone)]pub enum Field {Title,- Body,+ CleanBody,StemmedTitle,- StemmedBody,+ StemmedCleanBody,+ AllBody,+ StemmedAllBody,Url,Host,Domain,@@ -40,11 +42,13 @@ pub enum Field {LastUpdated,Description,}-pub static ALL_FIELDS: [Field; 15] = [+pub static ALL_FIELDS: [Field; 17] = [Field::Title,- Field::Body,+ Field::CleanBody,Field::StemmedTitle,- Field::StemmedBody,+ Field::StemmedCleanBody,+ Field::AllBody,+ Field::StemmedAllBody,Field::Url,Field::Host,Field::Domain,@@ -77,10 +81,15 @@ impl Field {pub fn options(&self) -> IndexingOption {match self {Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),- Field::Body => IndexingOption::Text(self.default_text_options()),+ Field::CleanBody => IndexingOption::Text(self.default_text_options()),Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),Field::Host => IndexingOption::Text(self.default_text_options()),Field::Domain => IndexingOption::Text(self.default_text_options()),+ Field::AllBody => IndexingOption::Text(self.default_text_options()),+ Field::StemmedAllBody => IndexingOption::Text(+ self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())+ .set_stored(),+ ),Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),Field::IsHomepage => IndexingOption::Numeric(NumericOptions::default()@@ -96,7 +105,7 @@ impl Field {Field::StemmedTitle => IndexingOption::Text(self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),),- Field::StemmedBody => IndexingOption::Text(+ Field::StemmedCleanBody => IndexingOption::Text(self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()).set_stored(),),@@ -121,13 +130,13 @@ impl Field {pub fn as_str(&self) -> &str {match self {Field::Title => "title",- Field::Body => "body",+ Field::CleanBody => "body",Field::Url => "url",Field::Host => "host",Field::BacklinkText => "backlink_text",Field::Centrality => "centrality",Field::StemmedTitle => "stemmed_title",- Field::StemmedBody => "stemmed_body",+ Field::StemmedCleanBody => "stemmed_body",Field::Domain => "domain",Field::DomainIfHomepage => "domain_if_homepage",Field::IsHomepage => "is_homepage",@@ -135,6 +144,8 @@ impl Field {Field::PrimaryImageUuid => "primary_image_uuid",Field::LastUpdated => "last_updated",Field::Description => "description",+ Field::AllBody => "all_body",+ Field::StemmedAllBody => "stemmed_all_body",}}@@ -142,9 +153,11 @@ impl Field {match self {Field::Host => Some(2.0),Field::DomainIfHomepage => Some(50.0),- Field::StemmedBody | Field::StemmedTitle => Some(0.01),+ Field::StemmedCleanBody | Field::StemmedTitle => Some(0.01),+ Field::AllBody => Some(0.01),+ Field::StemmedAllBody => Some(0.001),Field::Title- | Field::Body+ | Field::CleanBody| Field::BacklinkText| Field::Centrality| Field::Urldiff --git a/src/snippet.rs b/src/snippet.rsindex ccc0bb0d..6715d565 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -251,12 +251,13 @@ use crate::{query::Query, schema::Field, Result};pub fn generate(query: &Query,text: &str,+ dirty_text: &str,description: &Option<String>,searcher: &tantivy::Searcher,) -> Result<String> {let field = searcher.schema()- .get_field(Field::StemmedBody.as_str())+ .get_field(Field::StemmedCleanBody.as_str()).expect("Failed to get body field");let tokenizer =@@ -276,9 +277,26 @@ pub fn generate(let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;snippet = generator.snippet(desc);+ if snippet.fragment.is_empty() {+ snippet.fragment = desc.chars().take(DEFAULT_MAX_NUM_CHARS).collect();+ }}None => {- snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();+ if text.is_empty() {+ let tokenizer = StemmedTokenizer::with_forced_language(+ whatlang::detect_lang(dirty_text).unwrap_or(Lang::Eng),+ )+ .into();+ let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;++ snippet = generator.snippet(dirty_text);++ if snippet.fragment.is_empty() {+ snippet.fragment = dirty_text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();+ }+ } else {+ snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();+ }}}}diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rsindex 9ca7997c..b12f8545 100644--- a/src/webpage/just_text.rs+++ b/src/webpage/just_text.rs@@ -26,9 +26,9 @@ use super::{lexer::Token, Preprocessor};// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.pyconst MAX_LINK_DENSITY_DEFAULT: f64 = 0.2;-const LENGTH_LOW_DEFAULT: usize = 50;+const LENGTH_LOW_DEFAULT: usize = 70;const LENGTH_HIGH_DEFAULT: usize = 200;-const STOPWORDS_LOW_DEFAULT: f64 = 0.28;+const STOPWORDS_LOW_DEFAULT: f64 = 0.30;const STOPWORDS_HIGH_DEFAULT: f64 = 0.32;const MAX_HEADING_DISTANCE_DEFAULT: usize = 200;@@ -124,12 +124,12 @@ lazy_static! {}pub struct JustText {- max_link_density: f64,- length_low: usize,- length_high: usize,- stopwords_low: f64,- stopwords_high: f64,- max_heading_distance: usize,+ pub max_link_density: f64,+ pub length_low: usize,+ pub length_high: usize,+ pub stopwords_low: f64,+ pub stopwords_high: f64,+ pub max_heading_distance: usize,}impl Default for JustText {diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex ba930920..b5663cc8 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -307,7 +307,7 @@ impl<'a> Html<'a> {None}- pub fn text(&self) -> Option<String> {+ pub fn clean_text(&self) -> Option<String> {let text = JustText::default().extract(self.raw);if text.is_empty() {@@ -317,6 +317,24 @@ impl<'a> Html<'a> {}}+ pub fn all_text(&self) -> Option<String> {+ let text = JustText {+ max_link_density: 2.0,+ length_low: 0,+ length_high: 0,+ stopwords_low: -1.0,+ stopwords_high: -1.0,+ max_heading_distance: 10000,+ }+ .extract(self.raw);++ if text.is_empty() {+ None+ } else {+ Some(text)+ }+ }+pub fn title(&self) -> Option<String> {let mut tokens = self.tokens.clone();let mut title = None;@@ -435,14 +453,10 @@ impl<'a> Html<'a> {doc.add_text(tantivy_field, title.unwrap())}- Field::Body | Field::StemmedBody => {- let text = self.text();+ Field::CleanBody | Field::StemmedCleanBody => {+ let text = self.clean_text();- if text.is_none() {- return Err(Error::EmptyField("body"));- }-- doc.add_text(tantivy_field, text.unwrap())+ doc.add_text(tantivy_field, text.unwrap_or_default())}Field::Description => {doc.add_text(tantivy_field, self.description().unwrap_or_default())@@ -466,6 +480,15 @@ impl<'a> Html<'a> {.map(|time| time.timestamp().max(0) as u64).unwrap_or(0),),+ Field::StemmedAllBody | Field::AllBody => {+ let text = self.all_text();++ if text.is_none() {+ return Err(Error::EmptyField("all body"));+ }++ doc.add_text(tantivy_field, text.unwrap())+ }Field::BacklinkText| Field::Centrality| Field::FetchTimeMs@@ -661,7 +684,7 @@ mod tests {text: "Link to example".to_string()}]);- assert_eq!(webpage.text(), Some(CONTENT.to_string()));+ assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));let mut expected_meta = HashMap::new();expected_meta.insert("name".to_string(), "meta1".to_string());@@ -690,7 +713,7 @@ mod tests {let webpage = Html::parse(&raw, "https://www.example.com/whatever");- assert_eq!(webpage.text(), Some(CONTENT.to_string()));+ assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));}#[test]@@ -718,7 +741,7 @@ mod tests {let webpage = Html::parse(&raw, "https://www.example.com");- assert!(!webpage.text().unwrap().contains("not"));+ assert!(!webpage.clean_text().unwrap().contains("not"));}#[test]@@ -746,7 +769,7 @@ mod tests {let webpage = Html::parse(&raw, "https://www.example.com");- assert!(!webpage.text().unwrap().contains("not"));+ assert!(!webpage.clean_text().unwrap().contains("not"));}#[test]@@ -789,16 +812,16 @@ mod tests {webpage.title(),Some("パチンコ大当たり情報 - Pジューシーハニー3 大当たり詳細ページ - やすだひばりヶ丘店".to_string()));- assert!(webpage.text().is_some());- assert!(!webpage.text().unwrap().is_empty());+ assert!(webpage.clean_text().is_some());+ assert!(!webpage.clean_text().unwrap().is_empty());let webpage = Html::parse(include_str!("../../testcases_parsing/5390001.html"), "");assert_eq!(webpage.title(),Some("特效烟机系列_山东壹线文化传播有限公司".to_string()));- assert!(webpage.text().is_some());- assert!(!webpage.text().unwrap().is_empty());+ assert!(webpage.clean_text().is_some());+ assert!(!webpage.clean_text().unwrap().is_empty());let webpage = Html::parse(include_str!("../../testcases_parsing/77p2p-7.live-105.html"),@@ -808,8 +831,8 @@ mod tests {webpage.title(),Some("77p2pЅu¤WЖ[¬Э - ҐDјЅ :: іnєс ".to_string()));- assert!(webpage.text().is_some());- assert!(!webpage.text().unwrap().is_empty());+ assert!(webpage.clean_text().is_some());+ assert!(!webpage.clean_text().unwrap().is_empty());}#[test]