web search done right
if no snippet found, use webapge description if present
Mikkel Denker499488d45487
diff --git a/src/index.rs b/src/index.rsindex 493dfc01..3d660d01 100644--- a/src/index.rs+++ b/src/index.rs@@ -134,7 +134,7 @@ impl Index {.collect();for page in &mut webpages {- page.snippet = snippet::generate(query, &page.body, &searcher)?;+ page.snippet = snippet::generate(query, &page.body, &page.description, &searcher)?;}Ok(SearchResult {@@ -256,6 +256,7 @@ pub struct RetrievedWebpage {pub url: String,pub snippet: String,pub body: String,+ pub description: Option<String>,pub favicon: Option<Image>,pub primary_image_uuid: Option<String>,pub updated_time: Option<NaiveDateTime>,@@ -281,6 +282,15 @@ impl From<Document> for RetrievedWebpage {.expect("Body field should be text").to_string()}+ Field::Description => {+ let desc = value+ .value+ .as_text()+ .expect("Description field should be text")+ .to_string();++ webpage.description = if desc.is_empty() { None } else { Some(desc) }+ }Field::Url => {webpage.url = value.valuediff --git a/src/schema.rs b/src/schema.rsindex a432acab..f4036516 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -38,8 +38,9 @@ pub enum Field {FetchTimeMs,PrimaryImageUuid,LastUpdated,+ Description,}-pub static ALL_FIELDS: [Field; 14] = [+pub static ALL_FIELDS: [Field; 15] = [Field::Title,Field::Body,Field::StemmedTitle,@@ -54,6 +55,7 @@ pub static ALL_FIELDS: [Field; 14] = [Field::FetchTimeMs,Field::PrimaryImageUuid,Field::LastUpdated,+ Field::Description,];impl Field {@@ -106,12 +108,13 @@ impl Field {Field::PrimaryImageUuid => {IndexingOption::Text(self.default_text_options().set_stored())}- &Field::LastUpdated => IndexingOption::Numeric(+ Field::LastUpdated => IndexingOption::Numeric(NumericOptions::default().set_fast(Cardinality::SingleValue).set_stored().set_indexed(),),+ Field::Description => IndexingOption::Text(self.default_text_options().set_stored()),}}@@ -131,6 +134,7 @@ impl Field {Field::FetchTimeMs => "fetch_time_ms",Field::PrimaryImageUuid => "primary_image_uuid",Field::LastUpdated => "last_updated",+ Field::Description => "description",}}@@ -148,6 +152,7 @@ impl Field {| Field::IsHomepage| Field::PrimaryImageUuid| Field::FetchTimeMs+ | Field::Description| Field::LastUpdated => None,}}diff --git a/src/snippet.rs b/src/snippet.rsindex c35d7b53..ccc0bb0d 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -248,11 +248,17 @@ impl SnippetGenerator {use crate::{query::Query, schema::Field, Result};-pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Result<String> {+pub fn generate(+ query: &Query,+ text: &str,+ description: &Option<String>,+ searcher: &tantivy::Searcher,+) -> Result<String> {let field = searcher.schema().get_field(Field::StemmedBody.as_str()).expect("Failed to get body field");+let tokenizer =StemmedTokenizer::with_forced_language(whatlang::detect_lang(text).unwrap_or(Lang::Eng)).into();@@ -261,7 +267,20 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resulet mut snippet = generator.snippet(text);if snippet.fragment.is_empty() {- snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();+ match description {+ Some(desc) => {+ let tokenizer = StemmedTokenizer::with_forced_language(+ whatlang::detect_lang(desc).unwrap_or(Lang::Eng),+ )+ .into();+ let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;++ snippet = generator.snippet(desc);+ }+ None => {+ snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();+ }+ }}let highlighted = snippet.to_html() + "...";diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex c1e47c3b..ba930920 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -444,6 +444,9 @@ impl<'a> Html<'a> {doc.add_text(tantivy_field, text.unwrap())}+ Field::Description => {+ doc.add_text(tantivy_field, self.description().unwrap_or_default())+ }Field::Url => doc.add_text(tantivy_field, self.url()),Field::Host => doc.add_text(tantivy_field, self.host()),Field::Domain => doc.add_text(tantivy_field, self.domain()),@@ -459,9 +462,9 @@ impl<'a> Html<'a> {}Field::LastUpdated => doc.add_u64(tantivy_field,- dbg!(dbg!(self.updated_time())+ self.updated_time().map(|time| time.timestamp().max(0) as u64)- .unwrap_or(0)),+ .unwrap_or(0),),Field::BacklinkText| Field::Centrality