web search done right
if there are no simple terms in the query, we should use prefix of description (or body if there is no description) as snippet
Mikkel Denkere008e7003366
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rsindex 09fca40c..e09c2ce5 100644--- a/crates/core/src/inverted_index.rs+++ b/crates/core/src/inverted_index.rs@@ -26,6 +26,7 @@//! This allows us to perform more advanced queries than just term lookups,//! but the principle is the same.use chrono::NaiveDateTime;+use itertools::Itertools;use serde::{Deserialize, Serialize};use tantivy::collector::Count;use tantivy::directory::MmapDirectory;@@ -45,8 +46,8 @@ use crate::ranking::pipeline::RankingWebsite;use crate::ranking::SignalAggregator;use crate::schema::{FastField, Field, TextField};use crate::search_ctx::Ctx;-use crate::snippet;use crate::snippet::TextSnippet;+use crate::snippet::{self, TextSnippetFragment};use crate::tokenizer::{BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer,};@@ -439,20 +440,36 @@ impl InvertedIndex {let url = Url::parse(&page.url).ok()?;Some((url, page))}) {- let min_body_len = if url.is_homepage() { 1024 } else { 256 };-- if page.body.len() < min_body_len- && !page.description.as_deref().unwrap_or_default().is_empty()- {- page.snippet = snippet::generate(- query,- page.description.as_deref().unwrap_or_default(),- &page.region,- self.snippet_config.clone(),- );+ if query.simple_terms().is_empty() {+ let snippet = if let Some(description) = page.description.as_deref() {+ description.split_whitespace().take(50).join(" ")+ } else {+ page.body.split_whitespace().take(50).join(" ")+ };++ page.snippet = TextSnippet {+ fragments: vec![TextSnippetFragment::new_unhighlighted(snippet)],+ };} else {- page.snippet =- snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());+ let min_body_len = if url.is_homepage() { 1024 } else { 256 };++ if page.body.len() < min_body_len+ && !page.description.as_deref().unwrap_or_default().is_empty()+ {+ page.snippet = snippet::generate(+ query,+ page.description.as_deref().unwrap_or_default(),+ &page.region,+ self.snippet_config.clone(),+ );+ } else {+ page.snippet = snippet::generate(+ query,+ &page.body,+ &page.region,+ self.snippet_config.clone(),+ );+ }}}diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 471b3bb4..589605bf 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -134,6 +134,7 @@ impl Query {.into_iter().filter_map(|term| term.as_simple_text().map(|s| s.to_string())).flat_map(|term| {+ // term might be a phrase, so we split it into wordsterm.split_ascii_whitespace().map(|s| s.to_string()).collect::<Vec<_>>()diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex 3f0c9f57..bd7065b2 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -61,6 +61,13 @@ pub struct TextSnippetFragment {}impl TextSnippetFragment {+ pub fn new_unhighlighted(text: String) -> Self {+ TextSnippetFragment {+ kind: TextSnippetFragmentKind::Normal,+ text,+ }+ }+pub fn text(&self) -> &str {&self.text}