Skip to content

web search done right

if there are no simple terms in the query, we should use prefix of description (or body if there is no description) as snippet

Mikkel Denkere008e7003366
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs
index 09fca40c..e09c2ce5 100644
--- a/crates/core/src/inverted_index.rs
+++ b/crates/core/src/inverted_index.rs
@@ -26,6 +26,7 @@
//! This allows us to perform more advanced queries than just term lookups,
//! but the principle is the same.
use chrono::NaiveDateTime;
+use itertools::Itertools;
use serde::{Deserialize, Serialize};
use tantivy::collector::Count;
use tantivy::directory::MmapDirectory;
@@ -45,8 +46,8 @@ use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::SignalAggregator;
use crate::schema::{FastField, Field, TextField};
use crate::search_ctx::Ctx;
-use crate::snippet;
use crate::snippet::TextSnippet;
+use crate::snippet::{self, TextSnippetFragment};
use crate::tokenizer::{
BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer,
};
@@ -439,20 +440,36 @@ impl InvertedIndex {
let url = Url::parse(&page.url).ok()?;
Some((url, page))
}) {
- let min_body_len = if url.is_homepage() { 1024 } else { 256 };
-
- if page.body.len() < min_body_len
- && !page.description.as_deref().unwrap_or_default().is_empty()
- {
- page.snippet = snippet::generate(
- query,
- page.description.as_deref().unwrap_or_default(),
- &page.region,
- self.snippet_config.clone(),
- );
+ if query.simple_terms().is_empty() {
+ let snippet = if let Some(description) = page.description.as_deref() {
+ description.split_whitespace().take(50).join(" ")
+ } else {
+ page.body.split_whitespace().take(50).join(" ")
+ };
+
+ page.snippet = TextSnippet {
+ fragments: vec![TextSnippetFragment::new_unhighlighted(snippet)],
+ };
} else {
- page.snippet =
- snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());
+ let min_body_len = if url.is_homepage() { 1024 } else { 256 };
+
+ if page.body.len() < min_body_len
+ && !page.description.as_deref().unwrap_or_default().is_empty()
+ {
+ page.snippet = snippet::generate(
+ query,
+ page.description.as_deref().unwrap_or_default(),
+ &page.region,
+ self.snippet_config.clone(),
+ );
+ } else {
+ page.snippet = snippet::generate(
+ query,
+ &page.body,
+ &page.region,
+ self.snippet_config.clone(),
+ );
+ }
}
}
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index 471b3bb4..589605bf 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -134,6 +134,7 @@ impl Query {
.into_iter()
.filter_map(|term| term.as_simple_text().map(|s| s.to_string()))
.flat_map(|term| {
+ // term might be a phrase, so we split it into words
term.split_ascii_whitespace()
.map(|s| s.to_string())
.collect::<Vec<_>>()
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index 3f0c9f57..bd7065b2 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -61,6 +61,13 @@ pub struct TextSnippetFragment {
}
impl TextSnippetFragment {
+ pub fn new_unhighlighted(text: String) -> Self {
+ TextSnippetFragment {
+ kind: TextSnippetFragmentKind::Normal,
+ text,
+ }
+ }
+
pub fn text(&self) -> &str {
&self.text
}