Skip to content

web search done right

if no snippet found, use webapge description if present

Mikkel Denker499488d45487
diff --git a/src/index.rs b/src/index.rs
index 493dfc01..3d660d01 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -134,7 +134,7 @@ impl Index {
.collect();
for page in &mut webpages {
- page.snippet = snippet::generate(query, &page.body, &searcher)?;
+ page.snippet = snippet::generate(query, &page.body, &page.description, &searcher)?;
}
Ok(SearchResult {
@@ -256,6 +256,7 @@ pub struct RetrievedWebpage {
pub url: String,
pub snippet: String,
pub body: String,
+ pub description: Option<String>,
pub favicon: Option<Image>,
pub primary_image_uuid: Option<String>,
pub updated_time: Option<NaiveDateTime>,
@@ -281,6 +282,15 @@ impl From<Document> for RetrievedWebpage {
.expect("Body field should be text")
.to_string()
}
+ Field::Description => {
+ let desc = value
+ .value
+ .as_text()
+ .expect("Description field should be text")
+ .to_string();
+
+ webpage.description = if desc.is_empty() { None } else { Some(desc) }
+ }
Field::Url => {
webpage.url = value
.value
diff --git a/src/schema.rs b/src/schema.rs
index a432acab..f4036516 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -38,8 +38,9 @@ pub enum Field {
FetchTimeMs,
PrimaryImageUuid,
LastUpdated,
+ Description,
}
-pub static ALL_FIELDS: [Field; 14] = [
+pub static ALL_FIELDS: [Field; 15] = [
Field::Title,
Field::Body,
Field::StemmedTitle,
@@ -54,6 +55,7 @@ pub static ALL_FIELDS: [Field; 14] = [
Field::FetchTimeMs,
Field::PrimaryImageUuid,
Field::LastUpdated,
+ Field::Description,
];
impl Field {
@@ -106,12 +108,13 @@ impl Field {
Field::PrimaryImageUuid => {
IndexingOption::Text(self.default_text_options().set_stored())
}
- &Field::LastUpdated => IndexingOption::Numeric(
+ Field::LastUpdated => IndexingOption::Numeric(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_stored()
.set_indexed(),
),
+ Field::Description => IndexingOption::Text(self.default_text_options().set_stored()),
}
}
@@ -131,6 +134,7 @@ impl Field {
Field::FetchTimeMs => "fetch_time_ms",
Field::PrimaryImageUuid => "primary_image_uuid",
Field::LastUpdated => "last_updated",
+ Field::Description => "description",
}
}
@@ -148,6 +152,7 @@ impl Field {
| Field::IsHomepage
| Field::PrimaryImageUuid
| Field::FetchTimeMs
+ | Field::Description
| Field::LastUpdated => None,
}
}
diff --git a/src/snippet.rs b/src/snippet.rs
index c35d7b53..ccc0bb0d 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -248,11 +248,17 @@ impl SnippetGenerator {
use crate::{query::Query, schema::Field, Result};
-pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Result<String> {
+pub fn generate(
+ query: &Query,
+ text: &str,
+ description: &Option<String>,
+ searcher: &tantivy::Searcher,
+) -> Result<String> {
let field = searcher
.schema()
.get_field(Field::StemmedBody.as_str())
.expect("Failed to get body field");
+
let tokenizer =
StemmedTokenizer::with_forced_language(whatlang::detect_lang(text).unwrap_or(Lang::Eng))
.into();
@@ -261,7 +267,20 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resu
let mut snippet = generator.snippet(text);
if snippet.fragment.is_empty() {
- snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ match description {
+ Some(desc) => {
+ let tokenizer = StemmedTokenizer::with_forced_language(
+ whatlang::detect_lang(desc).unwrap_or(Lang::Eng),
+ )
+ .into();
+ let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
+
+ snippet = generator.snippet(desc);
+ }
+ None => {
+ snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ }
+ }
}
let highlighted = snippet.to_html() + "...";
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index c1e47c3b..ba930920 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -444,6 +444,9 @@ impl<'a> Html<'a> {
doc.add_text(tantivy_field, text.unwrap())
}
+ Field::Description => {
+ doc.add_text(tantivy_field, self.description().unwrap_or_default())
+ }
Field::Url => doc.add_text(tantivy_field, self.url()),
Field::Host => doc.add_text(tantivy_field, self.host()),
Field::Domain => doc.add_text(tantivy_field, self.domain()),
@@ -459,9 +462,9 @@ impl<'a> Html<'a> {
}
Field::LastUpdated => doc.add_u64(
tantivy_field,
- dbg!(dbg!(self.updated_time())
+ self.updated_time()
.map(|time| time.timestamp().max(0) as u64)
- .unwrap_or(0)),
+ .unwrap_or(0),
),
Field::BacklinkText
| Field::Centrality