Skip to content

web search done right

Index (almost) all text on page. Only use clean text for snippets unless absolutely necesarry.

Mikkel Denker7f42601f4662
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index a89333fb..c807951d 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -83,7 +83,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
trace!("inserting webpage: {:?}", html.url());
trace!("title = {:?}", html.title());
- trace!("text = {:?}", html.text());
+ trace!("text = {:?}", html.clean_text());
let webpage = Webpage {
html,
diff --git a/src/index.rs b/src/index.rs
index 3d660d01..59f52a8a 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -134,7 +134,13 @@ impl Index {
.collect();
for page in &mut webpages {
- page.snippet = snippet::generate(query, &page.body, &page.description, &searcher)?;
+ page.snippet = snippet::generate(
+ query,
+ &page.body,
+ &page.dirty_body,
+ &page.description,
+ &searcher,
+ )?;
}
Ok(SearchResult {
@@ -256,6 +262,7 @@ pub struct RetrievedWebpage {
pub url: String,
pub snippet: String,
pub body: String,
+ pub dirty_body: String,
pub description: Option<String>,
pub favicon: Option<Image>,
pub primary_image_uuid: Option<String>,
@@ -275,7 +282,7 @@ impl From<Document> for RetrievedWebpage {
.expect("Title field should be text")
.to_string()
}
- Field::StemmedBody => {
+ Field::StemmedCleanBody => {
webpage.body = value
.value
.as_text()
@@ -323,14 +330,22 @@ impl From<Document> for RetrievedWebpage {
}
}
}
+ Field::StemmedAllBody => {
+ webpage.dirty_body = value
+ .value
+ .as_text()
+ .expect("Stemmed all body field should be text")
+ .to_string()
+ }
Field::BacklinkText
| Field::Centrality
| Field::Host
| Field::StemmedTitle
- | Field::Body
+ | Field::CleanBody
| Field::Domain
| Field::DomainIfHomepage
| Field::IsHomepage
+ | Field::AllBody
| Field::FetchTimeMs => {}
}
}
diff --git a/src/schema.rs b/src/schema.rs
index f4036516..646debec 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -25,9 +25,11 @@ pub const CENTRALITY_SCALING: u64 = 1_000_000_000;
#[derive(Clone)]
pub enum Field {
Title,
- Body,
+ CleanBody,
StemmedTitle,
- StemmedBody,
+ StemmedCleanBody,
+ AllBody,
+ StemmedAllBody,
Url,
Host,
Domain,
@@ -40,11 +42,13 @@ pub enum Field {
LastUpdated,
Description,
}
-pub static ALL_FIELDS: [Field; 15] = [
+pub static ALL_FIELDS: [Field; 17] = [
Field::Title,
- Field::Body,
+ Field::CleanBody,
Field::StemmedTitle,
- Field::StemmedBody,
+ Field::StemmedCleanBody,
+ Field::AllBody,
+ Field::StemmedAllBody,
Field::Url,
Field::Host,
Field::Domain,
@@ -77,10 +81,15 @@ impl Field {
pub fn options(&self) -> IndexingOption {
match self {
Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),
- Field::Body => IndexingOption::Text(self.default_text_options()),
+ Field::CleanBody => IndexingOption::Text(self.default_text_options()),
Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Host => IndexingOption::Text(self.default_text_options()),
Field::Domain => IndexingOption::Text(self.default_text_options()),
+ Field::AllBody => IndexingOption::Text(self.default_text_options()),
+ Field::StemmedAllBody => IndexingOption::Text(
+ self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())
+ .set_stored(),
+ ),
Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),
Field::IsHomepage => IndexingOption::Numeric(
NumericOptions::default()
@@ -96,7 +105,7 @@ impl Field {
Field::StemmedTitle => IndexingOption::Text(
self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),
),
- Field::StemmedBody => IndexingOption::Text(
+ Field::StemmedCleanBody => IndexingOption::Text(
self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())
.set_stored(),
),
@@ -121,13 +130,13 @@ impl Field {
pub fn as_str(&self) -> &str {
match self {
Field::Title => "title",
- Field::Body => "body",
+ Field::CleanBody => "body",
Field::Url => "url",
Field::Host => "host",
Field::BacklinkText => "backlink_text",
Field::Centrality => "centrality",
Field::StemmedTitle => "stemmed_title",
- Field::StemmedBody => "stemmed_body",
+ Field::StemmedCleanBody => "stemmed_body",
Field::Domain => "domain",
Field::DomainIfHomepage => "domain_if_homepage",
Field::IsHomepage => "is_homepage",
@@ -135,6 +144,8 @@ impl Field {
Field::PrimaryImageUuid => "primary_image_uuid",
Field::LastUpdated => "last_updated",
Field::Description => "description",
+ Field::AllBody => "all_body",
+ Field::StemmedAllBody => "stemmed_all_body",
}
}
@@ -142,9 +153,11 @@ impl Field {
match self {
Field::Host => Some(2.0),
Field::DomainIfHomepage => Some(50.0),
- Field::StemmedBody | Field::StemmedTitle => Some(0.01),
+ Field::StemmedCleanBody | Field::StemmedTitle => Some(0.01),
+ Field::AllBody => Some(0.01),
+ Field::StemmedAllBody => Some(0.001),
Field::Title
- | Field::Body
+ | Field::CleanBody
| Field::BacklinkText
| Field::Centrality
| Field::Url
diff --git a/src/snippet.rs b/src/snippet.rs
index ccc0bb0d..6715d565 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -251,12 +251,13 @@ use crate::{query::Query, schema::Field, Result};
pub fn generate(
query: &Query,
text: &str,
+ dirty_text: &str,
description: &Option<String>,
searcher: &tantivy::Searcher,
) -> Result<String> {
let field = searcher
.schema()
- .get_field(Field::StemmedBody.as_str())
+ .get_field(Field::StemmedCleanBody.as_str())
.expect("Failed to get body field");
let tokenizer =
@@ -276,9 +277,26 @@ pub fn generate(
let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
snippet = generator.snippet(desc);
+ if snippet.fragment.is_empty() {
+ snippet.fragment = desc.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ }
}
None => {
- snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ if text.is_empty() {
+ let tokenizer = StemmedTokenizer::with_forced_language(
+ whatlang::detect_lang(dirty_text).unwrap_or(Lang::Eng),
+ )
+ .into();
+ let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
+
+ snippet = generator.snippet(dirty_text);
+
+ if snippet.fragment.is_empty() {
+ snippet.fragment = dirty_text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ }
+ } else {
+ snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();
+ }
}
}
}
diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rs
index 9ca7997c..b12f8545 100644
--- a/src/webpage/just_text.rs
+++ b/src/webpage/just_text.rs
@@ -26,9 +26,9 @@ use super::{lexer::Token, Preprocessor};
// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py
const MAX_LINK_DENSITY_DEFAULT: f64 = 0.2;
-const LENGTH_LOW_DEFAULT: usize = 50;
+const LENGTH_LOW_DEFAULT: usize = 70;
const LENGTH_HIGH_DEFAULT: usize = 200;
-const STOPWORDS_LOW_DEFAULT: f64 = 0.28;
+const STOPWORDS_LOW_DEFAULT: f64 = 0.30;
const STOPWORDS_HIGH_DEFAULT: f64 = 0.32;
const MAX_HEADING_DISTANCE_DEFAULT: usize = 200;
@@ -124,12 +124,12 @@ lazy_static! {
}
pub struct JustText {
- max_link_density: f64,
- length_low: usize,
- length_high: usize,
- stopwords_low: f64,
- stopwords_high: f64,
- max_heading_distance: usize,
+ pub max_link_density: f64,
+ pub length_low: usize,
+ pub length_high: usize,
+ pub stopwords_low: f64,
+ pub stopwords_high: f64,
+ pub max_heading_distance: usize,
}
impl Default for JustText {
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index ba930920..b5663cc8 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -307,7 +307,7 @@ impl<'a> Html<'a> {
None
}
- pub fn text(&self) -> Option<String> {
+ pub fn clean_text(&self) -> Option<String> {
let text = JustText::default().extract(self.raw);
if text.is_empty() {
@@ -317,6 +317,24 @@ impl<'a> Html<'a> {
}
}
+ pub fn all_text(&self) -> Option<String> {
+ let text = JustText {
+ max_link_density: 2.0,
+ length_low: 0,
+ length_high: 0,
+ stopwords_low: -1.0,
+ stopwords_high: -1.0,
+ max_heading_distance: 10000,
+ }
+ .extract(self.raw);
+
+ if text.is_empty() {
+ None
+ } else {
+ Some(text)
+ }
+ }
+
pub fn title(&self) -> Option<String> {
let mut tokens = self.tokens.clone();
let mut title = None;
@@ -435,14 +453,10 @@ impl<'a> Html<'a> {
doc.add_text(tantivy_field, title.unwrap())
}
- Field::Body | Field::StemmedBody => {
- let text = self.text();
+ Field::CleanBody | Field::StemmedCleanBody => {
+ let text = self.clean_text();
- if text.is_none() {
- return Err(Error::EmptyField("body"));
- }
-
- doc.add_text(tantivy_field, text.unwrap())
+ doc.add_text(tantivy_field, text.unwrap_or_default())
}
Field::Description => {
doc.add_text(tantivy_field, self.description().unwrap_or_default())
@@ -466,6 +480,15 @@ impl<'a> Html<'a> {
.map(|time| time.timestamp().max(0) as u64)
.unwrap_or(0),
),
+ Field::StemmedAllBody | Field::AllBody => {
+ let text = self.all_text();
+
+ if text.is_none() {
+ return Err(Error::EmptyField("all body"));
+ }
+
+ doc.add_text(tantivy_field, text.unwrap())
+ }
Field::BacklinkText
| Field::Centrality
| Field::FetchTimeMs
@@ -661,7 +684,7 @@ mod tests {
text: "Link to example".to_string()
}]
);
- assert_eq!(webpage.text(), Some(CONTENT.to_string()));
+ assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));
let mut expected_meta = HashMap::new();
expected_meta.insert("name".to_string(), "meta1".to_string());
@@ -690,7 +713,7 @@ mod tests {
let webpage = Html::parse(&raw, "https://www.example.com/whatever");
- assert_eq!(webpage.text(), Some(CONTENT.to_string()));
+ assert_eq!(webpage.clean_text(), Some(CONTENT.to_string()));
}
#[test]
@@ -718,7 +741,7 @@ mod tests {
let webpage = Html::parse(&raw, "https://www.example.com");
- assert!(!webpage.text().unwrap().contains("not"));
+ assert!(!webpage.clean_text().unwrap().contains("not"));
}
#[test]
@@ -746,7 +769,7 @@ mod tests {
let webpage = Html::parse(&raw, "https://www.example.com");
- assert!(!webpage.text().unwrap().contains("not"));
+ assert!(!webpage.clean_text().unwrap().contains("not"));
}
#[test]
@@ -789,16 +812,16 @@ mod tests {
webpage.title(),
Some("パチンコ大当たり情報 - Pジューシーハニー3 大当たり詳細ページ - やすだひばりヶ丘店".to_string())
);
- assert!(webpage.text().is_some());
- assert!(!webpage.text().unwrap().is_empty());
+ assert!(webpage.clean_text().is_some());
+ assert!(!webpage.clean_text().unwrap().is_empty());
let webpage = Html::parse(include_str!("../../testcases_parsing/5390001.html"), "");
assert_eq!(
webpage.title(),
Some("特效烟机系列_山东壹线文化传播有限公司".to_string())
);
- assert!(webpage.text().is_some());
- assert!(!webpage.text().unwrap().is_empty());
+ assert!(webpage.clean_text().is_some());
+ assert!(!webpage.clean_text().unwrap().is_empty());
let webpage = Html::parse(
include_str!("../../testcases_parsing/77p2p-7.live-105.html"),
@@ -808,8 +831,8 @@ mod tests {
webpage.title(),
Some("77p2pЅu¤WЖ[¬Э - ҐDјЅ :: іnєс ".to_string())
);
- assert!(webpage.text().is_some());
- assert!(!webpage.text().unwrap().is_empty());
+ assert!(webpage.clean_text().is_some());
+ assert!(!webpage.clean_text().unwrap().is_empty());
}
#[test]