Skip to content

web search done right

The query 'linus torvalds' now produces the correct snippet. (#52)

Mikkel Denkerd81fe0cc5820
diff --git a/src/snippet.rs b/src/snippet.rs
index 8b77cd3b..59b2d455 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -199,28 +199,46 @@ impl SnippetGenerator {
fn create(
searcher: &Searcher,
query: &dyn tantivy::query::Query,
- field: tantivy::schema::Field,
tokenizer: tantivy::tokenizer::TextAnalyzer,
) -> crate::Result<SnippetGenerator> {
+ let clean_field = searcher
+ .schema()
+ .get_field(Field::CleanBody.as_str())
+ .unwrap();
+
+ let stemmed_field = searcher
+ .schema()
+ .get_field(Field::StemmedCleanBody.as_str())
+ .unwrap();
+
let mut terms: BTreeSet<&tantivy::Term> = BTreeSet::new();
query.query_terms(&mut |term, _| {
- if term.field() == field {
+ if term.field() == clean_field {
terms.insert(term);
}
});
+
let mut terms_text: BTreeMap<String, Score> = Default::default();
for term in terms {
- let term_str = if let Some(term_str) = term.as_str() {
- term_str
+ let term_string = if let Some(term_str) = term.as_str() {
+ tokenizer
+ .token_stream(term_str)
+ .next()
+ .unwrap()
+ .text
+ .to_string()
} else {
continue;
};
- let doc_freq = searcher.doc_freq(term)?;
+
+ let term = tantivy::Term::from_field_text(stemmed_field, &term_string);
+ let doc_freq = searcher.doc_freq(&term)?;
if doc_freq > 0 {
let score = 1.0 / (1.0 + doc_freq as Score);
- terms_text.insert(term_str.to_string(), score);
+ terms_text.insert(term_string, score);
}
}
+
Ok(SnippetGenerator {
terms_text,
tokenizer,
@@ -228,11 +246,6 @@ impl SnippetGenerator {
})
}
- #[cfg(test)]
- fn terms_text(&self) -> &BTreeMap<String, Score> {
- &self.terms_text
- }
-
/// Generates a snippet for the given text.
fn snippet(&self, text: &str) -> Snippet {
let fragment_candidates =
@@ -251,18 +264,13 @@ pub fn generate(
region: &Region,
searcher: &tantivy::Searcher,
) -> Result<String> {
- let field = searcher
- .schema()
- .get_field(Field::StemmedCleanBody.as_str())
- .expect("Failed to get body field");
-
let lang = match region.lang() {
Some(lang) => lang,
None => whatlang::detect_lang(text).unwrap_or(Lang::Eng),
};
let tokenizer = Stemmed::with_forced_language(lang).into();
- let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
+ let generator = SnippetGenerator::create(searcher, query, tokenizer)?;
let mut snippet = generator.snippet(text);
@@ -276,7 +284,7 @@ pub fn generate(
};
let tokenizer = Stemmed::with_forced_language(lang).into();
- let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
+ let generator = SnippetGenerator::create(searcher, query, tokenizer)?;
snippet = generator.snippet(desc);
if snippet.fragment.is_empty() {
@@ -290,7 +298,7 @@ pub fn generate(
};
let tokenizer = Stemmed::with_forced_language(lang).into();
- let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
+ let generator = SnippetGenerator::create(searcher, query, tokenizer)?;
snippet = generator.snippet(dirty_text);
@@ -564,97 +572,45 @@ Survey in 2016, 2017, and 2018."#;
}
#[test]
- fn test_snippet_generator_term_score() {
- let mut schema_builder = tantivy::schema::Schema::builder();
- let text_field = schema_builder.add_text_field("text", tantivy::schema::TEXT);
- let schema = schema_builder.build();
- let index = tantivy::Index::create_in_ram(schema);
- let tokenizer: TextAnalyzer = SimpleTokenizer.into();
- {
- // writing the segment
- let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
- index_writer
- .add_document(tantivy::doc!(text_field => "a"))
- .unwrap();
- index_writer
- .add_document(tantivy::doc!(text_field => "a"))
- .unwrap();
- index_writer
- .add_document(tantivy::doc!(text_field => "a b"))
- .unwrap();
- index_writer.commit().unwrap();
- }
- let searcher = index.reader().unwrap().searcher();
- let query_parser = tantivy::query::QueryParser::for_index(&index, vec![text_field]);
- {
- let query = query_parser.parse_query("e").unwrap();
- let snippet_generator =
- SnippetGenerator::create(&searcher, &*query, text_field, tokenizer.clone())
- .unwrap();
- assert!(snippet_generator.terms_text().is_empty());
- }
- {
- let query = query_parser.parse_query("a").unwrap();
- let snippet_generator =
- SnippetGenerator::create(&searcher, &*query, text_field, tokenizer.clone())
- .unwrap();
- assert_eq!(
- &btreemap!("a".to_string() => 0.25),
- snippet_generator.terms_text()
- );
- }
- {
- let query = query_parser.parse_query("a b").unwrap();
- let snippet_generator =
- SnippetGenerator::create(&searcher, &*query, text_field, tokenizer.clone())
- .unwrap();
- assert_eq!(
- &btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),
- snippet_generator.terms_text()
- );
- }
- {
- let query = query_parser.parse_query("a b c").unwrap();
- let snippet_generator =
- SnippetGenerator::create(&searcher, &*query, text_field, tokenizer).unwrap();
- assert_eq!(
- &btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),
- snippet_generator.terms_text()
- );
- }
- }
+ fn test_stemmed_term() {
+ let mut index = Index::temporary().expect("Unable to open index");
- #[test]
- fn test_snippet_generator() -> crate::Result<()> {
- let mut schema_builder = tantivy::schema::Schema::builder();
- let text_options = tantivy::schema::TextOptions::default().set_indexing_options(
- tantivy::schema::TextFieldIndexing::default()
- .set_tokenizer("en_stem")
- .set_index_option(tantivy::schema::IndexRecordOption::Basic),
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website for runners</title>
+ </head>
+ <body>
+ {}
+ </body>
+ </html>
+ "#,
+ TEST_TEXT
+ ),
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let searcher = Searcher::from(index);
+
+ let result = searcher
+ .search("paradigms", None, None, None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
+
+ assert_eq!(result.webpages.num_docs, 1);
+ assert_eq!(result.webpages.documents.len(), 1);
+ assert_eq!(
+ result.webpages.documents[0].snippet,
+ "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural <b>paradigms</b>. Rust is...".to_string()
);
- let text_field = schema_builder.add_text_field("text", text_options);
- let schema = schema_builder.build();
- let index = tantivy::Index::create_in_ram(schema);
- {
- // writing the segment
- let mut index_writer = index.writer_with_num_threads(1, 10_000_000)?;
- let doc = tantivy::doc!(text_field => TEST_TEXT);
- index_writer.add_document(doc)?;
- index_writer.commit()?;
- }
- let searcher = index.reader().unwrap().searcher();
- let tokenizer = searcher.index().tokenizer_for_field(text_field).unwrap();
- let query_parser = tantivy::query::QueryParser::for_index(&index, vec![text_field]);
- let query = query_parser.parse_query("rust design").unwrap();
- let snippet_generator =
- SnippetGenerator::create(&searcher, &*query, text_field, tokenizer).unwrap();
- {
- let snippet = snippet_generator.snippet(TEST_TEXT);
- assert_eq!(
- snippet.to_html(),
- "<b>Rust</b> is a systems programming language sponsored by\nMozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and\nimperative-procedural paradigms. <b>Rust</b> is"
- );
- }
- Ok(())
}
}