Skip to content

web search done right

Match webage terms across fields. If a webpage matches TermA in url and TermB in body it should still be a correct match even though no single field has both TermB and TermA.

Mikkel Denkere6c88d64e1c9
diff --git a/src/index.rs b/src/index.rs
index d02f503b..1ed894f1 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -669,4 +669,45 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
assert_eq!(result.documents[1].url, "https://www.example.com");
}
+
+ #[test]
+ fn match_across_fields() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("example test", index.schema()).expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.documents.len(), 0);
+ assert_eq!(result.num_docs, 0);
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ }
}
diff --git a/src/query/weight.rs b/src/query/weight.rs
index cd9f9008..ded3bff5 100644
--- a/src/query/weight.rs
+++ b/src/query/weight.rs
@@ -33,13 +33,14 @@ impl tantivy::query::Weight for Weight {
reader: &SegmentReader,
boost: Score,
) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
- let mut field_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
+ let mut term_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
- for field_data in &self.fields {
- let inverted_index = reader.inverted_index(field_data.tantivy)?;
- let mut term_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
+ for term_text in &self.terms {
+ let mut field_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
+
+ for field_data in &self.fields {
+ let inverted_index = reader.inverted_index(field_data.tantivy)?;
- for term_text in &self.terms {
let processed_terms: Vec<String> = field_data
.analyzer
.as_ref()
@@ -79,7 +80,7 @@ impl tantivy::query::Weight for Weight {
let posting = postings_opt.unwrap_or_else(SegmentPostings::empty);
- term_scorers.push(Box::new(Scorer::new(
+ field_scorers.push(Box::new(Scorer::new(
posting,
fieldnorm_reader,
similarity_weight,
@@ -87,10 +88,10 @@ impl tantivy::query::Weight for Weight {
}
}
- field_scorers.push(intersect_scorers(term_scorers));
+ term_scorers.push(Box::new(Union::from(field_scorers)));
}
- Ok(Box::new(Union::from(field_scorers)))
+ Ok(intersect_scorers(term_scorers))
}
fn explain(&self, _reader: &SegmentReader, _doc: DocId) -> tantivy::Result<Explanation> {