Skip to content

web search done right

use backlink text during ranking

Mikkel Denker847cf07e842e
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 86b50d96..e0b3376c 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -484,7 +484,7 @@ mod tests {
}
#[test]
- fn searchable_backlinks() {
+ fn not_searchable_backlinks() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
let query = Query::parse("great site", index.schema(), index.tokenizers())
.expect("Failed to parse query");
@@ -544,9 +544,8 @@ mod tests {
.documents
.sort_by(|a, b| a.url.partial_cmp(&b.url).unwrap());
- assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.a.com");
- assert_eq!(result.documents[1].url, "https://www.b.com");
}
#[test]
diff --git a/src/query/mod.rs b/src/query/mod.rs
index f91c22da..ade301a4 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -63,8 +63,7 @@ impl Query {
let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
- .map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
- // .map(|term_query| (Occur::Must, term_query))
+ .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
.collect();
queries.push((Occur::Should, Box::new(AllQuery)));
diff --git a/src/query/parser.rs b/src/query/parser.rs
index ae49bea8..30a71063 100644
--- a/src/query/parser.rs
+++ b/src/query/parser.rs
@@ -37,30 +37,50 @@ impl Term {
&self,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
match self {
- Term::Simple(term) => (
- Occur::Must,
- Box::new(BooleanQuery::new(Term::into_tantivy_simple(
- term,
- fields,
- tokenizer_manager,
- ))),
- ),
- Term::Not(subterm) => (
+ Term::Simple(term) => {
+ let (backlink_field, backlink_field_entry) = fields
+ .iter()
+ .find(|(field, _)| {
+ matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText)
+ })
+ .unwrap();
+
+ vec![
+ (
+ Occur::Must,
+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(
+ term,
+ fields,
+ tokenizer_manager,
+ ))),
+ ),
+ (
+ Occur::Should,
+ Box::new(Term::tantivy_term_query(
+ backlink_field,
+ backlink_field_entry,
+ tokenizer_manager,
+ term,
+ )),
+ ),
+ ]
+ }
+ Term::Not(subterm) => vec![(
Occur::MustNot,
- Box::new(BooleanQuery::new(vec![
- subterm.as_tantivy_query(fields, tokenizer_manager)
- ])),
- ),
- Term::Site(site) => (
+ Box::new(BooleanQuery::new(
+ subterm.as_tantivy_query(fields, tokenizer_manager),
+ )),
+ )],
+ Term::Site(site) => vec![(
Occur::Must,
Box::new(BooleanQuery::new(Term::into_tantivy_site(
site,
fields,
tokenizer_manager,
))),
- ),
+ )],
Term::Title(title) => {
let (field, entry) = fields
.iter()
@@ -68,10 +88,10 @@ impl Term {
matches!(ALL_FIELDS[field.field_id() as usize], Field::Title)
})
.unwrap();
- (
+ vec![(
Occur::Must,
Term::tantivy_term_query(field, entry, tokenizer_manager, title),
- )
+ )]
}
Term::Body(body) => {
let (field, entry) = fields
@@ -80,20 +100,20 @@ impl Term {
matches!(ALL_FIELDS[field.field_id() as usize], Field::AllBody)
})
.unwrap();
- (
+ vec![(
Occur::Must,
Term::tantivy_term_query(field, entry, tokenizer_manager, body),
- )
+ )]
}
Term::Url(url) => {
let (field, entry) = fields
.iter()
.find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::Url))
.unwrap();
- (
+ vec![(
Occur::Must,
Term::tantivy_term_query(field, entry, tokenizer_manager, url),
- )
+ )]
}
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index ef2ba76f..e03f0f49 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -71,8 +71,10 @@ mod tests {
<head>
<title>Website A</title>
</head>
- {CONTENT}
- <a href="https://www.b.com">B site is great</a>
+ <body>
+ {CONTENT}
+ example example example
+ </body>
</html>
"#
),
@@ -97,11 +99,7 @@ mod tests {
"#
),
"https://www.b.com",
- vec![Link {
- source: "https://www.a.com".to_string().into(),
- destination: "https://www.b.com".to_string().into(),
- text: "B site is great".to_string(),
- }],
+ vec![],
5.0,
500,
))
@@ -109,7 +107,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
- let result = searcher.search("great site", None).expect("Search failed");
+ let result = searcher.search("example", None).expect("Search failed");
assert_eq!(result.webpages.documents.len(), 2);
assert_eq!(result.webpages.documents[0].url, "https://www.b.com");
assert_eq!(result.webpages.documents[1].url, "https://www.a.com");
@@ -302,4 +300,57 @@ mod tests {
assert_eq!(result.webpages.documents.len(), 2);
assert_eq!(result.webpages.documents[0].url, "https://www.first.com");
}
+
+ #[test]
+ fn backlink_text() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test site</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "https://www.first.com",
+ vec![Link {
+ source: "https://www.second.com".to_string().into(),
+ destination: "https://www.first.com".to_string().into(),
+ text: "test this is the best test site".to_string(),
+ }],
+ 0.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Second test site</title>
+ </head>
+ <body>
+ test test test test test test test
+ </body>
+ </html>
+ "#,
+ "https://www.second.com",
+ vec![],
+ 0.003,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::from(index);
+ let result = searcher.search("test", None).expect("Search failed");
+
+ assert_eq!(result.webpages.documents.len(), 2);
+ assert_eq!(result.webpages.documents[0].url, "https://www.first.com");
+ }
}
diff --git a/src/schema.rs b/src/schema.rs
index 5d761e70..f76d8acd 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -176,8 +176,8 @@ impl Field {
Field::Domain => Some(4.0),
Field::AllBody => Some(0.01),
Field::StemmedAllBody => Some(0.001),
- Field::BacklinkText
- | Field::Centrality
+ Field::BacklinkText => Some(4.0),
+ Field::Centrality
| Field::IsHomepage
| Field::PrimaryImageUuid
| Field::FetchTimeMs
@@ -189,7 +189,7 @@ impl Field {
}
pub fn is_searchable(&self) -> bool {
- !matches!(self, Field::PrimaryImageUuid)
+ !matches!(self, Field::PrimaryImageUuid | Field::BacklinkText)
}
}