web search done right
phrase queries shouldn't be augmented with adjacent ngrams as this results in unexpected matches on the phrase query
Mikkel Denker0b56d7fa929d
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex ebd592ba..3a857146 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -1175,6 +1175,169 @@ mod tests {assert_eq!(result.webpages.len(), 1);}+ #[test]+ fn exact_url_operator() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ This is a test website {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "https://www.first.com/example",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This is a test website {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "https://www.second.com",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This is a test website {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "https://www.third.io",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");+ let searcher = LocalSearcher::from(index);++ let query = SearchQuery {+ query: "test exacturl:https://www.first.com/example".to_string(),+ ..Default::default()+ };+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.webpages.len(), 1);++ let query = SearchQuery {+ query: "test exacturl:https://www.first.com".to_string(),+ ..Default::default()+ };+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.webpages.len(), 0);+ }++ #[test]+ fn mix_phrase_term_query() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ This is a test website {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "https://www.first.com/example",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This is a test website {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "https://www.second.com",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This is a test website {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "https://www.third.io",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");+ let searcher = LocalSearcher::from(index);++ let query = SearchQuery {+ query: "\"test test\" website".to_string(),+ ..Default::default()+ };+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.webpages.len(), 2);+ }+fn fixture(query: &str) -> Result<(), TestCaseError> {if query.trim().is_empty() {return Ok(());diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rsindex 01810f32..4c9b79b2 100644--- a/crates/core/src/query/parser/mod.rs+++ b/crates/core/src/query/parser/mod.rs@@ -123,7 +123,13 @@ fn exact_url(orig_input: &str) -> nom::IResult<&str, Term> {let (input, _) = nom::bytes::complete::tag("exacturl:")(orig_input)?;let (input, output) = simple_str(input)?;- match url::Url::parse(output) {+ let url = if output.starts_with("http://") || output.starts_with("https://") {+ url::Url::parse(output)+ } else {+ url::Url::parse(&format!("https://{}", output))+ };++ match url {Ok(url) => Ok((input, Term::ExactUrl(url.normalize().to_string()))),Err(_) => Err(nom::Err::Error(nom::error::Error::new(orig_input,diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rsindex c84b52d7..0eca5598 100644--- a/crates/core/src/query/plan/mod.rs+++ b/crates/core/src/query/plan/mod.rs@@ -242,7 +242,7 @@ pub fn initial(terms: Vec<super::Term>) -> Option<Node> {let mut adjacent = Vec::new();if augment_with_adjacent {- if let super::Term::SimpleOrPhrase(_) = &term {+ if let super::Term::SimpleOrPhrase(SimpleOrPhrase::Simple(_)) = &term {for window_size in 2..=3 {for (start, end) in sliding_window(window_size, i) {let mut compounds = Vec::new();diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rsindex 74b5a2f5..5f155e41 100644--- a/crates/core/src/query/plan/node.rs+++ b/crates/core/src/query/plan/node.rs@@ -116,8 +116,7 @@ impl Node {.reduce(|left, right| left.or(right)).expect("fields should not be empty"),SimpleOrPhrase::Phrase(p) => TextFieldEnum::all()- .filter(|f| f.is_searchable())- .filter(|f| f.is_phrase_searchable())+ .filter(|f| f.is_searchable() && f.is_phrase_searchable()).map(|field| {Node::Term(Term {text: SimpleOrPhrase::Phrase(p.clone()),diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 7fbd1723..91c7aeeb 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -198,7 +198,7 @@ whereT: Graph,{async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {- self.batch_raw_ingoing(nodes, EdgeLimit::Limit(4096))+ self.batch_raw_ingoing(nodes, EdgeLimit::Limit(1024)).await.into_iter().map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect())diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex f436aa48..4e24dc7b 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -75,7 +75,7 @@ impl SimilarHostsFinder {let in_edges = self.webgraph- .batch_raw_ingoing_edges(&nodes, EdgeLimit::Limit(128))+ .batch_raw_ingoing_edges(&nodes, EdgeLimit::Limit(64)).await.unwrap_or_default();@@ -88,7 +88,7 @@ impl SimilarHostsFinder {let outgoing_edges = self.webgraph- .batch_raw_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(4096))+ .batch_raw_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(512)).await.unwrap_or_default();diff --git a/crates/core/src/webpage/adservers.txt b/crates/core/src/webpage/adservers.txtindex 64741788..dec966b2 100644--- a/crates/core/src/webpage/adservers.txt+++ b/crates/core/src/webpage/adservers.txt@@ -1541,7 +1541,6 @@ delicatecascade.com,deloo.de,deloplen.com,deloton.com,-demandbase.com,demdex.net,deployads.com,desertedbreath.com,diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex 4cb47a69..7a0a49b1 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -80,11 +80,18 @@ impl Html {robots: None,};- if let Some(canonical) = res.canonical_url() {- if canonical.root_domain() == res.url.root_domain() {- res.url = canonical;- }- }+ // TODO: need to figure out when to respect+ // the canonical url, it's not always the case that+ // the website use the tag correctly.+ // for instance, https://www.elastic.co/CN/blog/author/brian-bergholm+ // has a canonical url to https://www.elastic.co/ but it's not the+ // same content.+ //+ // if let Some(canonical) = res.canonical_url() {+ // if canonical.root_domain() == res.url.root_domain() {+ // res.url = canonical;+ // }+ // }res.url.normalize_in_place();res.robots = res.parse_robots_meta();@@ -983,6 +990,7 @@ mod tests {}#[test]+ #[ignore]fn canonical_url() {let html = Html::parse(r#"diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex a67d3659..1cf6e8ee 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -14,6 +14,7 @@ param_grid = {"metric": ["ndcg"],"ndcg_at": [[1, 2, 3, 5, 10]],"learning_rate": [0.003],+ # "num_iterations": [100],# "max_depth": [-1, 2, 4, 8],"max_depth": [-1],# "num_leaves": [7, 15, 31],@@ -23,6 +24,11 @@ param_grid = {}+accepted_queries = set()+with open("data/queries_us.csv") as f:+ for query in f.readlines():+ accepted_queries.add(query.strip().lower())+con = sqlite3.connect("data/auto-ranking-annotation.sqlite")cur = con.cursor()@@ -36,6 +42,11 @@ res = cur.execute(""")queries = {qid: {"query": query} for qid, query in res.fetchall()}+qids = list(queries.keys())++for qid in qids:+ if not queries[qid]["query"].lower() in accepted_queries:+ del queries[qid]for qid in queries:res = cur.execute(diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.pyindex cf975ffe..2f1db7a6 100644--- a/ltr/leechy_annotate.py+++ b/ltr/leechy_annotate.py@@ -34,6 +34,9 @@ unannotated_queries = db.get_unannotated_queries()eng = leechy.Engine()for qid, query in tqdm(unannotated_queries.items()):+ if query not in all_queries:+ continue+tqdm.write(query)leechy_results = []