Skip to content

web search done right

phrase queries shouldn't be augmented with adjacent ngrams as this results in unexpected matches on the phrase query

Mikkel Denker0b56d7fa929d
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index ebd592ba..3a857146 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -1175,6 +1175,169 @@ mod tests {
assert_eq!(result.webpages.len(), 1);
}
+ #[test]
+ fn exact_url_operator() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.first.com/example",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.third.io",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ query: "test exacturl:https://www.first.com/example".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.webpages.len(), 1);
+
+ let query = SearchQuery {
+ query: "test exacturl:https://www.first.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.webpages.len(), 0);
+ }
+
+ #[test]
+ fn mix_phrase_term_query() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.first.com/example",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.third.io",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ query: "\"test test\" website".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.webpages.len(), 2);
+ }
+
fn fixture(query: &str) -> Result<(), TestCaseError> {
if query.trim().is_empty() {
return Ok(());
diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rs
index 01810f32..4c9b79b2 100644
--- a/crates/core/src/query/parser/mod.rs
+++ b/crates/core/src/query/parser/mod.rs
@@ -123,7 +123,13 @@ fn exact_url(orig_input: &str) -> nom::IResult<&str, Term> {
let (input, _) = nom::bytes::complete::tag("exacturl:")(orig_input)?;
let (input, output) = simple_str(input)?;
- match url::Url::parse(output) {
+ let url = if output.starts_with("http://") || output.starts_with("https://") {
+ url::Url::parse(output)
+ } else {
+ url::Url::parse(&format!("https://{}", output))
+ };
+
+ match url {
Ok(url) => Ok((input, Term::ExactUrl(url.normalize().to_string()))),
Err(_) => Err(nom::Err::Error(nom::error::Error::new(
orig_input,
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rs
index c84b52d7..0eca5598 100644
--- a/crates/core/src/query/plan/mod.rs
+++ b/crates/core/src/query/plan/mod.rs
@@ -242,7 +242,7 @@ pub fn initial(terms: Vec<super::Term>) -> Option<Node> {
let mut adjacent = Vec::new();
if augment_with_adjacent {
- if let super::Term::SimpleOrPhrase(_) = &term {
+ if let super::Term::SimpleOrPhrase(SimpleOrPhrase::Simple(_)) = &term {
for window_size in 2..=3 {
for (start, end) in sliding_window(window_size, i) {
let mut compounds = Vec::new();
diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rs
index 74b5a2f5..5f155e41 100644
--- a/crates/core/src/query/plan/node.rs
+++ b/crates/core/src/query/plan/node.rs
@@ -116,8 +116,7 @@ impl Node {
.reduce(|left, right| left.or(right))
.expect("fields should not be empty"),
SimpleOrPhrase::Phrase(p) => TextFieldEnum::all()
- .filter(|f| f.is_searchable())
- .filter(|f| f.is_phrase_searchable())
+ .filter(|f| f.is_searchable() && f.is_phrase_searchable())
.map(|field| {
Node::Term(Term {
text: SimpleOrPhrase::Phrase(p.clone()),
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 7fbd1723..91c7aeeb 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -198,7 +198,7 @@ where
T: Graph,
{
async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {
- self.batch_raw_ingoing(nodes, EdgeLimit::Limit(4096))
+ self.batch_raw_ingoing(nodes, EdgeLimit::Limit(1024))
.await
.into_iter()
.map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect())
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index f436aa48..4e24dc7b 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -75,7 +75,7 @@ impl SimilarHostsFinder {
let in_edges = self
.webgraph
- .batch_raw_ingoing_edges(&nodes, EdgeLimit::Limit(128))
+ .batch_raw_ingoing_edges(&nodes, EdgeLimit::Limit(64))
.await
.unwrap_or_default();
@@ -88,7 +88,7 @@ impl SimilarHostsFinder {
let outgoing_edges = self
.webgraph
- .batch_raw_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(4096))
+ .batch_raw_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(512))
.await
.unwrap_or_default();
diff --git a/crates/core/src/webpage/adservers.txt b/crates/core/src/webpage/adservers.txt
index 64741788..dec966b2 100644
--- a/crates/core/src/webpage/adservers.txt
+++ b/crates/core/src/webpage/adservers.txt
@@ -1541,7 +1541,6 @@ delicatecascade.com,
deloo.de,
deloplen.com,
deloton.com,
-demandbase.com,
demdex.net,
deployads.com,
desertedbreath.com,
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 4cb47a69..7a0a49b1 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -80,11 +80,18 @@ impl Html {
robots: None,
};
- if let Some(canonical) = res.canonical_url() {
- if canonical.root_domain() == res.url.root_domain() {
- res.url = canonical;
- }
- }
+ // TODO: need to figure out when to respect
+ // the canonical url, it's not always the case that
+ // the website use the tag correctly.
+ // for instance, https://www.elastic.co/CN/blog/author/brian-bergholm
+ // has a canonical url to https://www.elastic.co/ but it's not the
+ // same content.
+ //
+ // if let Some(canonical) = res.canonical_url() {
+ // if canonical.root_domain() == res.url.root_domain() {
+ // res.url = canonical;
+ // }
+ // }
res.url.normalize_in_place();
res.robots = res.parse_robots_meta();
@@ -983,6 +990,7 @@ mod tests {
}
#[test]
+ #[ignore]
fn canonical_url() {
let html = Html::parse(
r#"
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index a67d3659..1cf6e8ee 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -14,6 +14,7 @@ param_grid = {
"metric": ["ndcg"],
"ndcg_at": [[1, 2, 3, 5, 10]],
"learning_rate": [0.003],
+ # "num_iterations": [100],
# "max_depth": [-1, 2, 4, 8],
"max_depth": [-1],
# "num_leaves": [7, 15, 31],
@@ -23,6 +24,11 @@ param_grid = {
}
+accepted_queries = set()
+with open("data/queries_us.csv") as f:
+ for query in f.readlines():
+ accepted_queries.add(query.strip().lower())
+
con = sqlite3.connect("data/auto-ranking-annotation.sqlite")
cur = con.cursor()
@@ -36,6 +42,11 @@ res = cur.execute(
"""
)
queries = {qid: {"query": query} for qid, query in res.fetchall()}
+qids = list(queries.keys())
+
+for qid in qids:
+ if not queries[qid]["query"].lower() in accepted_queries:
+ del queries[qid]
for qid in queries:
res = cur.execute(
diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.py
index cf975ffe..2f1db7a6 100644
--- a/ltr/leechy_annotate.py
+++ b/ltr/leechy_annotate.py
@@ -34,6 +34,9 @@ unannotated_queries = db.get_unannotated_queries()
eng = leechy.Engine()
for qid, query in tqdm(unannotated_queries.items()):
+ if query not in all_queries:
+ continue
+
tqdm.write(query)
leechy_results = []