web search done right
exacturl search operator
Mikkel Denker44b285c41a90
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex f27f6063..d4673f04 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -189,7 +189,7 @@ impl<S: DatumStream> JobExecutor<S> {.into_iter().chain(self.sitemap_urls.drain().map(|url| (url.clone(), 0.0))).map(|(mut url, score)| {- url.normalize();+ url.normalize_in_place();(url, score)}).filter(|(url, _)| !self.crawled_urls.contains(url))@@ -206,7 +206,7 @@ impl<S: DatumStream> JobExecutor<S> {.into_iter().map(|(url, _)| url).map(|mut url| {- url.normalize();+ url.normalize_in_place();url}).filter(|url| !self.crawled_urls.contains(url))@@ -345,7 +345,7 @@ impl<S: DatumStream> JobExecutor<S> {.into_iter().map(|link| link.destination).map(|mut url| {- url.normalize();+ url.normalize_in_place();url}).filter(|url| url.as_str().len() <= MAX_URL_LEN_BYTES)@@ -495,7 +495,7 @@ impl<S: DatumStream> JobExecutor<S> {async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {let mut url = url;- url.normalize();+ url.normalize_in_place();if self.crawled_urls.contains(&url) {return Err(Error::from(anyhow!("url already crawled: {}", url)));@@ -526,7 +526,7 @@ impl<S: DatumStream> JobExecutor<S> {}let mut res_url = res.url().clone();- res_url.normalize();+ res_url.normalize_in_place();self.crawled_urls.insert(res_url.clone());diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rsindex c454bedb..01810f32 100644--- a/crates/core/src/query/parser/mod.rs+++ b/crates/core/src/query/parser/mod.rs@@ -16,7 +16,7 @@pub const MAX_TERMS_PER_QUERY: usize = 32;-use crate::bangs::BANG_PREFIXES;+use crate::{bangs::BANG_PREFIXES, webpage::url_ext::UrlExt};mod term;@@ -119,6 +119,19 @@ fn site_field(input: &str) -> nom::IResult<&str, Term> {Ok((input, Term::Site(output.to_string())))}+fn exact_url(orig_input: &str) -> nom::IResult<&str, Term> {+ let (input, _) = nom::bytes::complete::tag("exacturl:")(orig_input)?;+ let (input, output) = simple_str(input)?;++ match url::Url::parse(output) {+ Ok(url) => Ok((input, Term::ExactUrl(url.normalize().to_string()))),+ Err(_) => Err(nom::Err::Error(nom::error::Error::new(+ orig_input,+ nom::error::ErrorKind::Fail,+ ))),+ }+}+fn links_to_field(input: &str) -> nom::IResult<&str, Term> {// parse 'linksto:' and then a simple termlet (input, _) = nom::branch::alt((@@ -147,7 +160,7 @@ fn body_field(input: &str) -> nom::IResult<&str, Term> {}fn url_field(input: &str) -> nom::IResult<&str, Term> {- // parse 'url:' and then a simple term+ // parse 'inurl:' and then a simple termlet (input, _) = nom::bytes::complete::tag("inurl:")(input)?;let (input, output) = simple_or_phrase(input)?;@@ -161,6 +174,7 @@ fn field_selector(input: &str) -> nom::IResult<&str, Term> {title_field,body_field,url_field,+ exact_url,))(input)}diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rsindex 7eb26d31..9c47d648 100644--- a/crates/core/src/query/parser/term.rs+++ b/crates/core/src/query/parser/term.rs@@ -102,6 +102,7 @@ pub enum Term {Title(SimpleOrPhrase),Body(SimpleOrPhrase),Url(SimpleOrPhrase),+ ExactUrl(String),PossibleBang { prefix: char, bang: String },Not(Box<Term>),}@@ -116,6 +117,7 @@ impl std::fmt::Display for Term {Term::Title(title) => write!(f, "intitle:{}", title),Term::Body(body) => write!(f, "inbody:{}", body),Term::Url(url) => write!(f, "inurl:{}", url),+ Term::ExactUrl(url) => write!(f, "exacturl:{}", url),Term::PossibleBang { prefix, bang } => write!(f, "{}{}", prefix, bang),}}@@ -137,6 +139,7 @@ impl Term {Term::Title(s) => Term::Title(s.truncate()),Term::Body(s) => Term::Body(s.truncate()),Term::Url(s) => Term::Url(s.truncate()),+ Term::ExactUrl(s) => Term::ExactUrl(s),Term::Not(n) => Term::Not(Box::new(n.truncate())),Term::PossibleBang { prefix, bang } => Term::PossibleBang {prefix,diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rsindex 2840e221..74b5a2f5 100644--- a/crates/core/src/query/plan/node.rs+++ b/crates/core/src/query/plan/node.rs@@ -147,6 +147,10 @@ impl Node {text: u,field: text_field::Url.into(),}),+ ParserTerm::ExactUrl(u) => Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(u)),+ field: text_field::UrlNoTokenizer.into(),+ }),ParserTerm::PossibleBang { prefix, bang } => {let mut s = String::new();s.push(prefix);diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex 9f02a28d..63a5b22f 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -171,7 +171,7 @@ impl From<Url> for Node {pub fn normalize_url(url: &Url) -> String {let mut url = url.clone();- url.normalize();+ url.normalize_in_place();let scheme = url.scheme();let mut normalized = urldiff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex 1759b646..4cb47a69 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -86,7 +86,7 @@ impl Html {}}- res.url.normalize();+ res.url.normalize_in_place();res.robots = res.parse_robots_meta();Ok(res)diff --git a/crates/core/src/webpage/url_ext.rs b/crates/core/src/webpage/url_ext.rsindex 12f10e5f..84c3df9e 100644--- a/crates/core/src/webpage/url_ext.rs+++ b/crates/core/src/webpage/url_ext.rs@@ -36,7 +36,16 @@ pub trait UrlExt {fn icann_domain(&self) -> Option<&str>;fn root_domain(&self) -> Option<&str>;fn normalized_host(&self) -> Option<&str>;- fn normalize(&mut self);+ fn normalize_in_place(&mut self);++ fn normalize(mut self) -> Self+ where+ Self: Sized,+ {+ self.normalize_in_place();+ self+ }+fn subdomain(&self) -> Option<&str>;fn is_homepage(&self) -> bool;fn tld(&self) -> Option<&str>;@@ -60,7 +69,7 @@ impl UrlExt for url::Url {self.host_str().map(|host| host.trim_start_matches("www."))}- fn normalize(&mut self) {+ fn normalize_in_place(&mut self) {self.set_fragment(None); // remove fragment (e.g. #commentslet queries: Vec<_> = selfdiff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.pyindex f1443efd..6f38b471 100644--- a/ltr/leechy_annotate.py+++ b/ltr/leechy_annotate.py@@ -42,7 +42,7 @@ for qid, query in tqdm(unannotated_queries.items()):try:data = next(res- for res in stract.search(f"{query} site:{result}")+ for res in stract.search(f"{query} exacturl:{result}")if res["url"] == result)except StopIteration: