Skip to content

web search done right

exacturl search operator

Mikkel Denker44b285c41a90
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index f27f6063..d4673f04 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -189,7 +189,7 @@ impl<S: DatumStream> JobExecutor<S> {
.into_iter()
.chain(self.sitemap_urls.drain().map(|url| (url.clone(), 0.0)))
.map(|(mut url, score)| {
- url.normalize();
+ url.normalize_in_place();
(url, score)
})
.filter(|(url, _)| !self.crawled_urls.contains(url))
@@ -206,7 +206,7 @@ impl<S: DatumStream> JobExecutor<S> {
.into_iter()
.map(|(url, _)| url)
.map(|mut url| {
- url.normalize();
+ url.normalize_in_place();
url
})
.filter(|url| !self.crawled_urls.contains(url))
@@ -345,7 +345,7 @@ impl<S: DatumStream> JobExecutor<S> {
.into_iter()
.map(|link| link.destination)
.map(|mut url| {
- url.normalize();
+ url.normalize_in_place();
url
})
.filter(|url| url.as_str().len() <= MAX_URL_LEN_BYTES)
@@ -495,7 +495,7 @@ impl<S: DatumStream> JobExecutor<S> {
async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {
let mut url = url;
- url.normalize();
+ url.normalize_in_place();
if self.crawled_urls.contains(&url) {
return Err(Error::from(anyhow!("url already crawled: {}", url)));
@@ -526,7 +526,7 @@ impl<S: DatumStream> JobExecutor<S> {
}
let mut res_url = res.url().clone();
- res_url.normalize();
+ res_url.normalize_in_place();
self.crawled_urls.insert(res_url.clone());
diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rs
index c454bedb..01810f32 100644
--- a/crates/core/src/query/parser/mod.rs
+++ b/crates/core/src/query/parser/mod.rs
@@ -16,7 +16,7 @@
pub const MAX_TERMS_PER_QUERY: usize = 32;
-use crate::bangs::BANG_PREFIXES;
+use crate::{bangs::BANG_PREFIXES, webpage::url_ext::UrlExt};
mod term;
@@ -119,6 +119,19 @@ fn site_field(input: &str) -> nom::IResult<&str, Term> {
Ok((input, Term::Site(output.to_string())))
}
+fn exact_url(orig_input: &str) -> nom::IResult<&str, Term> {
+ let (input, _) = nom::bytes::complete::tag("exacturl:")(orig_input)?;
+ let (input, output) = simple_str(input)?;
+
+ match url::Url::parse(output) {
+ Ok(url) => Ok((input, Term::ExactUrl(url.normalize().to_string()))),
+ Err(_) => Err(nom::Err::Error(nom::error::Error::new(
+ orig_input,
+ nom::error::ErrorKind::Fail,
+ ))),
+ }
+}
+
fn links_to_field(input: &str) -> nom::IResult<&str, Term> {
// parse 'linksto:' and then a simple term
let (input, _) = nom::branch::alt((
@@ -147,7 +160,7 @@ fn body_field(input: &str) -> nom::IResult<&str, Term> {
}
fn url_field(input: &str) -> nom::IResult<&str, Term> {
- // parse 'url:' and then a simple term
+ // parse 'inurl:' and then a simple term
let (input, _) = nom::bytes::complete::tag("inurl:")(input)?;
let (input, output) = simple_or_phrase(input)?;
@@ -161,6 +174,7 @@ fn field_selector(input: &str) -> nom::IResult<&str, Term> {
title_field,
body_field,
url_field,
+ exact_url,
))(input)
}
diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rs
index 7eb26d31..9c47d648 100644
--- a/crates/core/src/query/parser/term.rs
+++ b/crates/core/src/query/parser/term.rs
@@ -102,6 +102,7 @@ pub enum Term {
Title(SimpleOrPhrase),
Body(SimpleOrPhrase),
Url(SimpleOrPhrase),
+ ExactUrl(String),
PossibleBang { prefix: char, bang: String },
Not(Box<Term>),
}
@@ -116,6 +117,7 @@ impl std::fmt::Display for Term {
Term::Title(title) => write!(f, "intitle:{}", title),
Term::Body(body) => write!(f, "inbody:{}", body),
Term::Url(url) => write!(f, "inurl:{}", url),
+ Term::ExactUrl(url) => write!(f, "exacturl:{}", url),
Term::PossibleBang { prefix, bang } => write!(f, "{}{}", prefix, bang),
}
}
@@ -137,6 +139,7 @@ impl Term {
Term::Title(s) => Term::Title(s.truncate()),
Term::Body(s) => Term::Body(s.truncate()),
Term::Url(s) => Term::Url(s.truncate()),
+ Term::ExactUrl(s) => Term::ExactUrl(s),
Term::Not(n) => Term::Not(Box::new(n.truncate())),
Term::PossibleBang { prefix, bang } => Term::PossibleBang {
prefix,
diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rs
index 2840e221..74b5a2f5 100644
--- a/crates/core/src/query/plan/node.rs
+++ b/crates/core/src/query/plan/node.rs
@@ -147,6 +147,10 @@ impl Node {
text: u,
field: text_field::Url.into(),
}),
+ ParserTerm::ExactUrl(u) => Node::Term(Term {
+ text: SimpleOrPhrase::Simple(SimpleTerm::from(u)),
+ field: text_field::UrlNoTokenizer.into(),
+ }),
ParserTerm::PossibleBang { prefix, bang } => {
let mut s = String::new();
s.push(prefix);
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index 9f02a28d..63a5b22f 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -171,7 +171,7 @@ impl From<Url> for Node {
pub fn normalize_url(url: &Url) -> String {
let mut url = url.clone();
- url.normalize();
+ url.normalize_in_place();
let scheme = url.scheme();
let mut normalized = url
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 1759b646..4cb47a69 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -86,7 +86,7 @@ impl Html {
}
}
- res.url.normalize();
+ res.url.normalize_in_place();
res.robots = res.parse_robots_meta();
Ok(res)
diff --git a/crates/core/src/webpage/url_ext.rs b/crates/core/src/webpage/url_ext.rs
index 12f10e5f..84c3df9e 100644
--- a/crates/core/src/webpage/url_ext.rs
+++ b/crates/core/src/webpage/url_ext.rs
@@ -36,7 +36,16 @@ pub trait UrlExt {
fn icann_domain(&self) -> Option<&str>;
fn root_domain(&self) -> Option<&str>;
fn normalized_host(&self) -> Option<&str>;
- fn normalize(&mut self);
+ fn normalize_in_place(&mut self);
+
+ fn normalize(mut self) -> Self
+ where
+ Self: Sized,
+ {
+ self.normalize_in_place();
+ self
+ }
+
fn subdomain(&self) -> Option<&str>;
fn is_homepage(&self) -> bool;
fn tld(&self) -> Option<&str>;
@@ -60,7 +69,7 @@ impl UrlExt for url::Url {
self.host_str().map(|host| host.trim_start_matches("www."))
}
- fn normalize(&mut self) {
+ fn normalize_in_place(&mut self) {
self.set_fragment(None); // remove fragment (e.g. #comments
let queries: Vec<_> = self
diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.py
index f1443efd..6f38b471 100644
--- a/ltr/leechy_annotate.py
+++ b/ltr/leechy_annotate.py
@@ -42,7 +42,7 @@ for qid, query in tqdm(unannotated_queries.items()):
try:
data = next(
res
- for res in stract.search(f"{query} site:{result}")
+ for res in stract.search(f"{query} exacturl:{result}")
if res["url"] == result
)
except StopIteration: