web search done right
[webgraph] match non-url-like queries against any term in parsed url
Mikkel Denker3e88c6e8a940
diff --git a/crates/core/src/webgraph/tokenizer/url.rs b/crates/core/src/webgraph/tokenizer/url.rsindex b0b39d1b..c19e6738 100644--- a/crates/core/src/webgraph/tokenizer/url.rs+++ b/crates/core/src/webgraph/tokenizer/url.rs@@ -29,7 +29,20 @@ struct ParsedUrl {impl ParsedUrl {fn parse(url: &str) -> Result<Self> {- let parsed_url = url::Url::robust_parse(url)?;+ let url = url.replace(" ", "%20");++ if !url.contains('.') {+ let mut host = VecDeque::new();+ host.push_back(url.to_string());++ return Ok(Self {+ scheme: VecDeque::new(),+ host,+ path: VecDeque::new(),+ });+ }++ let parsed_url = url::Url::robust_parse(&url)?;let scheme: VecDeque<String> = if url.starts_with(&format!("{}:", parsed_url.scheme())) {parsed_url.scheme()@@ -158,5 +171,7 @@ mod tests {);assert_eq!(tokenize("example.com"), ["example", ".", "com ", "/"]);assert_eq!(tokenize(".com"), [".", "com ", "/"]);+ assert_eq!(tokenize("example"), ["example"]);+ assert_eq!(tokenize("example-site"), ["example-site"]);}}