Skip to content

web search done right

[webgraph] refactor url tokenizer into smaller helper functions

Mikkel Denkerf56012e770b8
diff --git a/crates/core/src/webgraph/tokenizer/url.rs b/crates/core/src/webgraph/tokenizer/url.rs
index c19e6738..a16d6897 100644
--- a/crates/core/src/webgraph/tokenizer/url.rs
+++ b/crates/core/src/webgraph/tokenizer/url.rs
@@ -28,10 +28,60 @@ struct ParsedUrl {
}
impl ParsedUrl {
+ fn parse_scheme(url: &url::Url) -> VecDeque<String> {
+ url.scheme()
+ .split_preserve(|c| c == ':')
+ .map(|s| {
+ let mut s = s.to_string();
+ s.push(':');
+ s
+ })
+ .add_space_last()
+ .collect()
+ }
+
+ fn parse_host(host: &str) -> VecDeque<String> {
+ host.split_preserve(|c| c == '.')
+ .map(|s| s.to_string())
+ .add_space_last()
+ .collect()
+ }
+
+ fn parse_path(path: &str) -> VecDeque<String> {
+ path.split_preserve(|c| matches!(c, '/' | '-' | '_'))
+ .filter(|s| !(*s).is_empty())
+ .map(|s| s.to_string())
+ .collect()
+ }
+
+ fn parse_non_http(raw_url: &str, parsed_url: url::Url) -> Result<Self> {
+ let scheme: VecDeque<String> = if raw_url.starts_with(&format!("{}:", parsed_url.scheme()))
+ {
+ Self::parse_scheme(&parsed_url)
+ } else {
+ VecDeque::new()
+ };
+
+ let (host, path) = match parsed_url.path().split_once('@') {
+ Some((rest, host_from_path)) => {
+ let host = Self::parse_host(host_from_path);
+ let path = Self::parse_path(rest);
+ (host, path)
+ }
+ None => {
+ let host = Self::parse_host(parsed_url.path());
+ let path = VecDeque::new();
+ (host, path)
+ }
+ };
+
+ Ok(Self { scheme, host, path })
+ }
+
fn parse(url: &str) -> Result<Self> {
let url = url.replace(" ", "%20");
- if !url.contains('.') {
+ if !url.contains('.') && !url.contains(':') {
let mut host = VecDeque::new();
host.push_back(url.to_string());
@@ -43,62 +93,30 @@ impl ParsedUrl {
}
let parsed_url = url::Url::robust_parse(&url)?;
- let scheme: VecDeque<String> = if url.starts_with(&format!("{}:", parsed_url.scheme())) {
- parsed_url
- .scheme()
- .split_preserve(|c| c == ':')
- .map(|s| {
- let mut s = s.to_string();
- s.push(':');
- s
- })
- .add_space_last()
- .collect()
+
+ if parsed_url.scheme() != "http" && parsed_url.scheme() != "https" {
+ return Self::parse_non_http(&url, parsed_url);
+ }
+ let scheme = if url.starts_with(&format!("{}:", parsed_url.scheme())) {
+ Self::parse_scheme(&parsed_url)
} else {
VecDeque::new()
};
let normalized_host = parsed_url.normalized_host().unwrap_or_default();
- let mut host: VecDeque<String> = if normalized_host != parsed_url.scheme() {
- normalized_host
- .split_preserve(|c| c == '.')
- .map(|s| s.to_string())
- .add_space_last()
- .collect()
+ let host = if normalized_host != parsed_url.scheme() {
+ Self::parse_host(normalized_host)
} else {
VecDeque::new()
};
- let mut path: VecDeque<_> = if normalized_host != parsed_url.scheme() {
- parsed_url
- .path()
- .split_preserve(|c| matches!(c, '/' | '-' | '_'))
- .filter(|s| !(*s).is_empty())
- .map(|s| s.to_string())
- .collect()
+ let path = if normalized_host != parsed_url.scheme() {
+ Self::parse_path(parsed_url.path())
} else {
VecDeque::new()
};
- if host.is_empty() {
- if let Some(maybe_host) = path.pop_front() {
- let host_from_path = match maybe_host.split_once('@') {
- Some((rest, host)) => {
- path.push_front(rest.to_string());
- host.to_string()
- }
- None => maybe_host.to_string(),
- };
-
- host = host_from_path
- .split_preserve(|c| c == '.')
- .map(|s| s.to_string())
- .add_space_last()
- .collect();
- }
- }
-
Ok(Self { scheme, host, path })
}
@@ -108,19 +126,43 @@ impl ParsedUrl {
.or_else(|| self.host.pop_front())
.or_else(|| self.path.pop_front())
}
+
+ pub fn is_empty(&self) -> bool {
+ self.scheme.is_empty() && self.host.is_empty() && self.path.is_empty()
+ }
}
pub struct UrlTokenStream {
- url: ParsedUrl,
+ urls: VecDeque<ParsedUrl>,
token: Token,
}
impl UrlTokenStream {
pub fn new(text: &str) -> Self {
- let url = ParsedUrl::parse(text).unwrap_or_default();
+ let mut urls = VecDeque::new();
+
+ if let Ok(url) = url::Url::robust_parse(text) {
+ if url.scheme() != "http" && url.scheme() != "https" {
+ let scheme = url.scheme().to_string();
+
+ for url in text.split(',') {
+ let mut url = url.to_string();
+
+ if !url.starts_with(&format!("{}:", scheme)) {
+ url = format!("{}:{}", scheme, url);
+ }
+
+ if let Ok(parsed_url) = ParsedUrl::parse(&url) {
+ urls.push_back(parsed_url);
+ }
+ }
+ } else if let Ok(parsed_url) = ParsedUrl::parse(text) {
+ urls.push_back(parsed_url);
+ }
+ }
Self {
- url,
+ urls,
token: Token::default(),
}
}
@@ -128,15 +170,23 @@ impl UrlTokenStream {
impl TokenStream for UrlTokenStream {
fn advance(&mut self) -> bool {
- match self.url.next() {
- Some(token) => {
- self.token.position = self.token.position.wrapping_add(1);
- self.token.offset_from = self.token.offset_to;
- self.token.offset_to = self.token.offset_from + token.len();
- self.token.text = token;
-
- true
+ if let Some(url) = self.urls.front() {
+ if url.is_empty() {
+ self.urls.pop_front();
}
+ }
+
+ match self.urls.front_mut() {
+ Some(url) => match url.next() {
+ Some(token) => {
+ self.token.position = self.token.position.wrapping_add(1);
+ self.token.offset_from = self.token.offset_to;
+ self.token.offset_to = self.token.offset_from + token.len();
+ self.token.text = token;
+ true
+ }
+ None => false,
+ },
None => false,
}
}