web search done right
[webgraph] refactor url tokenizer into smaller helper functions
Mikkel Denkerf56012e770b8
diff --git a/crates/core/src/webgraph/tokenizer/url.rs b/crates/core/src/webgraph/tokenizer/url.rsindex c19e6738..a16d6897 100644--- a/crates/core/src/webgraph/tokenizer/url.rs+++ b/crates/core/src/webgraph/tokenizer/url.rs@@ -28,10 +28,60 @@ struct ParsedUrl {}impl ParsedUrl {+ fn parse_scheme(url: &url::Url) -> VecDeque<String> {+ url.scheme()+ .split_preserve(|c| c == ':')+ .map(|s| {+ let mut s = s.to_string();+ s.push(':');+ s+ })+ .add_space_last()+ .collect()+ }++ fn parse_host(host: &str) -> VecDeque<String> {+ host.split_preserve(|c| c == '.')+ .map(|s| s.to_string())+ .add_space_last()+ .collect()+ }++ fn parse_path(path: &str) -> VecDeque<String> {+ path.split_preserve(|c| matches!(c, '/' | '-' | '_'))+ .filter(|s| !(*s).is_empty())+ .map(|s| s.to_string())+ .collect()+ }++ fn parse_non_http(raw_url: &str, parsed_url: url::Url) -> Result<Self> {+ let scheme: VecDeque<String> = if raw_url.starts_with(&format!("{}:", parsed_url.scheme()))+ {+ Self::parse_scheme(&parsed_url)+ } else {+ VecDeque::new()+ };++ let (host, path) = match parsed_url.path().split_once('@') {+ Some((rest, host_from_path)) => {+ let host = Self::parse_host(host_from_path);+ let path = Self::parse_path(rest);+ (host, path)+ }+ None => {+ let host = Self::parse_host(parsed_url.path());+ let path = VecDeque::new();+ (host, path)+ }+ };++ Ok(Self { scheme, host, path })+ }+fn parse(url: &str) -> Result<Self> {let url = url.replace(" ", "%20");- if !url.contains('.') {+ if !url.contains('.') && !url.contains(':') {let mut host = VecDeque::new();host.push_back(url.to_string());@@ -43,62 +93,30 @@ impl ParsedUrl {}let parsed_url = url::Url::robust_parse(&url)?;- let scheme: VecDeque<String> = if url.starts_with(&format!("{}:", parsed_url.scheme())) {- parsed_url- .scheme()- .split_preserve(|c| c == ':')- .map(|s| {- let mut s = s.to_string();- s.push(':');- s- })- .add_space_last()- .collect()++ if parsed_url.scheme() != "http" && parsed_url.scheme() != "https" {+ return Self::parse_non_http(&url, parsed_url);+ }+ let scheme = if url.starts_with(&format!("{}:", parsed_url.scheme())) {+ Self::parse_scheme(&parsed_url)} else {VecDeque::new()};let normalized_host = parsed_url.normalized_host().unwrap_or_default();- let mut host: VecDeque<String> = if normalized_host != parsed_url.scheme() {- normalized_host- .split_preserve(|c| c == '.')- .map(|s| s.to_string())- .add_space_last()- .collect()+ let host = if normalized_host != parsed_url.scheme() {+ Self::parse_host(normalized_host)} else {VecDeque::new()};- let mut path: VecDeque<_> = if normalized_host != parsed_url.scheme() {- parsed_url- .path()- .split_preserve(|c| matches!(c, '/' | '-' | '_'))- .filter(|s| !(*s).is_empty())- .map(|s| s.to_string())- .collect()+ let path = if normalized_host != parsed_url.scheme() {+ Self::parse_path(parsed_url.path())} else {VecDeque::new()};- if host.is_empty() {- if let Some(maybe_host) = path.pop_front() {- let host_from_path = match maybe_host.split_once('@') {- Some((rest, host)) => {- path.push_front(rest.to_string());- host.to_string()- }- None => maybe_host.to_string(),- };-- host = host_from_path- .split_preserve(|c| c == '.')- .map(|s| s.to_string())- .add_space_last()- .collect();- }- }-Ok(Self { scheme, host, path })}@@ -108,19 +126,43 @@ impl ParsedUrl {.or_else(|| self.host.pop_front()).or_else(|| self.path.pop_front())}++ pub fn is_empty(&self) -> bool {+ self.scheme.is_empty() && self.host.is_empty() && self.path.is_empty()+ }}pub struct UrlTokenStream {- url: ParsedUrl,+ urls: VecDeque<ParsedUrl>,token: Token,}impl UrlTokenStream {pub fn new(text: &str) -> Self {- let url = ParsedUrl::parse(text).unwrap_or_default();+ let mut urls = VecDeque::new();++ if let Ok(url) = url::Url::robust_parse(text) {+ if url.scheme() != "http" && url.scheme() != "https" {+ let scheme = url.scheme().to_string();++ for url in text.split(',') {+ let mut url = url.to_string();++ if !url.starts_with(&format!("{}:", scheme)) {+ url = format!("{}:{}", scheme, url);+ }++ if let Ok(parsed_url) = ParsedUrl::parse(&url) {+ urls.push_back(parsed_url);+ }+ }+ } else if let Ok(parsed_url) = ParsedUrl::parse(text) {+ urls.push_back(parsed_url);+ }+ }Self {- url,+ urls,token: Token::default(),}}@@ -128,15 +170,23 @@ impl UrlTokenStream {impl TokenStream for UrlTokenStream {fn advance(&mut self) -> bool {- match self.url.next() {- Some(token) => {- self.token.position = self.token.position.wrapping_add(1);- self.token.offset_from = self.token.offset_to;- self.token.offset_to = self.token.offset_from + token.len();- self.token.text = token;-- true+ if let Some(url) = self.urls.front() {+ if url.is_empty() {+ self.urls.pop_front();}+ }++ match self.urls.front_mut() {+ Some(url) => match url.next() {+ Some(token) => {+ self.token.position = self.token.position.wrapping_add(1);+ self.token.offset_from = self.token.offset_to;+ self.token.offset_to = self.token.offset_from + token.len();+ self.token.text = token;+ true+ }+ None => false,+ },None => false,}}