web search done right
[webgraph] use specialized url tokenizer
Mikkel Denker1994460d9c1f
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 4360c4e7..76746efa 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -421,6 +421,7 @@ impl<S: DatumStream> JobExecutor<S> {html.anchor_links().into_iter().map(|link| link.destination)+ .filter(|url| matches!(url.scheme(), "http" | "https")).map(|mut url| {url.normalize_in_place();urldiff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rsindex 1e156101..38258339 100644--- a/crates/core/src/distributed/sonic/service.rs+++ b/crates/core/src/distributed/sonic/service.rs@@ -71,9 +71,11 @@ impl<S: Service> Server<S> {let mut res = Vec::new();for req in bodies {- res.push(S::handle(req, &service).await);+ res.push(S::handle(req, &service));}+ let res = futures::future::join_all(res).await;+if let Err(e) = req.respond(OneOrMany::Many(res)).await {tracing::error!("failed to respond to request: {}", e);}diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex d714447f..54344935 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -110,11 +110,7 @@ impl WebgraphWorker {.and_then(|store| store.get(&source.clone().into_host().id()).unwrap()).unwrap_or(0.0);- for mut link in webpage- .anchor_links()- .into_iter()- .filter(|link| matches!(link.destination.scheme(), "http" | "https"))- {+ for mut link in webpage.anchor_links().into_iter() {let mut destination = link.destination.clone();if let Some(index) = &self.canonical_index {diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 725739ef..ca790d9a 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -69,7 +69,7 @@ pub mod iter_ext;mod kahan_sum;mod leaky_queue;mod live_index;-mod log_group;+pub mod log_group;mod metrics;mod models;pub mod naive_bayes;diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rsindex 13b33005..68409364 100644--- a/crates/core/src/live_index/crawler/checker/frontpage.rs+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs@@ -52,6 +52,7 @@ impl Checker for Frontpage {let urls = page.anchor_links().into_iter()+ .filter(|link| matches!(link.destination.scheme(), "http" | "https")).map(|link| CrawlableUrl::from(link.destination)).collect::<Vec<_>>();diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex 3d0d954c..6c791d1b 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -101,8 +101,12 @@ impl SimilarHostsFinder {.iter().flatten().filter(|e| !e.rel_flags.contains(RelFlags::NOFOLLOW))- .map(|e| e.to)- .unique()+ .map(|e| (e.to, 1))+ .into_group_map()+ .into_iter()+ .sorted_by_key(|(_, c)| Reverse(c.len()))+ .map(|(n, _)| n)+ .take(1024).filter(|n| !nodes.contains(n)).collect()}diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rsindex 3386ec31..a871de30 100644--- a/crates/core/src/tokenizer/mod.rs+++ b/crates/core/src/tokenizer/mod.rs@@ -14,15 +14,15 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-mod add_space_last;+pub mod add_space_last;pub mod fields;pub mod normalizer;-mod script;-mod script_tokenizer;-mod segmenter;-mod split_preserve;-mod split_with_range;-mod stemmer;+pub mod script;+pub mod script_tokenizer;+pub mod segmenter;+pub mod split_preserve;+pub mod split_with_range;+pub mod stemmer;use std::borrow::{Borrow, Cow};diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rsindex baaef16a..d09ae463 100644--- a/crates/core/src/webgraph/schema.rs+++ b/crates/core/src/webgraph/schema.rs@@ -27,6 +27,7 @@ use crate::enum_dispatch_from_discriminant;use crate::Result;use super::document::{Edge, ReferenceValue};+use super::tokenizer;use super::tokenizer::Tokenizer;use super::tokenizer::TokenizerEnum;use super::Node;@@ -64,6 +65,10 @@ impl Field for FromUrl {"from_url"}+ fn tokenizer(&self) -> TokenizerEnum {+ tokenizer::UrlTokenizer.into()+ }+fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {ReferenceValue::Str(edge.from.as_str())}@@ -95,6 +100,10 @@ impl Field for ToUrl {"to_url"}+ fn tokenizer(&self) -> TokenizerEnum {+ tokenizer::UrlTokenizer.into()+ }+fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {ReferenceValue::Str(edge.to.as_str())}diff --git a/crates/core/src/webgraph/tokenizer.rs b/crates/core/src/webgraph/tokenizer/mod.rssimilarity index 86%rename from crates/core/src/webgraph/tokenizer.rsrename to crates/core/src/webgraph/tokenizer/mod.rsindex 45d393d2..2d4f9e8c 100644--- a/crates/core/src/webgraph/tokenizer.rs+++ b/crates/core/src/webgraph/tokenizer/mod.rs@@ -14,9 +14,12 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+mod url;+use enum_dispatch::enum_dispatch;use strum::{EnumDiscriminants, VariantArray};use tantivy::tokenizer::{BoxTokenStream, Tokenizer as _};+use url::UrlTokenStream;#[enum_dispatch]pub trait Tokenizer: Into<TokenizerEnum> + Clone {@@ -41,11 +44,25 @@ impl Tokenizer for SimpleTokenizer {}}+#[derive(Clone, Default)]+pub struct UrlTokenizer;++impl Tokenizer for UrlTokenizer {+ fn name(&self) -> &'static str {+ "url"+ }++ fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {+ BoxTokenStream::new(UrlTokenStream::new(text))+ }+}+#[enum_dispatch(Tokenizer)]#[derive(Clone, EnumDiscriminants)]#[strum_discriminants(derive(VariantArray))]pub enum TokenizerEnum {SimpleTokenizer,+ UrlTokenizer,}impl TokenizerEnum {@@ -66,6 +83,7 @@ impl From<TokenizerEnumDiscriminants> for TokenizerEnum {fn from(value: TokenizerEnumDiscriminants) -> Self {match value {TokenizerEnumDiscriminants::SimpleTokenizer => SimpleTokenizer::default().into(),+ TokenizerEnumDiscriminants::UrlTokenizer => UrlTokenizer.into(),}}}diff --git a/crates/core/src/webgraph/tokenizer/url.rs b/crates/core/src/webgraph/tokenizer/url.rsnew file mode 100644index 00000000..6ecf6884--- /dev/null+++ b/crates/core/src/webgraph/tokenizer/url.rs@@ -0,0 +1,165 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::tokenizer::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};+use crate::{webpage::url_ext::UrlExt, Result};+use std::collections::VecDeque;++use tantivy::tokenizer::{Token, TokenStream};++#[derive(Default)]+struct ParsedUrl {+ scheme: VecDeque<String>,+ host: VecDeque<String>,+ path: VecDeque<String>,+}++impl ParsedUrl {+ fn parse(url: &str) -> Result<Self> {+ let url = url::Url::robust_parse(url)?;+ let scheme: VecDeque<String> = url+ .scheme()+ .split_preserve(|c| c == ':')+ .map(|s| {+ let mut s = s.to_string();+ s.push(':');+ s+ })+ .add_space_last()+ .collect();++ let mut host: VecDeque<String> = url+ .normalized_host()+ .unwrap_or_default()+ .split_preserve(|c| c == '.')+ .map(|s| s.to_string())+ .add_space_last()+ .collect();++ let mut path: VecDeque<_> = url+ .path()+ .split_preserve(|c| matches!(c, '/' | '-' | '_'))+ .filter(|s| !(*s).is_empty())+ .map(|s| s.to_string())+ .collect();++ if host.is_empty() {+ if let Some(maybe_host) = path.pop_front() {+ let host_from_path = match maybe_host.split_once('@') {+ Some((rest, host)) => {+ path.push_front(rest.to_string());+ host.to_string()+ }+ None => maybe_host.to_string(),+ };++ host = host_from_path+ .split_preserve(|c| c == '.')+ .map(|s| s.to_string())+ .add_space_last()+ .collect();+ }+ }++ Ok(Self { scheme, host, path })+ }++ pub fn next(&mut self) -> Option<String> {+ self.scheme+ .pop_front()+ .or_else(|| self.host.pop_front())+ .or_else(|| self.path.pop_front())+ }+}++pub struct UrlTokenStream {+ url: ParsedUrl,+ token: Token,+}++impl UrlTokenStream {+ pub fn new(text: &str) -> Self {+ let url = ParsedUrl::parse(text).unwrap_or_default();++ Self {+ url,+ token: Token::default(),+ }+ }+}++impl TokenStream for UrlTokenStream {+ fn advance(&mut self) -> bool {+ match self.url.next() {+ Some(token) => {+ self.token.position = self.token.position.wrapping_add(1);+ self.token.offset_from = self.token.offset_to;+ self.token.offset_to = self.token.offset_from + token.len();+ self.token.text = token;++ true+ }+ None => false,+ }+ }++ fn token(&self) -> &Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut Token {+ &mut self.token+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn tokenize(url: &str) -> Vec<String> {+ let mut url = UrlTokenStream::new(url);+ let mut tokens = Vec::new();+ while url.advance() {+ tokens.push(url.token().text.clone());+ }+ tokens+ }++ #[test]+ fn test_parse() {+ assert_eq!(+ tokenize("https://example.com/path/to/resource"),+ ["https: ", "example", ".", "com ", "/", "path", "/", "to", "/", "resource"]+ );+ assert_eq!(+ tokenize("example.com"),+ ["https: ", "example", ".", "com ", "/"]+ );+ assert_eq!(tokenize(".com"), ["https: ", ".", "com ", "/"]);++ assert_eq!(+ tokenize("mailto:[email protected]"),+ ["mailto: ", "example", ".", "com ", "hello"]+ );++ assert_eq!(+ tokenize("mailto:example.com"),+ ["mailto: ", "example", ".", "com "]+ );++ assert_eq!(tokenize("tel:+4512345678"), ["tel: ", "+4512345678 "]);+ }+}diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rsindex 65fdb040..c1dc47ab 100644--- a/crates/core/src/webpage/html/links.rs+++ b/crates/core/src/webpage/html/links.rs@@ -288,10 +288,6 @@ impl Html {if &element.name.local == "a" {if let Some((text, attributes)) = open_links.pop() {if let Some(dest) = attributes.borrow().get("href") {- if dest.starts_with("mailto:") || dest.starts_with("tel:") {- continue;- }-if let Ok(dest) =Url::parse_with_base_url(self.base_url(), dest){@@ -338,10 +334,6 @@ impl Html {while let Some((text, attributes)) = open_links.pop() {if let Some(dest) = attributes.borrow().get("href") {- if dest.starts_with("mailto:") || dest.starts_with("tel:") {- continue;- }-if let Ok(dest) = Url::parse_with_base_url(self.base_url(), dest) {let mut rel = RelFlags::from_html(&dest, &attributes.borrow(), &location);diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex c5a1e6f6..94f9d783 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -558,7 +558,6 @@ mod tests {<body><a href="https://example.com">Link to example</a><p>{CONTENT}</p>- <a href="mailto:[email protected]">Email me</a></body></html>"#