Skip to content

web search done right

[webgraph] use specialized url tokenizer

Mikkel Denker1994460d9c1f
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index 4360c4e7..76746efa 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -421,6 +421,7 @@ impl<S: DatumStream> JobExecutor<S> {
html.anchor_links()
.into_iter()
.map(|link| link.destination)
+ .filter(|url| matches!(url.scheme(), "http" | "https"))
.map(|mut url| {
url.normalize_in_place();
url
diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rs
index 1e156101..38258339 100644
--- a/crates/core/src/distributed/sonic/service.rs
+++ b/crates/core/src/distributed/sonic/service.rs
@@ -71,9 +71,11 @@ impl<S: Service> Server<S> {
let mut res = Vec::new();
for req in bodies {
- res.push(S::handle(req, &service).await);
+ res.push(S::handle(req, &service));
}
+ let res = futures::future::join_all(res).await;
+
if let Err(e) = req.respond(OneOrMany::Many(res)).await {
tracing::error!("failed to respond to request: {}", e);
}
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index d714447f..54344935 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -110,11 +110,7 @@ impl WebgraphWorker {
.and_then(|store| store.get(&source.clone().into_host().id()).unwrap())
.unwrap_or(0.0);
- for mut link in webpage
- .anchor_links()
- .into_iter()
- .filter(|link| matches!(link.destination.scheme(), "http" | "https"))
- {
+ for mut link in webpage.anchor_links().into_iter() {
let mut destination = link.destination.clone();
if let Some(index) = &self.canonical_index {
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 725739ef..ca790d9a 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -69,7 +69,7 @@ pub mod iter_ext;
mod kahan_sum;
mod leaky_queue;
mod live_index;
-mod log_group;
+pub mod log_group;
mod metrics;
mod models;
pub mod naive_bayes;
diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rs
index 13b33005..68409364 100644
--- a/crates/core/src/live_index/crawler/checker/frontpage.rs
+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs
@@ -52,6 +52,7 @@ impl Checker for Frontpage {
let urls = page
.anchor_links()
.into_iter()
+ .filter(|link| matches!(link.destination.scheme(), "http" | "https"))
.map(|link| CrawlableUrl::from(link.destination))
.collect::<Vec<_>>();
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index 3d0d954c..6c791d1b 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -101,8 +101,12 @@ impl SimilarHostsFinder {
.iter()
.flatten()
.filter(|e| !e.rel_flags.contains(RelFlags::NOFOLLOW))
- .map(|e| e.to)
- .unique()
+ .map(|e| (e.to, 1))
+ .into_group_map()
+ .into_iter()
+ .sorted_by_key(|(_, c)| Reverse(c.len()))
+ .map(|(n, _)| n)
+ .take(1024)
.filter(|n| !nodes.contains(n))
.collect()
}
diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rs
index 3386ec31..a871de30 100644
--- a/crates/core/src/tokenizer/mod.rs
+++ b/crates/core/src/tokenizer/mod.rs
@@ -14,15 +14,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-mod add_space_last;
+pub mod add_space_last;
pub mod fields;
pub mod normalizer;
-mod script;
-mod script_tokenizer;
-mod segmenter;
-mod split_preserve;
-mod split_with_range;
-mod stemmer;
+pub mod script;
+pub mod script_tokenizer;
+pub mod segmenter;
+pub mod split_preserve;
+pub mod split_with_range;
+pub mod stemmer;
use std::borrow::{Borrow, Cow};
diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rs
index baaef16a..d09ae463 100644
--- a/crates/core/src/webgraph/schema.rs
+++ b/crates/core/src/webgraph/schema.rs
@@ -27,6 +27,7 @@ use crate::enum_dispatch_from_discriminant;
use crate::Result;
use super::document::{Edge, ReferenceValue};
+use super::tokenizer;
use super::tokenizer::Tokenizer;
use super::tokenizer::TokenizerEnum;
use super::Node;
@@ -64,6 +65,10 @@ impl Field for FromUrl {
"from_url"
}
+ fn tokenizer(&self) -> TokenizerEnum {
+ tokenizer::UrlTokenizer.into()
+ }
+
fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
ReferenceValue::Str(edge.from.as_str())
}
@@ -95,6 +100,10 @@ impl Field for ToUrl {
"to_url"
}
+ fn tokenizer(&self) -> TokenizerEnum {
+ tokenizer::UrlTokenizer.into()
+ }
+
fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
ReferenceValue::Str(edge.to.as_str())
}
diff --git a/crates/core/src/webgraph/tokenizer.rs b/crates/core/src/webgraph/tokenizer/mod.rs
similarity index 86%
rename from crates/core/src/webgraph/tokenizer.rs
rename to crates/core/src/webgraph/tokenizer/mod.rs
index 45d393d2..2d4f9e8c 100644
--- a/crates/core/src/webgraph/tokenizer.rs
+++ b/crates/core/src/webgraph/tokenizer/mod.rs
@@ -14,9 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod url;
+
use enum_dispatch::enum_dispatch;
use strum::{EnumDiscriminants, VariantArray};
use tantivy::tokenizer::{BoxTokenStream, Tokenizer as _};
+use url::UrlTokenStream;
#[enum_dispatch]
pub trait Tokenizer: Into<TokenizerEnum> + Clone {
@@ -41,11 +44,25 @@ impl Tokenizer for SimpleTokenizer {
}
}
+#[derive(Clone, Default)]
+pub struct UrlTokenizer;
+
+impl Tokenizer for UrlTokenizer {
+ fn name(&self) -> &'static str {
+ "url"
+ }
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
+ BoxTokenStream::new(UrlTokenStream::new(text))
+ }
+}
+
#[enum_dispatch(Tokenizer)]
#[derive(Clone, EnumDiscriminants)]
#[strum_discriminants(derive(VariantArray))]
pub enum TokenizerEnum {
SimpleTokenizer,
+ UrlTokenizer,
}
impl TokenizerEnum {
@@ -66,6 +83,7 @@ impl From<TokenizerEnumDiscriminants> for TokenizerEnum {
fn from(value: TokenizerEnumDiscriminants) -> Self {
match value {
TokenizerEnumDiscriminants::SimpleTokenizer => SimpleTokenizer::default().into(),
+ TokenizerEnumDiscriminants::UrlTokenizer => UrlTokenizer.into(),
}
}
}
diff --git a/crates/core/src/webgraph/tokenizer/url.rs b/crates/core/src/webgraph/tokenizer/url.rs
new file mode 100644
index 00000000..6ecf6884
--- /dev/null
+++ b/crates/core/src/webgraph/tokenizer/url.rs
@@ -0,0 +1,165 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::tokenizer::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};
+use crate::{webpage::url_ext::UrlExt, Result};
+use std::collections::VecDeque;
+
+use tantivy::tokenizer::{Token, TokenStream};
+
+#[derive(Default)]
+struct ParsedUrl {
+ scheme: VecDeque<String>,
+ host: VecDeque<String>,
+ path: VecDeque<String>,
+}
+
+impl ParsedUrl {
+ fn parse(url: &str) -> Result<Self> {
+ let url = url::Url::robust_parse(url)?;
+ let scheme: VecDeque<String> = url
+ .scheme()
+ .split_preserve(|c| c == ':')
+ .map(|s| {
+ let mut s = s.to_string();
+ s.push(':');
+ s
+ })
+ .add_space_last()
+ .collect();
+
+ let mut host: VecDeque<String> = url
+ .normalized_host()
+ .unwrap_or_default()
+ .split_preserve(|c| c == '.')
+ .map(|s| s.to_string())
+ .add_space_last()
+ .collect();
+
+ let mut path: VecDeque<_> = url
+ .path()
+ .split_preserve(|c| matches!(c, '/' | '-' | '_'))
+ .filter(|s| !(*s).is_empty())
+ .map(|s| s.to_string())
+ .collect();
+
+ if host.is_empty() {
+ if let Some(maybe_host) = path.pop_front() {
+ let host_from_path = match maybe_host.split_once('@') {
+ Some((rest, host)) => {
+ path.push_front(rest.to_string());
+ host.to_string()
+ }
+ None => maybe_host.to_string(),
+ };
+
+ host = host_from_path
+ .split_preserve(|c| c == '.')
+ .map(|s| s.to_string())
+ .add_space_last()
+ .collect();
+ }
+ }
+
+ Ok(Self { scheme, host, path })
+ }
+
+ pub fn next(&mut self) -> Option<String> {
+ self.scheme
+ .pop_front()
+ .or_else(|| self.host.pop_front())
+ .or_else(|| self.path.pop_front())
+ }
+}
+
+pub struct UrlTokenStream {
+ url: ParsedUrl,
+ token: Token,
+}
+
+impl UrlTokenStream {
+ pub fn new(text: &str) -> Self {
+ let url = ParsedUrl::parse(text).unwrap_or_default();
+
+ Self {
+ url,
+ token: Token::default(),
+ }
+ }
+}
+
+impl TokenStream for UrlTokenStream {
+ fn advance(&mut self) -> bool {
+ match self.url.next() {
+ Some(token) => {
+ self.token.position = self.token.position.wrapping_add(1);
+ self.token.offset_from = self.token.offset_to;
+ self.token.offset_to = self.token.offset_from + token.len();
+ self.token.text = token;
+
+ true
+ }
+ None => false,
+ }
+ }
+
+ fn token(&self) -> &Token {
+ &self.token
+ }
+
+ fn token_mut(&mut self) -> &mut Token {
+ &mut self.token
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn tokenize(url: &str) -> Vec<String> {
+ let mut url = UrlTokenStream::new(url);
+ let mut tokens = Vec::new();
+ while url.advance() {
+ tokens.push(url.token().text.clone());
+ }
+ tokens
+ }
+
+ #[test]
+ fn test_parse() {
+ assert_eq!(
+ tokenize("https://example.com/path/to/resource"),
+ ["https: ", "example", ".", "com ", "/", "path", "/", "to", "/", "resource"]
+ );
+ assert_eq!(
+ tokenize("example.com"),
+ ["https: ", "example", ".", "com ", "/"]
+ );
+ assert_eq!(tokenize(".com"), ["https: ", ".", "com ", "/"]);
+
+ assert_eq!(
+ tokenize("mailto:[email protected]"),
+ ["mailto: ", "example", ".", "com ", "hello"]
+ );
+
+ assert_eq!(
+ tokenize("mailto:example.com"),
+ ["mailto: ", "example", ".", "com "]
+ );
+
+ assert_eq!(tokenize("tel:+4512345678"), ["tel: ", "+4512345678 "]);
+ }
+}
diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rs
index 65fdb040..c1dc47ab 100644
--- a/crates/core/src/webpage/html/links.rs
+++ b/crates/core/src/webpage/html/links.rs
@@ -288,10 +288,6 @@ impl Html {
if &element.name.local == "a" {
if let Some((text, attributes)) = open_links.pop() {
if let Some(dest) = attributes.borrow().get("href") {
- if dest.starts_with("mailto:") || dest.starts_with("tel:") {
- continue;
- }
-
if let Ok(dest) =
Url::parse_with_base_url(self.base_url(), dest)
{
@@ -338,10 +334,6 @@ impl Html {
while let Some((text, attributes)) = open_links.pop() {
if let Some(dest) = attributes.borrow().get("href") {
- if dest.starts_with("mailto:") || dest.starts_with("tel:") {
- continue;
- }
-
if let Ok(dest) = Url::parse_with_base_url(self.base_url(), dest) {
let mut rel = RelFlags::from_html(&dest, &attributes.borrow(), &location);
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index c5a1e6f6..94f9d783 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -558,7 +558,6 @@ mod tests {
<body>
<a href="https://example.com">Link to example</a>
<p>{CONTENT}</p>
- <a href="mailto:[email protected]">Email me</a>
</body>
</html>
"#