Skip to content

web search done right

Make `site:` operator respect prefix matching in path and suffix matching in domain. This allows us to search for stuff like 'site:.com', 'site:.com/en', 'site:github.io' and 'site:example.github.io'. This essentially provides much finer grained control of the 'site:' operator.

Mikkel Denkerd96ff7aad991
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 8fc76aec..d1169ed4 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -45,7 +45,7 @@ use crate::ranking::SignalAggregator;
use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
use crate::search_ctx::Ctx;
use crate::snippet::TextSnippet;
-use crate::tokenizer::{BigramTokenizer, Identity, TrigramTokenizer};
+use crate::tokenizer::{BigramTokenizer, Identity, SiteOperatorUrlTokenizer, TrigramTokenizer};
use crate::webgraph::NodeID;
use crate::webpage::region::Region;
use crate::webpage::{schema_org, Webpage};
@@ -154,6 +154,11 @@ impl InvertedIndex {
.tokenizers()
.register(tokenizer.as_str(), tokenizer);
+ let tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);
+ tantivy_index
+ .tokenizers()
+ .register(tokenizer.as_str(), tokenizer);
+
let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;
let merge_policy = NoMergePolicy;
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 337512a5..e5439f26 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -144,7 +144,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Site => ConstQuery::new(
PatternQuery::new(
self.pattern.clone(),
- TextField::SiteWithout,
+ TextField::UrlForSiteOperator,
schema,
fastfield_reader.clone(),
)
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 5688f2eb..cebec5c9 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -92,6 +92,10 @@ impl Term {
{
let mut processed_terms = Term::process_tantivy_term(phrase, *field);
+ if processed_terms.is_empty() {
+ continue;
+ }
+
if processed_terms.len() == 1 {
let options = ALL_FIELDS[field.field_id() as usize]
.as_text()
@@ -192,11 +196,29 @@ impl Term {
.filter(|field| {
matches!(
ALL_FIELDS[field.field_id() as usize],
- Field::Text(TextField::DomainNoTokenizer)
- | Field::Text(TextField::SiteNoTokenizer)
+ Field::Text(TextField::UrlForSiteOperator)
)
})
- .map(|field| (Occur::Should, Term::tantivy_text_query(field, term)))
+ .map(|field| {
+ let processed_terms = Term::process_tantivy_term(term, *field);
+
+ if processed_terms.len() == 1 {
+ let term = processed_terms.get(0).unwrap().clone();
+ (
+ Occur::Should,
+ Box::new(TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ )) as Box<dyn tantivy::query::Query>,
+ )
+ } else {
+ (
+ Occur::Should,
+ Box::new(PhraseQuery::new(processed_terms))
+ as Box<dyn tantivy::query::Query>,
+ )
+ }
+ })
.collect()
}
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 9f91b9f2..88b3eefd 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -59,22 +59,27 @@ impl PatternQuery {
schema: &tantivy::schema::Schema,
fastfield_reader: FastFieldReader,
) -> Self {
- let field = Field::Text(field);
- let tv_field = schema.get_field(field.name()).unwrap();
+ let mut field = Field::Text(field);
+ let mut tv_field = schema.get_field(field.name()).unwrap();
if can_optimize_site_domain(&patterns, field) {
if patterns.len() == 3 {
- if let PatternPart::Raw(term) = &patterns[1] {
- return Self {
- patterns: Vec::new(),
- field: tv_field,
- can_optimize_site_domain: true,
- raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],
- fastfield_reader,
- };
- } else {
+ let PatternPart::Raw(term) = &patterns[1] else {
unreachable!()
+ };
+
+ if let Field::Text(TextField::UrlForSiteOperator) = field {
+ field = Field::Text(TextField::SiteWithout);
+ tv_field = schema.get_field(field.name()).unwrap();
}
+
+ return Self {
+ patterns: Vec::new(),
+ field: tv_field,
+ can_optimize_site_domain: true,
+ raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],
+ fastfield_reader,
+ };
} else {
let term: String = patterns
.iter()
@@ -324,7 +329,9 @@ impl PatternWeight {
Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),
Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),
Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
- Field::Text(TextField::SiteWithout) => Ok(FastField::NumSiteTokens),
+ Field::Text(TextField::UrlForSiteOperator) => {
+ Ok(FastField::NumUrlForSiteOperatorTokens)
+ }
Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
Field::Text(TextField::MicroformatTags) => Ok(FastField::NumMicroformatTagsTokens),
Field::Text(TextField::FlattenedSchemaOrgJson) => {
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 7dd2b511..7cf6baa9 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -18,7 +18,9 @@ use tantivy::schema::{
Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
};
-use crate::tokenizer::{BigramTokenizer, Identity, JsonField, Tokenizer, TrigramTokenizer};
+use crate::tokenizer::{
+ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,
+};
pub const FLOAT_SCALING: u64 = 1_000_000_000;
@@ -31,6 +33,7 @@ pub enum TextField {
AllBody,
Url,
UrlNoTokenizer,
+ UrlForSiteOperator,
SiteWithout,
Domain,
SiteNoTokenizer,
@@ -86,6 +89,7 @@ impl TextField {
TextField::AllBody => Tokenizer::default(),
TextField::Url => Tokenizer::default(),
TextField::UrlNoTokenizer => Tokenizer::Identity(Identity {}),
+ TextField::UrlForSiteOperator => Tokenizer::SiteOperator(SiteOperatorUrlTokenizer),
TextField::SiteWithout => Tokenizer::default(),
TextField::Domain => Tokenizer::default(),
TextField::SiteNoTokenizer => Tokenizer::Identity(Identity {}),
@@ -127,6 +131,7 @@ impl TextField {
TextField::AllBody => false,
TextField::Url => true,
TextField::UrlNoTokenizer => false,
+ TextField::UrlForSiteOperator => true,
TextField::SiteWithout => true,
TextField::Domain => true,
TextField::SiteNoTokenizer => false,
@@ -157,6 +162,7 @@ impl TextField {
TextField::CleanBody => "body",
TextField::Url => "url",
TextField::UrlNoTokenizer => "url_no_tokenizer",
+ TextField::UrlForSiteOperator => "url_for_site_operator",
TextField::SiteWithout => "site",
TextField::Domain => "domain",
TextField::SiteNoTokenizer => "site_no_tokenizer",
@@ -198,7 +204,7 @@ pub enum FastField {
NumTitleTokens,
NumCleanBodyTokens,
NumDescriptionTokens,
- NumSiteTokens,
+ NumUrlForSiteOperatorTokens,
NumDomainTokens,
NumMicroformatTagsTokens,
SiteHash1,
@@ -235,7 +241,7 @@ impl FastField {
FastField::NumCleanBodyTokens => "num_clean_body_tokens",
FastField::NumDescriptionTokens => "num_description_tokens",
FastField::NumDomainTokens => "num_domain_tokens",
- FastField::NumSiteTokens => "num_site_tokens",
+ FastField::NumUrlForSiteOperatorTokens => "num_url_for_site_operator_tokens",
FastField::NumFlattenedSchemaTokens => "num_flattened_schema_tokens",
FastField::NumMicroformatTagsTokens => "num_microformat_tags_tokens",
FastField::SiteHash1 => "site_hash1",
@@ -270,7 +276,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 59] = [
+pub static ALL_FIELDS: [Field; 60] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -278,6 +284,7 @@ pub static ALL_FIELDS: [Field; 59] = [
Field::Text(TextField::AllBody),
Field::Text(TextField::Url),
Field::Text(TextField::UrlNoTokenizer),
+ Field::Text(TextField::UrlForSiteOperator),
Field::Text(TextField::SiteWithout),
Field::Text(TextField::Domain),
Field::Text(TextField::SiteNoTokenizer),
@@ -312,7 +319,7 @@ pub static ALL_FIELDS: [Field; 59] = [
Field::Fast(FastField::NumCleanBodyTokens),
Field::Fast(FastField::NumDescriptionTokens),
Field::Fast(FastField::NumDomainTokens),
- Field::Fast(FastField::NumSiteTokens),
+ Field::Fast(FastField::NumUrlForSiteOperatorTokens),
Field::Fast(FastField::NumFlattenedSchemaTokens),
Field::Fast(FastField::NumMicroformatTagsTokens),
Field::Fast(FastField::SiteHash1),
@@ -364,6 +371,9 @@ impl Field {
Field::Text(TextField::UrlNoTokenizer) => {
IndexingOption::Text(self.default_text_options())
}
+ Field::Text(TextField::UrlForSiteOperator) => {
+ IndexingOption::Text(self.default_text_options())
+ }
Field::Text(TextField::SiteWithout) => {
IndexingOption::Text(self.default_text_options())
}
@@ -499,7 +509,7 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::Fast(FastField::NumSiteTokens) => IndexingOption::Integer(
+ Field::Fast(FastField::NumUrlForSiteOperatorTokens) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
@@ -598,6 +608,7 @@ impl Field {
| Field::Text(TextField::MicroformatTags)
| Field::Text(TextField::SafetyClassification)
| Field::Text(TextField::FlattenedSchemaOrgJson)
+ | Field::Text(TextField::UrlForSiteOperator)
) && !self.is_fast()
}
@@ -661,7 +672,7 @@ impl FastField {
FastField::NumCleanBodyTokens => DataType::U64,
FastField::NumDescriptionTokens => DataType::U64,
FastField::NumDomainTokens => DataType::U64,
- FastField::NumSiteTokens => DataType::U64,
+ FastField::NumUrlForSiteOperatorTokens => DataType::U64,
FastField::NumFlattenedSchemaTokens => DataType::U64,
FastField::SiteHash1 => DataType::U64,
FastField::SiteHash2 => DataType::U64,
diff --git a/core/src/tokenizer/add_space_last.rs b/core/src/tokenizer/add_space_last.rs
new file mode 100644
index 00000000..97a4e499
--- /dev/null
+++ b/core/src/tokenizer/add_space_last.rs
@@ -0,0 +1,61 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::iter::Peekable;
+
+pub struct AddSpaceToLast<I: Iterator<Item = String>> {
+ inner: Peekable<I>,
+ last_item: Option<String>,
+}
+
+impl<I: Iterator<Item = String>> AddSpaceToLast<I> {
+ fn new(inner: I) -> Self {
+ Self {
+ inner: inner.peekable(),
+ last_item: None,
+ }
+ }
+}
+
+impl<I: Iterator<Item = String>> Iterator for AddSpaceToLast<I> {
+ type Item = String;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if let Some(last_item) = self.last_item.take() {
+ return Some(last_item);
+ }
+
+ match self.inner.next() {
+ Some(mut item) => {
+ if self.inner.peek().is_none() {
+ item.push(' ');
+ }
+
+ self.last_item = Some(item);
+ self.next()
+ }
+ None => None,
+ }
+ }
+}
+
+pub trait AddSpaceLast: Iterator<Item = String> + Sized {
+ fn add_space_last(self) -> AddSpaceToLast<Self> {
+ AddSpaceToLast::new(self)
+ }
+}
+
+impl<I: Iterator<Item = String>> AddSpaceLast for I {}
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer/mod.rs
similarity index 81%
rename from core/src/tokenizer.rs
rename to core/src/tokenizer/mod.rs
index ce65dfcc..051430c9 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{array, str::CharIndices};
+use std::{array, collections::VecDeque, str::CharIndices};
use logos::{Lexer, Logos};
use tantivy::tokenizer::{
@@ -25,6 +25,11 @@ use whatlang::Lang;
use crate::{ceil_char_boundary, floor_char_boundary};
+use self::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};
+
+mod add_space_last;
+mod split_preserve;
+
struct MyStemmer(Stemmer);
impl From<Lang> for MyStemmer {
@@ -58,6 +63,7 @@ pub enum Tokenizer {
Bigram(BigramTokenizer),
Trigram(TrigramTokenizer),
Json(JsonField),
+ SiteOperator(SiteOperatorUrlTokenizer),
}
impl Tokenizer {
@@ -73,6 +79,7 @@ impl Tokenizer {
Tokenizer::Bigram(_) => BigramTokenizer::as_str(),
Tokenizer::Trigram(_) => TrigramTokenizer::as_str(),
Tokenizer::Json(_) => JsonField::as_str(),
+ Tokenizer::SiteOperator(_) => SiteOperatorUrlTokenizer::as_str(),
}
}
}
@@ -152,6 +159,7 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {
Tokenizer::Json(tokenizer) => tokenizer.token_stream(text),
Tokenizer::Bigram(tokenizer) => tokenizer.token_stream(text),
Tokenizer::Trigram(tokenizer) => tokenizer.token_stream(text),
+ Tokenizer::SiteOperator(tokenizer) => tokenizer.token_stream(text),
}
}
}
@@ -553,6 +561,131 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {
}
}
+#[derive(Clone, Default)]
+struct ParsedUrl {
+ protocol: Option<VecDeque<String>>,
+ domain: Option<VecDeque<String>>,
+ path: VecDeque<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SiteOperatorUrlTokenizer;
+
+impl SiteOperatorUrlTokenizer {
+ pub fn as_str() -> &'static str {
+ "site_operator_url_tokenizer"
+ }
+}
+
+impl tantivy::tokenizer::Tokenizer for SiteOperatorUrlTokenizer {
+ fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+ let parsed_url = url::Url::parse(text)
+ .or_else(|_| url::Url::parse(&format!("http://{}", text)))
+ .map(|url| {
+ let domain = Some(
+ url.host_str()
+ .unwrap_or("")
+ .split_preserve(|c| matches!(c, '.'))
+ .filter(|s| !(*s).is_empty())
+ .map(|s| s.to_string())
+ .add_space_last()
+ .collect(),
+ );
+ let path: VecDeque<_> = url
+ .path()
+ .split_preserve(|c| matches!(c, '/' | '-' | '_'))
+ .filter(|s| !(*s).is_empty())
+ .map(|s| s.to_string())
+ .collect();
+
+ if matches!(url.scheme(), "http" | "https") {
+ ParsedUrl {
+ protocol: None,
+ domain,
+ path,
+ }
+ } else {
+ let mut v = VecDeque::new();
+ v.push_back(url.scheme().to_string());
+
+ ParsedUrl {
+ protocol: Some(v),
+ domain,
+ path,
+ }
+ }
+ })
+ .unwrap_or_default();
+
+ BoxTokenStream::from(SiteOperatorUrlTokenStream {
+ url: parsed_url,
+ token: tantivy::tokenizer::Token::default(),
+ })
+ }
+}
+
+pub struct SiteOperatorUrlTokenStream {
+ url: ParsedUrl,
+ token: tantivy::tokenizer::Token,
+}
+
+impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {
+ fn advance(&mut self) -> bool {
+ if let Some(protocol) = self.url.protocol.as_mut() {
+ self.token.position = self.token.position.wrapping_add(1);
+ self.token.text.clear();
+
+ if let Some(s) = protocol.pop_front() {
+ self.token.text.push_str(&s);
+ self.token.offset_from = 0;
+ self.token.offset_to = s.len();
+ } else {
+ self.token.offset_from = self.token.offset_to;
+ self.token.text.push_str("://");
+ self.token.offset_to += self.token.text.len();
+
+ self.url.protocol = None;
+ }
+
+ return true;
+ }
+
+ if let Some(domain) = self.url.domain.as_mut() {
+ if let Some(s) = domain.pop_front() {
+ self.token.text.clear();
+ self.token.position = self.token.position.wrapping_add(1);
+
+ self.token.text.push_str(&s);
+
+ self.token.offset_from = self.token.offset_to;
+ self.token.offset_to += self.token.text.len();
+ return true;
+ }
+ }
+
+ if let Some(s) = self.url.path.pop_front() {
+ self.token.text.clear();
+ self.token.position = self.token.position.wrapping_add(1);
+
+ self.token.text.push_str(&s);
+ self.token.offset_from = self.token.offset_to;
+ self.token.offset_to += self.token.text.len();
+
+ return true;
+ }
+
+ false
+ }
+
+ fn token(&self) -> &tantivy::tokenizer::Token {
+ &self.token
+ }
+
+ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
+ &mut self.token
+ }
+}
+
#[cfg(test)]
mod tests {
use tantivy::tokenizer::Tokenizer as _;
@@ -603,6 +736,17 @@ mod tests {
res
}
+ fn tokenize_url(s: &str) -> Vec<String> {
+ let mut res = Vec::new();
+ let mut stream = SiteOperatorUrlTokenizer.token_stream(s);
+
+ while let Some(token) = stream.next() {
+ res.push(token.text.clone());
+ }
+
+ res
+ }
+
#[test]
fn simple_tokenization() {
assert_eq!(
@@ -904,4 +1048,54 @@ key1.key2="this\" is @ a # test""#;
]
);
}
+
+ #[test]
+ fn url() {
+ assert_eq!(
+ tokenize_url("https://www.example.com"),
+ vec![
+ "www".to_string(),
+ ".".to_string(),
+ "example".to_string(),
+ ".".to_string(),
+ "com ".to_string(),
+ ]
+ );
+
+ assert_eq!(
+ tokenize_url("https://www.example.com/test"),
+ vec![
+ "www".to_string(),
+ ".".to_string(),
+ "example".to_string(),
+ ".".to_string(),
+ "com ".to_string(),
+ "/".to_string(),
+ "test".to_string(),
+ ]
+ );
+
+ assert_eq!(
+ tokenize_url("example.com"),
+ vec!["example".to_string(), ".".to_string(), "com ".to_string(),]
+ );
+
+ assert_eq!(
+ tokenize_url("example.com/another/path"),
+ vec![
+ "example".to_string(),
+ ".".to_string(),
+ "com ".to_string(),
+ "/".to_string(),
+ "another".to_string(),
+ "/".to_string(),
+ "path".to_string(),
+ ]
+ );
+
+ assert_eq!(
+ tokenize_url(".com"),
+ vec![".".to_string(), "com ".to_string(),]
+ )
+ }
}
diff --git a/core/src/tokenizer/split_preserve.rs b/core/src/tokenizer/split_preserve.rs
new file mode 100644
index 00000000..f0bb4afc
--- /dev/null
+++ b/core/src/tokenizer/split_preserve.rs
@@ -0,0 +1,118 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub struct SplitPreserve<'a, P>
+where
+ P: Fn(char) -> bool,
+{
+ s: &'a str,
+ pred: P,
+ start: usize,
+ last_pred: Option<char>,
+}
+
+impl<'a, P> SplitPreserve<'a, P>
+where
+ P: Fn(char) -> bool,
+{
+ fn new(s: &'a str, pred: P) -> Self {
+ Self {
+ s,
+ pred,
+ start: 0,
+ last_pred: None,
+ }
+ }
+}
+
+impl<'a, P> Iterator for SplitPreserve<'a, P>
+where
+ P: Fn(char) -> bool,
+{
+ type Item = &'a str;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.start >= self.s.len() {
+ return None;
+ }
+
+ if let Some(c) = self.last_pred.take() {
+ return Some(&self.s[self.start - c.len_utf8()..self.start]);
+ }
+
+ for (i, c) in self.s[self.start..].char_indices() {
+ if (self.pred)(c) {
+ self.last_pred = Some(c);
+
+ let res = &self.s[self.start..self.start + i];
+
+ self.start += i + c.len_utf8();
+
+ return Some(res);
+ }
+ }
+
+ if self.start < self.s.len() {
+ let res = &self.s[self.start..];
+
+ self.start = self.s.len();
+
+ Some(res)
+ } else {
+ None
+ }
+ }
+}
+
+pub trait StrSplitPreserve {
+ fn split_preserve<F>(&self, pred: F) -> SplitPreserve<F>
+ where
+ F: Fn(char) -> bool;
+}
+
+impl StrSplitPreserve for str {
+ fn split_preserve<F>(&self, pred: F) -> SplitPreserve<F>
+ where
+ F: Fn(char) -> bool,
+ {
+ SplitPreserve::new(self, pred)
+ }
+}
+
+impl StrSplitPreserve for String {
+ fn split_preserve<F>(&self, pred: F) -> SplitPreserve<F>
+ where
+ F: Fn(char) -> bool,
+ {
+ SplitPreserve::new(self, pred)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple() {
+ let res = "hello.brave.new.world"
+ .split_preserve(|c| c == '.')
+ .collect::<Vec<_>>();
+ assert_eq!(res, vec!["hello", ".", "brave", ".", "new", ".", "world"]);
+
+ let res = "hello".split_preserve(|c| c == '.').collect::<Vec<_>>();
+ assert_eq!(res, vec!["hello"]);
+ }
+}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 039049ef..6cab2121 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -907,10 +907,18 @@ impl Html {
}
fn pretokenize_string(&self, text: String) -> PreTokenizedString {
+ self.pretokenize_string_with(text, tokenizer::Tokenizer::default())
+ }
+
+ fn pretokenize_string_with(
+ &self,
+ text: String,
+ tokenizer: tokenizer::Tokenizer,
+ ) -> PreTokenizedString {
let mut tokens = Vec::new();
{
- let mut stream = tokenizer::Normal::default().token_stream(&text);
+ let mut stream = tokenizer.token_stream(&text);
while let Some(token) = stream.next() {
tokens.push(token.clone());
}
@@ -930,6 +938,10 @@ impl Html {
let site = self.pretokenize_site();
let description = self.pretokenize_description();
let microformats = self.pretokenize_microformats();
+ let url_for_site_operator = self.pretokenize_string_with(
+ self.url().to_string(),
+ tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),
+ );
let domain_name = self
.url()
@@ -1038,6 +1050,9 @@ impl Html {
Field::Text(TextField::Url) => {
doc.add_pre_tokenized_text(tantivy_field, url.clone())
}
+ Field::Text(TextField::UrlForSiteOperator) => {
+ doc.add_pre_tokenized_text(tantivy_field, url_for_site_operator.clone())
+ }
Field::Text(TextField::UrlNoTokenizer) => {
let url = self.url().to_string();
@@ -1199,8 +1214,8 @@ impl Html {
Field::Fast(FastField::NumDescriptionTokens) => {
doc.add_u64(tantivy_field, description.tokens.len() as u64)
}
- Field::Fast(FastField::NumSiteTokens) => {
- doc.add_u64(tantivy_field, site.tokens.len() as u64)
+ Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {
+ doc.add_u64(tantivy_field, url_for_site_operator.tokens.len() as u64)
}
Field::Fast(FastField::NumDomainTokens) => {
doc.add_u64(tantivy_field, domain.tokens.len() as u64)