web search done right
Make `site:` operator respect prefix matching in path and suffix matching in domain. This allows us to search for stuff like 'site:.com', 'site:.com/en', 'site:github.io' and 'site:example.github.io'. This essentially provides much finer grained control of the 'site:' operator.
Mikkel Denkerd96ff7aad991
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 8fc76aec..d1169ed4 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -45,7 +45,7 @@ use crate::ranking::SignalAggregator;use crate::schema::{FastField, Field, TextField, ALL_FIELDS};use crate::search_ctx::Ctx;use crate::snippet::TextSnippet;-use crate::tokenizer::{BigramTokenizer, Identity, TrigramTokenizer};+use crate::tokenizer::{BigramTokenizer, Identity, SiteOperatorUrlTokenizer, TrigramTokenizer};use crate::webgraph::NodeID;use crate::webpage::region::Region;use crate::webpage::{schema_org, Webpage};@@ -154,6 +154,11 @@ impl InvertedIndex {.tokenizers().register(tokenizer.as_str(), tokenizer);+ let tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);+ tantivy_index+ .tokenizers()+ .register(tokenizer.as_str(), tokenizer);+let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;let merge_policy = NoMergePolicy;diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 337512a5..e5439f26 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -144,7 +144,7 @@ impl AsTantivyQuery for Matching {MatchLocation::Site => ConstQuery::new(PatternQuery::new(self.pattern.clone(),- TextField::SiteWithout,+ TextField::UrlForSiteOperator,schema,fastfield_reader.clone(),)diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 5688f2eb..cebec5c9 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -92,6 +92,10 @@ impl Term {{let mut processed_terms = Term::process_tantivy_term(phrase, *field);+ if processed_terms.is_empty() {+ continue;+ }+if processed_terms.len() == 1 {let options = ALL_FIELDS[field.field_id() as usize].as_text()@@ -192,11 +196,29 @@ impl Term {.filter(|field| {matches!(ALL_FIELDS[field.field_id() as usize],- Field::Text(TextField::DomainNoTokenizer)- | Field::Text(TextField::SiteNoTokenizer)+ Field::Text(TextField::UrlForSiteOperator))})- .map(|field| (Occur::Should, Term::tantivy_text_query(field, term)))+ .map(|field| {+ let processed_terms = Term::process_tantivy_term(term, *field);++ if processed_terms.len() == 1 {+ let term = processed_terms.get(0).unwrap().clone();+ (+ Occur::Should,+ Box::new(TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ )) as Box<dyn tantivy::query::Query>,+ )+ } else {+ (+ Occur::Should,+ Box::new(PhraseQuery::new(processed_terms))+ as Box<dyn tantivy::query::Query>,+ )+ }+ }).collect()}diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex 9f91b9f2..88b3eefd 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -59,22 +59,27 @@ impl PatternQuery {schema: &tantivy::schema::Schema,fastfield_reader: FastFieldReader,) -> Self {- let field = Field::Text(field);- let tv_field = schema.get_field(field.name()).unwrap();+ let mut field = Field::Text(field);+ let mut tv_field = schema.get_field(field.name()).unwrap();if can_optimize_site_domain(&patterns, field) {if patterns.len() == 3 {- if let PatternPart::Raw(term) = &patterns[1] {- return Self {- patterns: Vec::new(),- field: tv_field,- can_optimize_site_domain: true,- raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],- fastfield_reader,- };- } else {+ let PatternPart::Raw(term) = &patterns[1] else {unreachable!()+ };++ if let Field::Text(TextField::UrlForSiteOperator) = field {+ field = Field::Text(TextField::SiteWithout);+ tv_field = schema.get_field(field.name()).unwrap();}++ return Self {+ patterns: Vec::new(),+ field: tv_field,+ can_optimize_site_domain: true,+ raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],+ fastfield_reader,+ };} else {let term: String = patterns.iter()@@ -324,7 +329,9 @@ impl PatternWeight {Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),- Field::Text(TextField::SiteWithout) => Ok(FastField::NumSiteTokens),+ Field::Text(TextField::UrlForSiteOperator) => {+ Ok(FastField::NumUrlForSiteOperatorTokens)+ }Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),Field::Text(TextField::MicroformatTags) => Ok(FastField::NumMicroformatTagsTokens),Field::Text(TextField::FlattenedSchemaOrgJson) => {diff --git a/core/src/schema.rs b/core/src/schema.rsindex 7dd2b511..7cf6baa9 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -18,7 +18,9 @@ use tantivy::schema::{Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};-use crate::tokenizer::{BigramTokenizer, Identity, JsonField, Tokenizer, TrigramTokenizer};+use crate::tokenizer::{+ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,+};pub const FLOAT_SCALING: u64 = 1_000_000_000;@@ -31,6 +33,7 @@ pub enum TextField {AllBody,Url,UrlNoTokenizer,+ UrlForSiteOperator,SiteWithout,Domain,SiteNoTokenizer,@@ -86,6 +89,7 @@ impl TextField {TextField::AllBody => Tokenizer::default(),TextField::Url => Tokenizer::default(),TextField::UrlNoTokenizer => Tokenizer::Identity(Identity {}),+ TextField::UrlForSiteOperator => Tokenizer::SiteOperator(SiteOperatorUrlTokenizer),TextField::SiteWithout => Tokenizer::default(),TextField::Domain => Tokenizer::default(),TextField::SiteNoTokenizer => Tokenizer::Identity(Identity {}),@@ -127,6 +131,7 @@ impl TextField {TextField::AllBody => false,TextField::Url => true,TextField::UrlNoTokenizer => false,+ TextField::UrlForSiteOperator => true,TextField::SiteWithout => true,TextField::Domain => true,TextField::SiteNoTokenizer => false,@@ -157,6 +162,7 @@ impl TextField {TextField::CleanBody => "body",TextField::Url => "url",TextField::UrlNoTokenizer => "url_no_tokenizer",+ TextField::UrlForSiteOperator => "url_for_site_operator",TextField::SiteWithout => "site",TextField::Domain => "domain",TextField::SiteNoTokenizer => "site_no_tokenizer",@@ -198,7 +204,7 @@ pub enum FastField {NumTitleTokens,NumCleanBodyTokens,NumDescriptionTokens,- NumSiteTokens,+ NumUrlForSiteOperatorTokens,NumDomainTokens,NumMicroformatTagsTokens,SiteHash1,@@ -235,7 +241,7 @@ impl FastField {FastField::NumCleanBodyTokens => "num_clean_body_tokens",FastField::NumDescriptionTokens => "num_description_tokens",FastField::NumDomainTokens => "num_domain_tokens",- FastField::NumSiteTokens => "num_site_tokens",+ FastField::NumUrlForSiteOperatorTokens => "num_url_for_site_operator_tokens",FastField::NumFlattenedSchemaTokens => "num_flattened_schema_tokens",FastField::NumMicroformatTagsTokens => "num_microformat_tags_tokens",FastField::SiteHash1 => "site_hash1",@@ -270,7 +276,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 59] = [+pub static ALL_FIELDS: [Field; 60] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -278,6 +284,7 @@ pub static ALL_FIELDS: [Field; 59] = [Field::Text(TextField::AllBody),Field::Text(TextField::Url),Field::Text(TextField::UrlNoTokenizer),+ Field::Text(TextField::UrlForSiteOperator),Field::Text(TextField::SiteWithout),Field::Text(TextField::Domain),Field::Text(TextField::SiteNoTokenizer),@@ -312,7 +319,7 @@ pub static ALL_FIELDS: [Field; 59] = [Field::Fast(FastField::NumCleanBodyTokens),Field::Fast(FastField::NumDescriptionTokens),Field::Fast(FastField::NumDomainTokens),- Field::Fast(FastField::NumSiteTokens),+ Field::Fast(FastField::NumUrlForSiteOperatorTokens),Field::Fast(FastField::NumFlattenedSchemaTokens),Field::Fast(FastField::NumMicroformatTagsTokens),Field::Fast(FastField::SiteHash1),@@ -364,6 +371,9 @@ impl Field {Field::Text(TextField::UrlNoTokenizer) => {IndexingOption::Text(self.default_text_options())}+ Field::Text(TextField::UrlForSiteOperator) => {+ IndexingOption::Text(self.default_text_options())+ }Field::Text(TextField::SiteWithout) => {IndexingOption::Text(self.default_text_options())}@@ -499,7 +509,7 @@ impl Field {.set_fast(Cardinality::SingleValue).set_indexed(),),- Field::Fast(FastField::NumSiteTokens) => IndexingOption::Integer(+ Field::Fast(FastField::NumUrlForSiteOperatorTokens) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),@@ -598,6 +608,7 @@ impl Field {| Field::Text(TextField::MicroformatTags)| Field::Text(TextField::SafetyClassification)| Field::Text(TextField::FlattenedSchemaOrgJson)+ | Field::Text(TextField::UrlForSiteOperator)) && !self.is_fast()}@@ -661,7 +672,7 @@ impl FastField {FastField::NumCleanBodyTokens => DataType::U64,FastField::NumDescriptionTokens => DataType::U64,FastField::NumDomainTokens => DataType::U64,- FastField::NumSiteTokens => DataType::U64,+ FastField::NumUrlForSiteOperatorTokens => DataType::U64,FastField::NumFlattenedSchemaTokens => DataType::U64,FastField::SiteHash1 => DataType::U64,FastField::SiteHash2 => DataType::U64,diff --git a/core/src/tokenizer/add_space_last.rs b/core/src/tokenizer/add_space_last.rsnew file mode 100644index 00000000..97a4e499--- /dev/null+++ b/core/src/tokenizer/add_space_last.rs@@ -0,0 +1,61 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::iter::Peekable;++pub struct AddSpaceToLast<I: Iterator<Item = String>> {+ inner: Peekable<I>,+ last_item: Option<String>,+}++impl<I: Iterator<Item = String>> AddSpaceToLast<I> {+ fn new(inner: I) -> Self {+ Self {+ inner: inner.peekable(),+ last_item: None,+ }+ }+}++impl<I: Iterator<Item = String>> Iterator for AddSpaceToLast<I> {+ type Item = String;++ fn next(&mut self) -> Option<Self::Item> {+ if let Some(last_item) = self.last_item.take() {+ return Some(last_item);+ }++ match self.inner.next() {+ Some(mut item) => {+ if self.inner.peek().is_none() {+ item.push(' ');+ }++ self.last_item = Some(item);+ self.next()+ }+ None => None,+ }+ }+}++pub trait AddSpaceLast: Iterator<Item = String> + Sized {+ fn add_space_last(self) -> AddSpaceToLast<Self> {+ AddSpaceToLast::new(self)+ }+}++impl<I: Iterator<Item = String>> AddSpaceLast for I {}diff --git a/core/src/tokenizer.rs b/core/src/tokenizer/mod.rssimilarity index 81%rename from core/src/tokenizer.rsrename to core/src/tokenizer/mod.rsindex ce65dfcc..051430c9 100644--- a/core/src/tokenizer.rs+++ b/core/src/tokenizer/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{array, str::CharIndices};+use std::{array, collections::VecDeque, str::CharIndices};use logos::{Lexer, Logos};use tantivy::tokenizer::{@@ -25,6 +25,11 @@ use whatlang::Lang;use crate::{ceil_char_boundary, floor_char_boundary};+use self::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};++mod add_space_last;+mod split_preserve;+struct MyStemmer(Stemmer);impl From<Lang> for MyStemmer {@@ -58,6 +63,7 @@ pub enum Tokenizer {Bigram(BigramTokenizer),Trigram(TrigramTokenizer),Json(JsonField),+ SiteOperator(SiteOperatorUrlTokenizer),}impl Tokenizer {@@ -73,6 +79,7 @@ impl Tokenizer {Tokenizer::Bigram(_) => BigramTokenizer::as_str(),Tokenizer::Trigram(_) => TrigramTokenizer::as_str(),Tokenizer::Json(_) => JsonField::as_str(),+ Tokenizer::SiteOperator(_) => SiteOperatorUrlTokenizer::as_str(),}}}@@ -152,6 +159,7 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {Tokenizer::Json(tokenizer) => tokenizer.token_stream(text),Tokenizer::Bigram(tokenizer) => tokenizer.token_stream(text),Tokenizer::Trigram(tokenizer) => tokenizer.token_stream(text),+ Tokenizer::SiteOperator(tokenizer) => tokenizer.token_stream(text),}}}@@ -553,6 +561,131 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {}}+#[derive(Clone, Default)]+struct ParsedUrl {+ protocol: Option<VecDeque<String>>,+ domain: Option<VecDeque<String>>,+ path: VecDeque<String>,+}++#[derive(Debug, Clone)]+pub struct SiteOperatorUrlTokenizer;++impl SiteOperatorUrlTokenizer {+ pub fn as_str() -> &'static str {+ "site_operator_url_tokenizer"+ }+}++impl tantivy::tokenizer::Tokenizer for SiteOperatorUrlTokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {+ let parsed_url = url::Url::parse(text)+ .or_else(|_| url::Url::parse(&format!("http://{}", text)))+ .map(|url| {+ let domain = Some(+ url.host_str()+ .unwrap_or("")+ .split_preserve(|c| matches!(c, '.'))+ .filter(|s| !(*s).is_empty())+ .map(|s| s.to_string())+ .add_space_last()+ .collect(),+ );+ let path: VecDeque<_> = url+ .path()+ .split_preserve(|c| matches!(c, '/' | '-' | '_'))+ .filter(|s| !(*s).is_empty())+ .map(|s| s.to_string())+ .collect();++ if matches!(url.scheme(), "http" | "https") {+ ParsedUrl {+ protocol: None,+ domain,+ path,+ }+ } else {+ let mut v = VecDeque::new();+ v.push_back(url.scheme().to_string());++ ParsedUrl {+ protocol: Some(v),+ domain,+ path,+ }+ }+ })+ .unwrap_or_default();++ BoxTokenStream::from(SiteOperatorUrlTokenStream {+ url: parsed_url,+ token: tantivy::tokenizer::Token::default(),+ })+ }+}++pub struct SiteOperatorUrlTokenStream {+ url: ParsedUrl,+ token: tantivy::tokenizer::Token,+}++impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {+ fn advance(&mut self) -> bool {+ if let Some(protocol) = self.url.protocol.as_mut() {+ self.token.position = self.token.position.wrapping_add(1);+ self.token.text.clear();++ if let Some(s) = protocol.pop_front() {+ self.token.text.push_str(&s);+ self.token.offset_from = 0;+ self.token.offset_to = s.len();+ } else {+ self.token.offset_from = self.token.offset_to;+ self.token.text.push_str("://");+ self.token.offset_to += self.token.text.len();++ self.url.protocol = None;+ }++ return true;+ }++ if let Some(domain) = self.url.domain.as_mut() {+ if let Some(s) = domain.pop_front() {+ self.token.text.clear();+ self.token.position = self.token.position.wrapping_add(1);++ self.token.text.push_str(&s);++ self.token.offset_from = self.token.offset_to;+ self.token.offset_to += self.token.text.len();+ return true;+ }+ }++ if let Some(s) = self.url.path.pop_front() {+ self.token.text.clear();+ self.token.position = self.token.position.wrapping_add(1);++ self.token.text.push_str(&s);+ self.token.offset_from = self.token.offset_to;+ self.token.offset_to += self.token.text.len();++ return true;+ }++ false+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ &mut self.token+ }+}+#[cfg(test)]mod tests {use tantivy::tokenizer::Tokenizer as _;@@ -603,6 +736,17 @@ mod tests {res}+ fn tokenize_url(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut stream = SiteOperatorUrlTokenizer.token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }+#[test]fn simple_tokenization() {assert_eq!(@@ -904,4 +1048,54 @@ key1.key2="this\" is @ a # test""#;]);}++ #[test]+ fn url() {+ assert_eq!(+ tokenize_url("https://www.example.com"),+ vec![+ "www".to_string(),+ ".".to_string(),+ "example".to_string(),+ ".".to_string(),+ "com ".to_string(),+ ]+ );++ assert_eq!(+ tokenize_url("https://www.example.com/test"),+ vec![+ "www".to_string(),+ ".".to_string(),+ "example".to_string(),+ ".".to_string(),+ "com ".to_string(),+ "/".to_string(),+ "test".to_string(),+ ]+ );++ assert_eq!(+ tokenize_url("example.com"),+ vec!["example".to_string(), ".".to_string(), "com ".to_string(),]+ );++ assert_eq!(+ tokenize_url("example.com/another/path"),+ vec![+ "example".to_string(),+ ".".to_string(),+ "com ".to_string(),+ "/".to_string(),+ "another".to_string(),+ "/".to_string(),+ "path".to_string(),+ ]+ );++ assert_eq!(+ tokenize_url(".com"),+ vec![".".to_string(), "com ".to_string(),]+ )+ }}diff --git a/core/src/tokenizer/split_preserve.rs b/core/src/tokenizer/split_preserve.rsnew file mode 100644index 00000000..f0bb4afc--- /dev/null+++ b/core/src/tokenizer/split_preserve.rs@@ -0,0 +1,118 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub struct SplitPreserve<'a, P>+where+ P: Fn(char) -> bool,+{+ s: &'a str,+ pred: P,+ start: usize,+ last_pred: Option<char>,+}++impl<'a, P> SplitPreserve<'a, P>+where+ P: Fn(char) -> bool,+{+ fn new(s: &'a str, pred: P) -> Self {+ Self {+ s,+ pred,+ start: 0,+ last_pred: None,+ }+ }+}++impl<'a, P> Iterator for SplitPreserve<'a, P>+where+ P: Fn(char) -> bool,+{+ type Item = &'a str;++ fn next(&mut self) -> Option<Self::Item> {+ if self.start >= self.s.len() {+ return None;+ }++ if let Some(c) = self.last_pred.take() {+ return Some(&self.s[self.start - c.len_utf8()..self.start]);+ }++ for (i, c) in self.s[self.start..].char_indices() {+ if (self.pred)(c) {+ self.last_pred = Some(c);++ let res = &self.s[self.start..self.start + i];++ self.start += i + c.len_utf8();++ return Some(res);+ }+ }++ if self.start < self.s.len() {+ let res = &self.s[self.start..];++ self.start = self.s.len();++ Some(res)+ } else {+ None+ }+ }+}++pub trait StrSplitPreserve {+ fn split_preserve<F>(&self, pred: F) -> SplitPreserve<F>+ where+ F: Fn(char) -> bool;+}++impl StrSplitPreserve for str {+ fn split_preserve<F>(&self, pred: F) -> SplitPreserve<F>+ where+ F: Fn(char) -> bool,+ {+ SplitPreserve::new(self, pred)+ }+}++impl StrSplitPreserve for String {+ fn split_preserve<F>(&self, pred: F) -> SplitPreserve<F>+ where+ F: Fn(char) -> bool,+ {+ SplitPreserve::new(self, pred)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple() {+ let res = "hello.brave.new.world"+ .split_preserve(|c| c == '.')+ .collect::<Vec<_>>();+ assert_eq!(res, vec!["hello", ".", "brave", ".", "new", ".", "world"]);++ let res = "hello".split_preserve(|c| c == '.').collect::<Vec<_>>();+ assert_eq!(res, vec!["hello"]);+ }+}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 039049ef..6cab2121 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -907,10 +907,18 @@ impl Html {}fn pretokenize_string(&self, text: String) -> PreTokenizedString {+ self.pretokenize_string_with(text, tokenizer::Tokenizer::default())+ }++ fn pretokenize_string_with(+ &self,+ text: String,+ tokenizer: tokenizer::Tokenizer,+ ) -> PreTokenizedString {let mut tokens = Vec::new();{- let mut stream = tokenizer::Normal::default().token_stream(&text);+ let mut stream = tokenizer.token_stream(&text);while let Some(token) = stream.next() {tokens.push(token.clone());}@@ -930,6 +938,10 @@ impl Html {let site = self.pretokenize_site();let description = self.pretokenize_description();let microformats = self.pretokenize_microformats();+ let url_for_site_operator = self.pretokenize_string_with(+ self.url().to_string(),+ tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),+ );let domain_name = self.url()@@ -1038,6 +1050,9 @@ impl Html {Field::Text(TextField::Url) => {doc.add_pre_tokenized_text(tantivy_field, url.clone())}+ Field::Text(TextField::UrlForSiteOperator) => {+ doc.add_pre_tokenized_text(tantivy_field, url_for_site_operator.clone())+ }Field::Text(TextField::UrlNoTokenizer) => {let url = self.url().to_string();@@ -1199,8 +1214,8 @@ impl Html {Field::Fast(FastField::NumDescriptionTokens) => {doc.add_u64(tantivy_field, description.tokens.len() as u64)}- Field::Fast(FastField::NumSiteTokens) => {- doc.add_u64(tantivy_field, site.tokens.len() as u64)+ Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {+ doc.add_u64(tantivy_field, url_for_site_operator.tokens.len() as u64)}Field::Fast(FastField::NumDomainTokens) => {doc.add_u64(tantivy_field, domain.tokens.len() as u64)