Skip to content

web search done right

removed fastfield cache and introduced a fastfield reader instead

Mikkel Denkere04b5069b904
diff --git a/core/src/collector.rs b/core/src/collector.rs
index 9d8fc14f..6801f658 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -24,7 +24,7 @@ use tantivy::{
};
use crate::{
- fastfield_cache,
+ fastfield_reader,
inverted_index::{DocAddress, WebsitePointer},
prehashed::{combine_u64s, Prehashed},
ranking::initial::Score,
@@ -61,18 +61,18 @@ pub struct TopDocs {
top_n: usize,
offset: usize,
max_docs: Option<MaxDocsConsidered>,
- fastfield_cache: Arc<fastfield_cache::FastFieldCache>,
+ fastfield_reader: fastfield_reader::FastFieldReader,
de_rank_similar: bool,
}
impl TopDocs {
- pub fn with_limit(top_n: usize, fastfield_cache: Arc<fastfield_cache::FastFieldCache>) -> Self {
+ pub fn with_limit(top_n: usize, fastfield_reader: fastfield_reader::FastFieldReader) -> Self {
Self {
top_n,
offset: 0,
max_docs: None,
de_rank_similar: false,
- fastfield_cache,
+ fastfield_reader,
}
}
@@ -117,7 +117,7 @@ impl TopDocs {
.map(|max_docs| max_docs.total_docs / max_docs.segments);
Ok(TopSegmentCollector {
- fastfield_segment_cache: self.fastfield_cache.get_segment(&segment.segment_id()),
+ fastfield_segment_reader: self.fastfield_reader.get_segment(&segment.segment_id()),
max_docs,
num_docs_taken: 0,
segment_ord: segment_local_id,
@@ -127,7 +127,7 @@ impl TopDocs {
}
pub struct TopSegmentCollector {
- fastfield_segment_cache: Arc<fastfield_cache::SegmentCache>,
+ fastfield_segment_reader: Arc<fastfield_reader::SegmentReader>,
max_docs: Option<usize>,
num_docs_taken: usize,
segment_ord: SegmentOrdinal,
@@ -135,16 +135,22 @@ pub struct TopSegmentCollector {
}
impl TopSegmentCollector {
- fn get_hash(&self, doc: &DocId, field: &FastField) -> Prehashed {
- let hash = self
- .fastfield_segment_cache
- .get_doc_cache(field)
- .get_u64s(doc)
- .unwrap();
-
- debug_assert_eq!(hash.len(), 2);
-
- let hash = [hash[0], hash[1]];
+ fn get_hash(&self, doc: &DocId, field1: &FastField, field2: &FastField) -> Prehashed {
+ let hash: Option<u64> = self
+ .fastfield_segment_reader
+ .get_field_reader(field1)
+ .get(doc)
+ .into();
+ let hash1 = hash.unwrap();
+
+ let hash: Option<u64> = self
+ .fastfield_segment_reader
+ .get_field_reader(field2)
+ .get(doc)
+ .into();
+ let hash2 = hash.unwrap();
+
+ let hash = [hash1, hash2];
combine_u64s(hash).into()
}
}
@@ -159,16 +165,18 @@ impl TopSegmentCollector {
self.num_docs_taken += 1;
}
+ let simhash: Option<u64> = self
+ .fastfield_segment_reader
+ .get_field_reader(&FastField::SimHash)
+ .get(&doc)
+ .into();
+
self.bucket_collector.insert(SegmentDoc {
hashes: Hashes {
- site: self.get_hash(&doc, &FastField::SiteHash),
- title: self.get_hash(&doc, &FastField::TitleHash),
- url: self.get_hash(&doc, &FastField::UrlHash),
- simhash: self
- .fastfield_segment_cache
- .get_doc_cache(&FastField::SimHash)
- .get_u64(&doc)
- .unwrap(),
+ site: self.get_hash(&doc, &FastField::SiteHash1, &FastField::SiteHash2),
+ title: self.get_hash(&doc, &FastField::TitleHash1, &FastField::TitleHash2),
+ url: self.get_hash(&doc, &FastField::UrlHash1, &FastField::UrlHash2),
+ simhash: simhash.unwrap(),
},
id: doc,
segment: self.segment_ord,
diff --git a/core/src/fastfield_cache.rs b/core/src/fastfield_cache.rs
deleted file mode 100644
index edc4e575..00000000
--- a/core/src/fastfield_cache.rs
+++ /dev/null
@@ -1,150 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{
- collections::HashMap,
- sync::{Arc, RwLock},
-};
-
-use tantivy::{DocId, SegmentId};
-
-use crate::schema::{DataType, FastField, Field, ALL_FIELDS};
-
-#[derive(Default)]
-pub struct FastFieldCache {
- segments: RwLock<HashMap<SegmentId, Arc<SegmentCache>>>,
-}
-
-impl FastFieldCache {
- pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentCache> {
- Arc::clone(self.segments.read().unwrap().get(segment).unwrap())
- }
-}
-
-impl tantivy::Warmer for FastFieldCache {
- fn warm(&self, searcher: &tantivy::Searcher) -> tantivy::Result<()> {
- let mut segments_wrt = self.segments.write().unwrap();
-
- let mut fast_fields: Vec<_> = ALL_FIELDS
- .into_iter()
- .filter_map(|field| field.as_fast())
- .collect();
- fast_fields.sort_by_key(|field| *field as usize);
-
- let schema = searcher.schema();
-
- for reader in searcher.segment_readers() {
- let fastfield_readers = reader.fast_fields();
- let max_doc = reader.max_doc();
-
- let mut doc_caches = Vec::new();
-
- for field in &fast_fields {
- let tv_field = schema.get_field(Field::Fast(*field).name()).unwrap();
-
- let doc_cache = match field.data_type() {
- DataType::U64 => {
- let reader = fastfield_readers.u64(tv_field).unwrap();
- let mut output = vec![0; max_doc as usize];
- reader.get_range(0, &mut output);
-
- let docs = output.into_iter().map(FieldValue::U64).collect();
-
- DocCache { docs }
- }
- DataType::F64 => {
- let reader = fastfield_readers.f64(tv_field).unwrap();
- let mut output = vec![0.0; max_doc as usize];
- reader.get_range(0, &mut output);
-
- let docs = output.into_iter().map(FieldValue::F64).collect();
- DocCache { docs }
- }
- DataType::U64s => {
- let reader = fastfield_readers.u64s(tv_field).unwrap();
- let mut docs = Vec::new();
-
- for doc_id in 0..max_doc {
- let mut output = Vec::new();
- reader.get_vals(doc_id, &mut output);
- docs.push(output);
- }
-
- let docs = docs.into_iter().map(FieldValue::U64s).collect();
-
- DocCache { docs }
- }
- };
-
- doc_caches.push((field, doc_cache));
- }
-
- doc_caches.sort_by_key(|(field, _)| **field as usize);
-
- segments_wrt.insert(
- reader.segment_id(),
- Arc::new(SegmentCache {
- doc_caches: doc_caches.into_iter().map(|(_, cache)| cache).collect(),
- }),
- );
- }
-
- Ok(())
- }
-
- fn garbage_collect(&self, _live_generations: &[&tantivy::SearcherGeneration]) {}
-}
-
-pub enum FieldValue {
- U64(u64),
- U64s(Vec<u64>),
- F64(f64),
-}
-
-#[derive(Default)]
-pub struct DocCache {
- docs: Vec<FieldValue>,
-}
-
-impl DocCache {
- pub fn get(&self, doc: &DocId) -> &FieldValue {
- self.docs.get(*doc as usize).unwrap()
- }
-
- pub fn get_u64(&self, doc: &DocId) -> Option<u64> {
- match self.get(doc) {
- FieldValue::U64(res) => Some(*res),
- _ => None,
- }
- }
-
- pub fn get_u64s(&self, doc: &DocId) -> Option<&[u64]> {
- match self.get(doc) {
- FieldValue::U64s(res) => Some(res),
- _ => None,
- }
- }
-}
-
-pub struct SegmentCache {
- doc_caches: Vec<DocCache>, // fast_field -> doc_cache
-}
-
-impl SegmentCache {
- pub fn get_doc_cache(&self, field: &FastField) -> &DocCache {
- self.doc_caches.get(*field as usize).unwrap()
- }
-}
diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rs
new file mode 100644
index 00000000..eca35961
--- /dev/null
+++ b/core/src/fastfield_reader.rs
@@ -0,0 +1,128 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, sync::Arc};
+
+use tantivy::{fastfield::Column, DocId, SegmentId};
+
+use crate::{
+ schema::{DataType, FastField, Field, ALL_FIELDS},
+ search_ctx::Ctx,
+};
+
+#[derive(Default, Clone)]
+pub struct FastFieldReader {
+ segments: HashMap<SegmentId, Arc<SegmentReader>>,
+}
+
+impl FastFieldReader {
+ pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentReader> {
+ Arc::clone(self.segments.get(segment).unwrap())
+ }
+}
+
+impl FastFieldReader {
+ pub fn new(ctx: &Ctx) -> Self {
+ let mut segments = HashMap::new();
+
+ let schema = ctx.tv_searcher.schema();
+
+ for reader in ctx.tv_searcher.segment_readers() {
+ let fastfield_readers = reader.fast_fields();
+
+ let mut field_readers = Vec::new();
+
+ for field in ALL_FIELDS.iter().filter_map(|field| field.as_fast()) {
+ let tv_field = schema.get_field(Field::Fast(field).name()).unwrap();
+
+ let field_reader = match field.data_type() {
+ DataType::U64 => {
+ let reader = fastfield_readers.u64(tv_field).unwrap();
+ FieldReader::U64(reader)
+ }
+ DataType::F64 => {
+ let reader = fastfield_readers.f64(tv_field).unwrap();
+ FieldReader::F64(reader)
+ }
+ };
+
+ field_readers.push((field, field_reader));
+ }
+
+ field_readers.sort_by_key(|(field, _)| *field as usize);
+
+ segments.insert(
+ reader.segment_id(),
+ Arc::new(SegmentReader {
+ field_readers: field_readers
+ .into_iter()
+ .map(|(_, reader)| reader)
+ .collect(),
+ }),
+ );
+ }
+
+ Self { segments }
+ }
+}
+
+pub enum FieldValue {
+ U64(u64),
+ U64s(Vec<u64>),
+ F64(f64),
+}
+
+impl From<FieldValue> for Option<Vec<u64>> {
+ fn from(val: FieldValue) -> Self {
+ match val {
+ FieldValue::U64s(vec) => Some(vec),
+ _ => None,
+ }
+ }
+}
+
+impl From<FieldValue> for Option<u64> {
+ fn from(val: FieldValue) -> Self {
+ match val {
+ FieldValue::U64(res) => Some(res),
+ _ => None,
+ }
+ }
+}
+
+pub enum FieldReader {
+ U64(Arc<dyn Column<u64>>),
+ F64(Arc<dyn Column<f64>>),
+}
+
+impl FieldReader {
+ pub fn get(&self, doc: &DocId) -> FieldValue {
+ match self {
+ FieldReader::U64(reader) => FieldValue::U64(reader.get_val(*doc)),
+ FieldReader::F64(reader) => FieldValue::F64(reader.get_val(*doc)),
+ }
+ }
+}
+
+pub struct SegmentReader {
+ field_readers: Vec<FieldReader>, // fast_field -> field_reader
+}
+
+impl SegmentReader {
+ pub fn get_field_reader(&self, field: &FastField) -> &FieldReader {
+ self.field_readers.get(*field as usize).unwrap()
+ }
+}
diff --git a/core/src/index.rs b/core/src/index.rs
index 599003fb..9f3859f1 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -30,8 +30,9 @@ use uuid::Uuid;
use crate::directory::{self, DirEntry};
use crate::image_downloader::{ImageDownloadJob, ImageDownloader};
use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};
-use crate::inverted_index::{self, InitialSearchResult, InvertedIndex};
+use crate::inverted_index::{self, InvertedIndex};
use crate::query::Query;
+use crate::search_ctx::Ctx;
use crate::spell::{
Correction, CorrectionTerm, Dictionary, LogarithmicEdit, SpellChecker, TermSplitter,
};
@@ -122,20 +123,13 @@ impl Index {
Ok(())
}
- pub fn search_initial<C>(&self, query: &Query, collector: C) -> Result<InitialSearchResult>
- where
- C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
- {
- self.inverted_index.search_initial(query, collector)
- }
-
- pub fn top_nodes<C>(&self, query: &Query, collector: C) -> Result<Vec<NodeID>>
+ pub fn top_nodes<C>(&self, query: &Query, ctx: &Ctx, collector: C) -> Result<Vec<NodeID>>
where
C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
{
let websites = self
.inverted_index
- .search_initial(query, collector)?
+ .search_initial(query, ctx, collector)?
.top_websites;
let mut hosts = HashSet::with_capacity(websites.len());
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index d13138f0..2f69f0f3 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -24,7 +24,7 @@ use tantivy::tokenizer::{Tokenizer as _, TokenizerManager};
use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
use crate::collector::Hashes;
-use crate::fastfield_cache::FastFieldCache;
+use crate::fastfield_reader::FastFieldReader;
use crate::human_website_annotations::Topic;
use crate::image_store::Image;
use crate::query::Query;
@@ -32,6 +32,7 @@ use crate::ranking::initial::Score;
use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::SignalAggregator;
use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
+use crate::search_ctx::Ctx;
use crate::snippet;
use crate::tokenizer::Identity;
use crate::webgraph::NodeID;
@@ -42,7 +43,7 @@ use crate::{schema::create_schema, tokenizer::Tokenizer};
use std::collections::HashSet;
use std::fs;
use std::path::Path;
-use std::sync::{Arc, Weak};
+use std::sync::Arc;
#[derive(Debug, Serialize, Deserialize)]
pub struct InitialSearchResult {
@@ -90,8 +91,7 @@ pub struct InvertedIndex {
pub path: String,
tantivy_index: tantivy::Index,
writer: IndexWriter,
- pub(crate) reader: IndexReader,
- pub(crate) fastfield_cache: Arc<FastFieldCache>,
+ reader: IndexReader,
schema: Arc<Schema>,
}
@@ -138,25 +138,19 @@ impl InvertedIndex {
let merge_policy = NoMergePolicy::default();
writer.set_merge_policy(Box::new(merge_policy));
- let fastfield_cache = Arc::new(FastFieldCache::default());
-
- let warmers: Vec<Weak<dyn tantivy::Warmer>> = vec![Arc::downgrade(
- &(Arc::clone(&fastfield_cache) as Arc<dyn tantivy::Warmer>),
- )];
- let reader = tantivy_index.reader_builder().warmers(warmers).try_into()?;
+ let reader = tantivy_index.reader_builder().try_into()?;
Ok(InvertedIndex {
writer,
reader,
schema: Arc::new(schema),
path: path.as_ref().to_str().unwrap().to_string(),
- fastfield_cache,
tantivy_index,
})
}
- pub fn fastfield_cache(&self) -> Arc<FastFieldCache> {
- Arc::clone(&self.fastfield_cache)
+ pub fn fastfield_reader(&self, ctx: &Ctx) -> FastFieldReader {
+ FastFieldReader::new(ctx)
}
pub fn tokenizers(&self) -> &TokenizerManager {
@@ -182,13 +176,16 @@ impl InvertedIndex {
Ok(())
}
- pub fn search_initial<C>(&self, query: &Query, collector: C) -> Result<InitialSearchResult>
+ pub fn search_initial<C>(
+ &self,
+ query: &Query,
+ ctx: &Ctx,
+ collector: C,
+ ) -> Result<InitialSearchResult>
where
C: Collector<Fruit = Vec<WebsitePointer>>,
{
- let searcher = self.reader.searcher();
-
- let (count, pointers) = searcher.search(query, &(Count, collector))?;
+ let (count, pointers) = ctx.tv_searcher.search(query, &(Count, collector))?;
Ok(InitialSearchResult {
num_websites: count,
@@ -196,16 +193,22 @@ impl InvertedIndex {
})
}
+ pub fn local_search_ctx(&self) -> Ctx {
+ Ctx {
+ tv_searcher: self.reader.searcher(),
+ }
+ }
+
pub fn retrieve_ranking_websites(
&self,
+ ctx: &Ctx,
pointers: Vec<WebsitePointer>,
aggregator: &SignalAggregator,
) -> Result<Vec<RankingWebsite>> {
- let searcher = self.reader.searcher();
let mut top_websites = Vec::new();
for pointer in pointers {
- let doc = searcher.doc(pointer.address.into())?;
+ let doc = ctx.tv_searcher.doc(pointer.address.into())?;
top_websites.push(RankingWebsite::new(doc, pointer, aggregator));
}
@@ -233,10 +236,10 @@ impl InvertedIndex {
websites: &[WebsitePointer],
query: &Query,
) -> Result<Vec<RetrievedWebpage>> {
- let searcher = self.reader.searcher();
+ let ctx = self.local_search_ctx();
let mut webpages: Vec<RetrievedWebpage> = websites
.iter()
- .map(|website| self.retrieve_doc(website.address, &searcher))
+ .map(|website| self.retrieve_doc(website.address, &ctx))
.filter_map(|res| res.ok())
.map(|mut doc| {
if let Some(image) = doc.primary_image.as_ref() {
@@ -264,7 +267,7 @@ impl InvertedIndex {
&page.description,
&page.dmoz_description,
&page.region,
- &searcher,
+ &ctx.tv_searcher,
)?;
}
@@ -326,12 +329,8 @@ impl InvertedIndex {
Ok(())
}
- fn retrieve_doc(
- &self,
- doc_address: DocAddress,
- searcher: &tantivy::Searcher,
- ) -> Result<RetrievedWebpage> {
- let doc = searcher.doc(doc_address.into())?;
+ fn retrieve_doc(&self, doc_address: DocAddress, ctx: &Ctx) -> Result<RetrievedWebpage> {
+ let doc = ctx.tv_searcher.doc(doc_address.into())?;
Ok(RetrievedWebpage::from(doc))
}
@@ -407,8 +406,9 @@ impl InvertedIndex {
}
pub(crate) fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
- let searcher = self.reader.searcher();
- let field = searcher
+ let ctx = self.local_search_ctx();
+ let field = ctx
+ .tv_searcher
.schema()
.get_field(Field::Text(TextField::Url).name())
.unwrap();
@@ -422,12 +422,13 @@ impl InvertedIndex {
}
let query = tantivy::query::PhraseQuery::new(term_queries);
- let mut res = searcher
+ let mut res = ctx
+ .tv_searcher
.search(&query, &tantivy::collector::TopDocs::with_limit(1))
.unwrap();
res.pop()
- .map(|(_, doc)| self.retrieve_doc(doc.into(), &searcher).unwrap())
+ .map(|(_, doc)| self.retrieve_doc(doc.into(), &ctx).unwrap())
}
}
@@ -569,11 +570,16 @@ mod tests {
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
- fn search<C>(index: &InvertedIndex, query: &Query, collector: C) -> Result<SearchResult>
+ fn search<C>(
+ index: &InvertedIndex,
+ query: &Query,
+ ctx: &Ctx,
+ collector: C,
+ ) -> Result<SearchResult>
where
C: Collector<Fruit = Vec<WebsitePointer>>,
{
- let initial_result = index.search_initial(query, collector)?;
+ let initial_result = index.search_initial(query, ctx, collector)?;
let pointers: Vec<_> = initial_result.top_websites;
@@ -588,6 +594,7 @@ mod tests {
#[test]
fn simple_search() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
let query = Query::parse(
"website",
index.schema(),
@@ -595,13 +602,14 @@ mod tests {
&SignalAggregator::default(),
)
.expect("Failed to parse query");
+
+ let ctx = index.local_search_ctx();
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
-
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -624,7 +632,14 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let ctx = index.local_search_ctx();
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_reader(&ctx),
+ );
+
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -633,18 +648,6 @@ mod tests {
#[test]
fn document_not_matching() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse(
- "this query should not match",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
index
.insert(Webpage::new(
@@ -665,16 +668,9 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
- assert_eq!(result.documents.len(), 0);
- assert_eq!(result.num_docs, 0);
- }
-
- #[test]
- fn english_stemming() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "runner",
+ "this query should not match",
index.schema(),
index.tokenizers(),
&SignalAggregator::default(),
@@ -683,9 +679,18 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ assert_eq!(result.documents.len(), 0);
+ assert_eq!(result.num_docs, 0);
+ }
+
+ #[test]
+ fn english_stemming() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
index
.insert(Webpage::new(
&format!(
@@ -705,16 +710,9 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
- }
-
- #[test]
- fn stemmed_query_english() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "runners",
+ "runner",
index.schema(),
index.tokenizers(),
&SignalAggregator::default(),
@@ -723,9 +721,18 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ }
+
+ #[test]
+ fn stemmed_query_english() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
index
.insert(Webpage::new(
&format!(
@@ -745,16 +752,9 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
- }
-
- #[test]
- fn not_searchable_backlinks() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "great site",
+ "runners",
index.schema(),
index.tokenizers(),
&SignalAggregator::default(),
@@ -763,9 +763,18 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ }
+
+ #[test]
+ fn not_searchable_backlinks() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
index
.insert(Webpage::new(
&format!(
@@ -818,7 +827,21 @@ mod tests {
index.commit().expect("failed to commit index");
- let mut result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let ctx = index.local_search_ctx();
+ let query = Query::parse(
+ "great site",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_reader(&ctx),
+ );
+
+ let mut result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
result
.documents
@@ -831,18 +854,6 @@ mod tests {
#[test]
fn limited_top_docs() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse(
- "runner",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
for _ in 0..100 {
let dedup_s = crate::rand_words(100);
@@ -868,15 +879,9 @@ mod tests {
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
- assert_eq!(result.documents.len(), 20);
- }
-
- #[test]
- fn host_search() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "dr",
+ "runner",
index.schema(),
index.tokenizers(),
&SignalAggregator::default(),
@@ -885,9 +890,17 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ assert_eq!(result.documents.len(), 20);
+ }
+
+ #[test]
+ fn host_search() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
index
.insert(Webpage::new(
&format!(
@@ -907,7 +920,21 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let ctx = index.local_search_ctx();
+ let query = Query::parse(
+ "dr",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_reader(&ctx),
+ );
+
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
@@ -958,6 +985,7 @@ mod tests {
let mut index = index1.merge(index2);
index.commit().unwrap();
+ let ctx = index.local_search_ctx();
let query = Query::parse(
"website",
@@ -969,10 +997,10 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 2);
assert_eq!(result.documents.len(), 2);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -982,6 +1010,8 @@ mod tests {
#[test]
fn match_across_fields() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ let ctx = index.local_search_ctx();
let query = Query::parse(
"example test",
index.schema(),
@@ -992,10 +1022,10 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -1018,7 +1048,13 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let ctx = index.local_search_ctx();
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_reader(&ctx),
+ );
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -1027,11 +1063,6 @@ mod tests {
#[test]
fn only_show_primary_images_when_relevant() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
let mut webpage = Webpage::new(
&format!(
@@ -1065,7 +1096,14 @@ mod tests {
)
.expect("Failed to parse query");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let ctx = index.local_search_ctx();
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_reader(&ctx),
+ );
+
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
@@ -1087,7 +1125,7 @@ mod tests {
)
.expect("Failed to parse query");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
@@ -1098,22 +1136,6 @@ mod tests {
#[test]
fn id_links_removed_during_indexing() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse(
- "website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
-
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
- assert_eq!(result.documents.len(), 0);
- assert_eq!(result.num_docs, 0);
index
.insert(Webpage::new(
@@ -1134,17 +1156,9 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
- }
-
- #[test]
- fn remove_duplicates() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "dr",
+ "website",
index.schema(),
index.tokenizers(),
&SignalAggregator::default(),
@@ -1153,9 +1167,19 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.example.com");
+ }
+
+ #[test]
+ fn remove_duplicates() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
index
.insert(Webpage::new(
&format!(
@@ -1194,16 +1218,9 @@ mod tests {
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.dr.dk");
- }
-
- #[test]
- fn schema_org_stored() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let ctx = index.local_search_ctx();
let query = Query::parse(
- "test",
+ "dr",
index.schema(),
index.tokenizers(),
&SignalAggregator::default(),
@@ -1212,9 +1229,18 @@ mod tests {
let ranker = Ranker::new(
RegionCount::default(),
SignalAggregator::default(),
- index.fastfield_cache(),
+ index.fastfield_reader(&ctx),
);
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.dr.dk");
+ }
+
+ #[test]
+ fn schema_org_stored() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
index
.insert(Webpage::new(
&format!(
@@ -1240,7 +1266,21 @@ mod tests {
index.commit().expect("failed to commit index");
- let result = search(&index, &query, ranker.collector()).expect("Search failed");
+ let ctx = index.local_search_ctx();
+ let query = Query::parse(
+ "test",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_reader(&ctx),
+ );
+
+ let result = search(&index, &query, &ctx, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 1);
let schema = result.documents[0].schema_org.clone();
@@ -1276,6 +1316,7 @@ mod tests {
#[test]
fn get_webpage() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
index
.insert(Webpage::new(
&format!(
diff --git a/core/src/lib.rs b/core/src/lib.rs
index dcb5b49d..a03716b8 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -40,7 +40,7 @@ mod directory;
mod entity_index;
mod executor;
mod exponential_backoff;
-mod fastfield_cache;
+mod fastfield_reader;
mod human_website_annotations;
mod hyperloglog;
mod image_downloader;
@@ -54,6 +54,7 @@ mod qa_model;
mod query;
pub mod ranking;
mod schema;
+mod search_ctx;
mod search_prettifier;
pub mod searcher;
mod simhash;
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index b371235f..9f01d033 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -18,6 +18,7 @@ use crate::{
index::Index,
ranking::SignalAggregator,
schema::{Field, TextField},
+ search_ctx::Ctx,
searcher::NUM_RESULTS_PER_PAGE,
webpage::region::Region,
Result,
@@ -165,11 +166,12 @@ impl Query {
})
}
- pub fn set_optic(&mut self, optic: &Optic, index: &Index) {
+ pub fn set_optic(&mut self, optic: &Optic, index: &Index, ctx: &Ctx) {
let mut subqueries = vec![(Occur::Must, self.tantivy_query.box_clone())];
subqueries.append(
- &mut optic.as_multiple_tantivy(&index.schema(), index.inverted_index.fastfield_cache()),
+ &mut optic
+ .as_multiple_tantivy(&index.schema(), index.inverted_index.fastfield_reader(ctx)),
);
self.tantivy_query = Box::new(BooleanQuery::new(subqueries));
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 7de4db7a..03f6a1f4 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -14,8 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::sync::Arc;
-
use itertools::Itertools;
use optics::{Action, MatchLocation, Matching, Optic, Rule};
use tantivy::{
@@ -23,7 +21,7 @@ use tantivy::{
schema::Schema,
};
-use crate::{fastfield_cache::FastFieldCache, ranking::optics::SCALE, schema::TextField};
+use crate::{fastfield_reader::FastFieldReader, ranking::optics::SCALE, schema::TextField};
use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};
@@ -31,7 +29,7 @@ pub trait AsTantivyQuery {
fn as_tantivy(
&self,
schema: &Schema,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Box<dyn tantivy::query::Query>;
}
@@ -39,7 +37,7 @@ pub trait AsMultipleTantivyQuery {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;
}
@@ -47,7 +45,7 @@ impl AsMultipleTantivyQuery for Optic {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- fastfields: Arc<FastFieldCache>,
+ fastfields: FastFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
if self.discard_non_matching {
vec![(
@@ -84,7 +82,7 @@ impl AsMultipleTantivyQuery for Rule {
fn as_multiple_tantivy(
&self,
schema: &Schema,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
let mut subqueries: Vec<_> = self
.matches
@@ -92,7 +90,7 @@ impl AsMultipleTantivyQuery for Rule {
.map(|matching| {
(
Occur::Must,
- matching.as_tantivy(schema, fastfield_cache.clone()),
+ matching.as_tantivy(schema, fastfield_reader.clone()),
)
})
.collect();
@@ -133,35 +131,35 @@ impl AsTantivyQuery for Matching {
fn as_tantivy(
&self,
schema: &Schema,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Box<dyn tantivy::query::Query> {
match &self.location {
MatchLocation::Site => PatternQuery::new(
self.pattern.clone(),
TextField::Site,
schema,
- fastfield_cache,
+ fastfield_reader,
)
.box_clone(),
MatchLocation::Url => PatternQuery::new(
self.pattern.clone(),
TextField::Url,
schema,
- fastfield_cache,
+ fastfield_reader,
)
.box_clone(),
MatchLocation::Domain => PatternQuery::new(
self.pattern.clone(),
TextField::Domain,
schema,
- fastfield_cache,
+ fastfield_reader,
)
.box_clone(),
MatchLocation::Title => PatternQuery::new(
self.pattern.clone(),
TextField::Title,
schema,
- fastfield_cache,
+ fastfield_reader,
)
.box_clone(),
MatchLocation::Description => UnionQuery::from(vec![
@@ -169,14 +167,14 @@ impl AsTantivyQuery for Matching {
self.pattern.clone(),
TextField::Description,
schema,
- fastfield_cache.clone(),
+ fastfield_reader.clone(),
)
.box_clone(),
PatternQuery::new(
self.pattern.clone(),
TextField::DmozDescription,
schema,
- fastfield_cache,
+ fastfield_reader,
)
.box_clone(),
])
@@ -185,14 +183,14 @@ impl AsTantivyQuery for Matching {
self.pattern.clone(),
TextField::CleanBody,
schema,
- fastfield_cache,
+ fastfield_reader,
)
.box_clone(),
MatchLocation::Schema => PatternQuery::new(
self.pattern.clone(),
TextField::FlattenedSchemaOrgJson,
schema,
- fastfield_cache,
+ fastfield_reader,
)
.box_clone(),
}
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 2fca2ddb..39515a9a 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -27,7 +27,7 @@ use tantivy::{
};
use crate::{
- fastfield_cache::FastFieldCache,
+ fastfield_reader::{self, FastFieldReader},
query::intersection::Intersection,
ranking::bm25::Bm25Weight,
schema::{FastField, Field, TextField, ALL_FIELDS},
@@ -38,7 +38,7 @@ pub struct PatternQuery {
patterns: Vec<PatternPart>,
field: tantivy::schema::Field,
raw_terms: Vec<tantivy::Term>,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
}
impl std::fmt::Debug for PatternQuery {
@@ -56,7 +56,7 @@ impl PatternQuery {
patterns: Vec<PatternPart>,
field: TextField,
schema: &tantivy::schema::Schema,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Self {
let mut raw_terms = Vec::new();
@@ -81,7 +81,7 @@ impl PatternQuery {
patterns,
field: tv_field,
raw_terms,
- fastfield_cache,
+ fastfield_reader,
}
}
}
@@ -103,7 +103,7 @@ impl tantivy::query::Query for PatternQuery {
raw_terms: self.raw_terms.clone(),
patterns: self.patterns.clone(),
field: self.field,
- fastfield_cache: self.fastfield_cache.clone(),
+ fastfield_reader: self.fastfield_reader.clone(),
}))
}
@@ -125,7 +125,7 @@ struct PatternWeight {
patterns: Vec<PatternPart>,
raw_terms: Vec<tantivy::Term>,
field: tantivy::schema::Field,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
}
impl PatternWeight {
@@ -196,7 +196,7 @@ impl PatternWeight {
small_patterns,
reader.segment_id(),
num_tokens_fastfield,
- self.fastfield_cache.clone(),
+ self.fastfield_reader.clone(),
)))
}
}
@@ -254,9 +254,8 @@ struct PatternScorer {
left: Vec<u32>,
right: Vec<u32>,
phrase_count: u32,
- segment: tantivy::SegmentId,
num_tokens_field: FastField,
- fastfield_cache: Arc<FastFieldCache>,
+ segment_reader: Arc<fastfield_reader::SegmentReader>,
}
impl PatternScorer {
@@ -267,9 +266,10 @@ impl PatternScorer {
pattern: Vec<SmallPatternPart>,
segment: tantivy::SegmentId,
num_tokens_field: FastField,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Self {
let num_query_terms = term_postings_list.len();
+ let segment_reader = fastfield_reader.get_segment(&segment);
Self {
intersection_docset: Intersection::new(term_postings_list),
@@ -280,9 +280,8 @@ impl PatternScorer {
left: Vec::with_capacity(100),
right: Vec::with_capacity(100),
phrase_count: 0,
- segment,
num_tokens_field,
- fastfield_cache,
+ segment_reader,
}
}
fn phrase_count(&self) -> u32 {
@@ -307,12 +306,12 @@ impl PatternScorer {
let mut current_right_term = 1;
let mut slop = 1;
- let num_tokens_doc = self
- .fastfield_cache
- .get_segment(&self.segment)
- .get_doc_cache(&self.num_tokens_field)
- .get_u64(&self.doc())
- .unwrap();
+ let num_tokens_doc: Option<u64> = self
+ .segment_reader
+ .get_field_reader(&self.num_tokens_field)
+ .get(&self.doc())
+ .into();
+ let num_tokens_doc = num_tokens_doc.unwrap();
for (i, pattern_part) in self.pattern.iter().enumerate().skip(1) {
match pattern_part {
diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rs
index ba5c3d2a..45aa5f0c 100644
--- a/core/src/ranking/initial.rs
+++ b/core/src/ranking/initial.rs
@@ -16,7 +16,7 @@
use std::sync::Arc;
-use crate::fastfield_cache::FastFieldCache;
+use crate::fastfield_reader::FastFieldReader;
use crate::webpage::region::{Region, RegionCount};
use chrono::Utc;
use serde::{Deserialize, Serialize};
@@ -28,7 +28,7 @@ use super::SignalAggregator;
pub(crate) struct InitialScoreTweaker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
aggregator: SignalAggregator,
}
@@ -37,13 +37,13 @@ impl InitialScoreTweaker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
aggregator: SignalAggregator,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Self {
Self {
region_count,
selected_region,
aggregator,
- fastfield_cache,
+ fastfield_reader,
}
}
}
@@ -61,11 +61,11 @@ impl ScoreTweaker<Score> for InitialScoreTweaker {
fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
let mut aggregator = self.aggregator.clone();
- let fastfield_cache_segment = self
- .fastfield_cache
+ let fastfield_segment_reader = self
+ .fastfield_reader
.get_segment(&segment_reader.segment_id());
- aggregator.register_segment(fastfield_cache_segment);
+ aggregator.register_segment(fastfield_segment_reader);
let current_timestamp = Utc::now().timestamp() as usize;
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 49c231c9..fd738f2c 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -32,7 +32,7 @@ use tantivy::collector::Collector;
use crate::{
collector::{MaxDocsConsidered, TopDocs},
- fastfield_cache::FastFieldCache,
+ fastfield_reader::FastFieldReader,
inverted_index,
searcher::NUM_RESULTS_PER_PAGE,
webgraph::{
@@ -57,7 +57,7 @@ pub struct Ranker {
max_docs: Option<MaxDocsConsidered>,
offset: Option<usize>,
aggregator: SignalAggregator,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
de_rank_similar: bool,
num_results: Option<usize>,
}
@@ -66,7 +66,7 @@ impl Ranker {
pub fn new(
region_count: RegionCount,
aggregator: SignalAggregator,
- fastfield_cache: Arc<FastFieldCache>,
+ fastfield_reader: FastFieldReader,
) -> Self {
Ranker {
region_count: Arc::new(region_count),
@@ -75,7 +75,7 @@ impl Ranker {
aggregator,
max_docs: None,
de_rank_similar: true,
- fastfield_cache,
+ fastfield_reader,
num_results: None,
}
}
@@ -118,12 +118,12 @@ impl Ranker {
Arc::clone(&self.region_count),
self.selected_region,
aggregator,
- Arc::clone(&self.fastfield_cache),
+ self.fastfield_reader.clone(),
);
let mut collector = TopDocs::with_limit(
self.num_results.unwrap_or(NUM_RESULTS_PER_PAGE),
- Arc::clone(&self.fastfield_cache),
+ self.fastfield_reader.clone(),
);
if self.de_rank_similar {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 7e31b706..dcb5ad32 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- fastfield_cache,
+ fastfield_reader,
schema::{FastField, TextField},
webgraph::{
centrality::{online_harmonic, topic},
@@ -263,7 +263,7 @@ impl FieldBoost {
#[derive(Clone)]
pub struct SignalAggregator {
- fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,
+ fastfield_reader: Option<Arc<fastfield_reader::SegmentReader>>,
signal_coefficients: SignalCoefficient,
personal_centrality: Option<Arc<online_harmonic::Scorer>>,
inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,
@@ -306,7 +306,7 @@ impl SignalAggregator {
.collect();
Self {
- fastfield_cache: None,
+ fastfield_reader: None,
personal_centrality: None,
inbound_similariy: None,
signal_coefficients,
@@ -318,8 +318,8 @@ impl SignalAggregator {
}
}
- pub fn register_segment(&mut self, cache: Arc<fastfield_cache::SegmentCache>) {
- self.fastfield_cache = Some(cache);
+ pub fn register_segment(&mut self, reader: Arc<fastfield_reader::SegmentReader>) {
+ self.fastfield_reader = Some(reader);
}
pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
@@ -372,11 +372,12 @@ impl SignalAggregator {
current_timestamp: usize,
selected_region: Option<Region>,
) -> Score {
- let host_id = self.fastfield_cache.as_ref().and_then(|cache| {
- let node_id = cache
- .get_doc_cache(&FastField::HostNodeID)
- .get_u64(&doc)
- .unwrap();
+ let host_id = self.fastfield_reader.as_ref().and_then(|reader| {
+ let node_id: Option<u64> = reader
+ .get_field_reader(&FastField::HostNodeID)
+ .get(&doc)
+ .into();
+ let node_id = node_id.unwrap();
if node_id == u64::MAX {
None
@@ -396,9 +397,9 @@ impl SignalAggregator {
.into_iter()
.map(|signal| {
let fastfield_value = signal.as_fastfield().and_then(|field| {
- self.fastfield_cache
+ self.fastfield_reader
.as_ref()
- .and_then(|cache| cache.get_doc_cache(&field).get_u64(&doc))
+ .and_then(|reader| reader.get_field_reader(&field).get(&doc).into())
});
self.coefficients().get(&signal)
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 387aa13a..152c6377 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -91,11 +91,16 @@ pub enum FastField {
NumDescriptionTokens,
NumSiteTokens,
NumDomainTokens,
- SiteHash,
- UrlWithoutQueryHash,
- TitleHash,
- UrlHash,
- DomainHash,
+ SiteHash1,
+ SiteHash2,
+ UrlWithoutQueryHash1,
+ UrlWithoutQueryHash2,
+ TitleHash1,
+ TitleHash2,
+ UrlHash1,
+ UrlHash2,
+ DomainHash1,
+ DomainHash2,
PreComputedScore,
HostNodeID,
SimHash,
@@ -109,7 +114,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 43] = [
+pub static ALL_FIELDS: [Field; 48] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -145,11 +150,16 @@ pub static ALL_FIELDS: [Field; 43] = [
Field::Fast(FastField::NumDomainTokens),
Field::Fast(FastField::NumSiteTokens),
Field::Fast(FastField::NumFlattenedSchemaTokens),
- Field::Fast(FastField::SiteHash),
- Field::Fast(FastField::UrlWithoutQueryHash),
- Field::Fast(FastField::TitleHash),
- Field::Fast(FastField::UrlHash),
- Field::Fast(FastField::DomainHash),
+ Field::Fast(FastField::SiteHash1),
+ Field::Fast(FastField::SiteHash2),
+ Field::Fast(FastField::UrlWithoutQueryHash1),
+ Field::Fast(FastField::UrlWithoutQueryHash2),
+ Field::Fast(FastField::TitleHash1),
+ Field::Fast(FastField::TitleHash2),
+ Field::Fast(FastField::UrlHash1),
+ Field::Fast(FastField::UrlHash2),
+ Field::Fast(FastField::DomainHash1),
+ Field::Fast(FastField::DomainHash2),
Field::Fast(FastField::PreComputedScore),
Field::Fast(FastField::HostNodeID),
Field::Fast(FastField::SimHash),
@@ -294,20 +304,35 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::Fast(FastField::SiteHash) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::MultiValues),
+ Field::Fast(FastField::SiteHash1) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
+ ),
+ Field::Fast(FastField::SiteHash2) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
+ ),
+ Field::Fast(FastField::UrlWithoutQueryHash1) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
),
- Field::Fast(FastField::UrlWithoutQueryHash) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::MultiValues),
+ Field::Fast(FastField::UrlWithoutQueryHash2) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
),
- Field::Fast(FastField::UrlHash) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::MultiValues),
+ Field::Fast(FastField::UrlHash1) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
),
- Field::Fast(FastField::DomainHash) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::MultiValues),
+ Field::Fast(FastField::UrlHash2) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
),
- Field::Fast(FastField::TitleHash) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::MultiValues),
+ Field::Fast(FastField::DomainHash1) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
+ ),
+ Field::Fast(FastField::DomainHash2) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
+ ),
+ Field::Fast(FastField::TitleHash1) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
+ ),
+ Field::Fast(FastField::TitleHash2) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
),
Field::Fast(FastField::PreComputedScore) => IndexingOption::Float(
NumericOptions::default()
@@ -372,12 +397,17 @@ impl Field {
Field::Fast(FastField::NumDomainTokens) => "num_domain_tokens",
Field::Fast(FastField::NumSiteTokens) => "num_site_tokens",
Field::Fast(FastField::NumFlattenedSchemaTokens) => "num_flattened_schema_tokens",
- Field::Fast(FastField::SiteHash) => "site_hash",
- Field::Fast(FastField::UrlWithoutQueryHash) => "url_without_query_hash",
+ Field::Fast(FastField::SiteHash1) => "site_hash1",
+ Field::Fast(FastField::SiteHash2) => "site_hash2",
+ Field::Fast(FastField::UrlWithoutQueryHash1) => "url_without_query_hash1",
+ Field::Fast(FastField::UrlWithoutQueryHash2) => "url_without_query_hash2",
+ Field::Fast(FastField::TitleHash1) => "title_hash1",
+ Field::Fast(FastField::TitleHash2) => "title_hash2",
+ Field::Fast(FastField::UrlHash1) => "url_hash1",
+ Field::Fast(FastField::UrlHash2) => "url_hash2",
+ Field::Fast(FastField::DomainHash1) => "domain_hash1",
+ Field::Fast(FastField::DomainHash2) => "domain_hash2",
Field::Fast(FastField::PreComputedScore) => "pre_computed_score",
- Field::Fast(FastField::TitleHash) => "title_hash",
- Field::Fast(FastField::UrlHash) => "url_hash",
- Field::Fast(FastField::DomainHash) => "domain_hash",
Field::Fast(FastField::HostNodeID) => "host_node_id",
Field::Fast(FastField::SimHash) => "sim_hash",
Field::Fast(FastField::CrawlStability) => "crawl_stability",
@@ -454,12 +484,17 @@ impl Field {
"last_updated" => Some(Field::Fast(FastField::LastUpdated)),
"tracker_score" => Some(Field::Fast(FastField::TrackerScore)),
"region" => Some(Field::Fast(FastField::Region)),
- "site_hash" => Some(Field::Fast(FastField::SiteHash)),
- "url_without_query_hash" => Some(Field::Fast(FastField::UrlWithoutQueryHash)),
+ "site_hash1" => Some(Field::Fast(FastField::SiteHash1)),
+ "site_hash2" => Some(Field::Fast(FastField::SiteHash2)),
+ "url_without_query_hash1" => Some(Field::Fast(FastField::UrlWithoutQueryHash1)),
+ "url_without_query_hash2" => Some(Field::Fast(FastField::UrlWithoutQueryHash2)),
+ "url_hash1" => Some(Field::Fast(FastField::UrlHash1)),
+ "url_hash2" => Some(Field::Fast(FastField::UrlHash2)),
+ "domain_hash1" => Some(Field::Fast(FastField::DomainHash1)),
+ "domain_hash2" => Some(Field::Fast(FastField::DomainHash2)),
+ "title_hash1" => Some(Field::Fast(FastField::TitleHash1)),
+ "title_hash2" => Some(Field::Fast(FastField::TitleHash2)),
"pre_computed_score" => Some(Field::Fast(FastField::PreComputedScore)),
- "url_hash" => Some(Field::Fast(FastField::UrlHash)),
- "domain_hash" => Some(Field::Fast(FastField::DomainHash)),
- "title_hash" => Some(Field::Fast(FastField::TitleHash)),
"host_node_id" => Some(Field::Fast(FastField::HostNodeID)),
"sim_hash" => Some(Field::Fast(FastField::SimHash)),
"crawl_stability" => Some(Field::Fast(FastField::CrawlStability)),
@@ -508,7 +543,6 @@ pub enum IndexingOption {
pub enum DataType {
U64,
- U64s,
F64,
}
@@ -529,11 +563,16 @@ impl FastField {
FastField::NumDomainTokens => DataType::U64,
FastField::NumSiteTokens => DataType::U64,
FastField::NumFlattenedSchemaTokens => DataType::U64,
- FastField::SiteHash => DataType::U64s,
- FastField::UrlWithoutQueryHash => DataType::U64s,
- FastField::TitleHash => DataType::U64s,
- FastField::UrlHash => DataType::U64s,
- FastField::DomainHash => DataType::U64s,
+ FastField::SiteHash1 => DataType::U64,
+ FastField::SiteHash2 => DataType::U64,
+ FastField::UrlWithoutQueryHash1 => DataType::U64,
+ FastField::UrlWithoutQueryHash2 => DataType::U64,
+ FastField::TitleHash1 => DataType::U64,
+ FastField::TitleHash2 => DataType::U64,
+ FastField::UrlHash1 => DataType::U64,
+ FastField::UrlHash2 => DataType::U64,
+ FastField::DomainHash1 => DataType::U64,
+ FastField::DomainHash2 => DataType::U64,
FastField::PreComputedScore => DataType::F64,
FastField::HostNodeID => DataType::U64,
FastField::SimHash => DataType::U64,
diff --git a/core/src/search_ctx.rs b/core/src/search_ctx.rs
new file mode 100644
index 00000000..753b5718
--- /dev/null
+++ b/core/src/search_ctx.rs
@@ -0,0 +1,19 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub struct Ctx {
+ pub tv_searcher: tantivy::Searcher,
+}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index f76b26a6..c3cdda4b 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -29,6 +29,7 @@ use crate::ranking::centrality_store::SearchCentralityStore;
use crate::ranking::optics::CreateAggregator;
use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
+use crate::search_ctx::Ctx;
use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
use crate::spell::Correction;
use crate::webgraph::centrality::topic::TopicCentrality;
@@ -75,7 +76,7 @@ impl LocalSearcher {
self.topic_centrality = Some(topic_centrality);
}
- fn parse_query(&self, query: &SearchQuery) -> Result<(Query, Optic)> {
+ fn parse_query(&self, query: &SearchQuery, ctx: &Ctx) -> Result<(Query, Optic)> {
let optic = match query.optic_program.as_ref() {
Some(program) => Some(optics::parse(program)?),
None => None,
@@ -121,7 +122,7 @@ impl LocalSearcher {
optic = optic.try_merge(o)?;
}
- parsed_query.set_optic(&optic, &self.index);
+ parsed_query.set_optic(&optic, &self.index, ctx);
if parsed_query.is_empty() {
Err(Error::EmptyQuery)
@@ -133,13 +134,14 @@ impl LocalSearcher {
fn ranker(
&self,
query: &Query,
+ ctx: &Ctx,
de_rank_similar: bool,
aggregator: SignalAggregator,
) -> Result<Ranker> {
let mut ranker = Ranker::new(
self.index.region_count.clone(),
aggregator,
- self.index.inverted_index.fastfield_cache(),
+ self.index.inverted_index.fastfield_reader(ctx),
);
if let Some(region) = query.region() {
@@ -160,7 +162,7 @@ impl LocalSearcher {
.with_max_docs(10_000, self.index.num_segments())
.with_num_results(100);
- let top_host_nodes = self.index.top_nodes(query, ranker.collector())?;
+ let top_host_nodes = self.index.top_nodes(query, ctx, ranker.collector())?;
if !top_host_nodes.is_empty() {
let harmonic = centrality_store
.online_harmonic
@@ -180,7 +182,9 @@ impl LocalSearcher {
query: &SearchQuery,
de_rank_similar: bool,
) -> Result<(Vec<RankingWebsite>, usize)> {
- let (parsed_query, optic) = self.parse_query(query)?;
+ let searcher = self.index.inverted_index.local_search_ctx();
+
+ let (parsed_query, optic) = self.parse_query(query, &searcher)?;
let mut aggregator = optic
.pipeline
@@ -217,16 +221,19 @@ impl LocalSearcher {
)
}
- let ranker = self.ranker(&parsed_query, de_rank_similar, aggregator)?;
+ let ranker = self.ranker(&parsed_query, &searcher, de_rank_similar, aggregator)?;
- let res = self
- .index
- .search_initial(&parsed_query, ranker.collector())?;
+ let res = self.index.inverted_index.search_initial(
+ &parsed_query,
+ &searcher,
+ ranker.collector(),
+ )?;
- let ranking_websites = self
- .index
- .inverted_index
- .retrieve_ranking_websites(res.top_websites, &ranker.aggregator())?;
+ let ranking_websites = self.index.inverted_index.retrieve_ranking_websites(
+ &searcher,
+ res.top_websites,
+ &ranker.aggregator(),
+ )?;
Ok((ranking_websites, res.num_websites))
}
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index 896dab59..1e1f35d3 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -83,10 +83,10 @@ impl TopicCentrality {
.get_field(Field::Text(TextField::CleanBody).name())
.unwrap();
- let searcher = index.inverted_index.reader.searcher();
+ let ctx = index.inverted_index.local_search_ctx();
let mut term_count: HashMap<String, u64> = HashMap::new();
- for segment in searcher.segment_readers() {
+ for segment in ctx.tv_searcher.segment_readers() {
let reader = segment.inverted_index(body_field).unwrap();
let mut stream = reader.terms().stream().unwrap();
@@ -100,7 +100,8 @@ impl TopicCentrality {
}
let top_topics = topics.top_topics(NUM_TOPICS);
- let topic_field = searcher
+ let topic_field = ctx
+ .tv_searcher
.schema()
.get_field(Field::Text(TextField::HostTopic).name())
.unwrap();
@@ -125,7 +126,7 @@ impl TopicCentrality {
let query = BooleanQuery::new(vec![(Occur::Must, topic), (Occur::Must, term)]);
- let topic_freq = searcher.search(&query, &collector).unwrap();
+ let topic_freq = ctx.tv_searcher.search(&query, &collector).unwrap();
scores[i] = topic_freq as f64 / total_doc_freq as f64;
}
@@ -137,7 +138,7 @@ impl TopicCentrality {
Arc::new(index.region_count.clone()),
None,
SignalAggregator::default(),
- Arc::clone(&index.inverted_index.fastfield_cache),
+ index.inverted_index.fastfield_reader(&ctx),
);
let graph_node_id = index
@@ -145,7 +146,7 @@ impl TopicCentrality {
.get_field(Field::Fast(FastField::HostNodeID).name())
.unwrap();
- let collector = TopDocs::with_limit(1000, index.inverted_index.fastfield_cache())
+ let collector = TopDocs::with_limit(1000, index.inverted_index.fastfield_reader(&ctx))
.tweak_score(score_tweaker);
let mut nodes: Vec<_> = webgraph.nodes().collect();
@@ -158,12 +159,13 @@ impl TopicCentrality {
let topic = tantivy::Term::from_facet(topic_field, &facet);
let query = TermQuery::new(topic, IndexRecordOption::Basic).box_clone();
- let top_sites: HashSet<_> = searcher
+ let top_sites: HashSet<_> = ctx
+ .tv_searcher
.search(&query, &collector)
.unwrap()
.into_iter()
.filter_map(|pointer| {
- let doc = searcher.doc(pointer.address.into()).unwrap();
+ let doc = ctx.tv_searcher.doc(pointer.address.into()).unwrap();
let id = doc.get_first(graph_node_id).unwrap().as_u64().unwrap();
if id == u64::MAX {
None
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index aebe9758..89689528 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -733,6 +733,12 @@ impl Html {
},
};
+ let site_hash = split_u128(hash(self.url().site()).0);
+ let url_without_query_hash = split_u128(hash(self.url().without_query()).0);
+ let url_hash = split_u128(hash(self.url().full()).0);
+ let domain_hash = split_u128(hash(self.url().domain()).0);
+ let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
+
for field in &ALL_FIELDS {
let tantivy_field = schema
.get_field(field.name())
@@ -881,35 +887,35 @@ impl Html {
Field::Fast(FastField::NumFlattenedSchemaTokens) => {
doc.add_u64(tantivy_field, pretokenized_schema_json.tokens.len() as u64)
}
- Field::Fast(FastField::SiteHash) => {
- let hash = hash(self.url().site()).0;
- let u64s = split_u128(hash);
- doc.add_u64(tantivy_field, u64s[0]);
- doc.add_u64(tantivy_field, u64s[1]);
+ Field::Fast(FastField::SiteHash1) => {
+ doc.add_u64(tantivy_field, site_hash[0]);
+ }
+ Field::Fast(FastField::SiteHash2) => {
+ doc.add_u64(tantivy_field, site_hash[1]);
+ }
+ Field::Fast(FastField::UrlWithoutQueryHash1) => {
+ doc.add_u64(tantivy_field, url_without_query_hash[0]);
+ }
+ Field::Fast(FastField::UrlWithoutQueryHash2) => {
+ doc.add_u64(tantivy_field, url_without_query_hash[1]);
+ }
+ Field::Fast(FastField::UrlHash1) => {
+ doc.add_u64(tantivy_field, url_hash[0]);
+ }
+ Field::Fast(FastField::UrlHash2) => {
+ doc.add_u64(tantivy_field, url_hash[1]);
}
- Field::Fast(FastField::UrlWithoutQueryHash) => {
- let hash = hash(self.url().without_query()).0;
- let u64s = split_u128(hash);
- doc.add_u64(tantivy_field, u64s[0]);
- doc.add_u64(tantivy_field, u64s[1]);
+ Field::Fast(FastField::DomainHash1) => {
+ doc.add_u64(tantivy_field, domain_hash[0]);
}
- Field::Fast(FastField::UrlHash) => {
- let hash = hash(self.url().full()).0;
- let u64s = split_u128(hash);
- doc.add_u64(tantivy_field, u64s[0]);
- doc.add_u64(tantivy_field, u64s[1]);
+ Field::Fast(FastField::DomainHash2) => {
+ doc.add_u64(tantivy_field, domain_hash[1]);
}
- Field::Fast(FastField::DomainHash) => {
- let hash = hash(self.url().domain()).0;
- let u64s = split_u128(hash);
- doc.add_u64(tantivy_field, u64s[0]);
- doc.add_u64(tantivy_field, u64s[1]);
+ Field::Fast(FastField::TitleHash1) => {
+ doc.add_u64(tantivy_field, title_hash[0]);
}
- Field::Fast(FastField::TitleHash) => {
- let hash = hash(self.title().unwrap_or_default()).0;
- let u64s = split_u128(hash);
- doc.add_u64(tantivy_field, u64s[0]);
- doc.add_u64(tantivy_field, u64s[1]);
+ Field::Fast(FastField::TitleHash2) => {
+ doc.add_u64(tantivy_field, title_hash[1]);
}
Field::Fast(FastField::SimHash) => {
let hash = if !clean_text.text.is_empty() {