Skip to content

web search done right

Use title hash, site hash and url hash in collector

Mikkel Denker8d1fdcc6fbb9
diff --git a/src/collector.rs b/src/collector.rs
index 741ff4c3..63dbd33c 100644
--- a/src/collector.rs
+++ b/src/collector.rs
@@ -14,23 +14,26 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::{collections::HashMap, sync::Arc};
+
use min_max_heap::MinMaxHeap;
+use serde::{Deserialize, Serialize};
use tantivy::{
collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},
- fastfield::MultiValuedFastFieldReader,
DocId, Score, SegmentOrdinal, SegmentReader,
};
use crate::{
+ fastfield_cache,
inverted_index::{DocAddress, WebsitePointer},
- prehashed::{combine_u64s, PrehashMap, Prehashed},
- schema::{FastField, Field},
+ prehashed::{combine_u64s, Prehashed},
+ schema::FastField,
};
-fn adjust_score(num_taken: usize, original_score: f64) -> f64 {
- const SCALE: f64 = 14.0;
- original_score * (SCALE / (num_taken as f64 + SCALE))
-}
+// lower scale -> higher penalty
+const SITE_SCALE: f64 = 14.0;
+const TITLE_SCALE: f64 = 6.0;
+const URL_SCALE: f64 = 0.1;
#[derive(Clone)]
pub struct MaxDocsConsidered {
@@ -38,32 +41,35 @@ pub struct MaxDocsConsidered {
pub segments: usize,
}
-pub trait Doc: PartialEq + Eq + PartialOrd + Ord + Clone {
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct Hashes {
+ site: Prehashed,
+ title: Prehashed,
+ url: Prehashed,
+}
+
+pub trait Doc: Clone {
fn score(&self) -> &f64;
- fn key(&self) -> &Prehashed;
fn id(&self) -> &DocId;
-
- fn set_score(&mut self, score: f64);
- fn set_key(&mut self, key: Prehashed);
- fn set_id(&mut self, id: DocId);
-
- fn update_values_from(&mut self, other: &Self);
+ fn hashes(&self) -> Hashes;
}
pub struct TopDocs {
top_n: usize,
offset: usize,
max_docs: Option<MaxDocsConsidered>,
+ fastfield_cache: Arc<fastfield_cache::FastFieldCache>,
de_rank_similar: bool,
}
impl TopDocs {
- pub fn with_limit(top_n: usize) -> Self {
+ pub fn with_limit(top_n: usize, fastfield_cache: Arc<fastfield_cache::FastFieldCache>) -> Self {
Self {
top_n,
offset: 0,
max_docs: None,
de_rank_similar: false,
+ fastfield_cache,
}
}
@@ -106,20 +112,13 @@ impl Collector for TopDocs {
segment_local_id: tantivy::SegmentOrdinal,
segment: &tantivy::SegmentReader,
) -> tantivy::Result<Self::Child> {
- let key_reader = segment.fast_fields().u64s(
- segment
- .schema()
- .get_field(Field::Fast(FastField::SiteHash).name())
- .unwrap(),
- )?;
-
let max_docs = self
.max_docs
.as_ref()
.map(|max_docs| max_docs.total_docs / max_docs.segments);
Ok(TopSegmentCollector {
- key_reader,
+ fastfield_segment_cache: self.fastfield_cache.get_segment(&segment.segment_id()),
max_docs,
num_docs_taken: 0,
segment_ord: segment_local_id,
@@ -149,7 +148,7 @@ impl Collector for TopDocs {
.skip(self.offset)
.map(|doc| WebsitePointer {
score: doc.score,
- site_hash: doc.key,
+ hashes: doc.hashes,
address: DocAddress {
segment: doc.segment,
doc_id: doc.id,
@@ -160,13 +159,28 @@ impl Collector for TopDocs {
}
pub struct TopSegmentCollector {
- key_reader: MultiValuedFastFieldReader<u64>,
+ fastfield_segment_cache: Arc<fastfield_cache::SegmentCache>,
max_docs: Option<usize>,
num_docs_taken: usize,
segment_ord: SegmentOrdinal,
bucket_collector: BucketCollector<SegmentDoc>,
}
+impl TopSegmentCollector {
+ fn get_hash(&self, doc: &DocId, field: &FastField) -> Prehashed {
+ let hash = self
+ .fastfield_segment_cache
+ .get_doc_cache(field)
+ .get_u64s(doc)
+ .unwrap();
+
+ debug_assert_eq!(hash.len(), 2);
+
+ let hash = [hash[0], hash[1]];
+ combine_u64s(hash).into()
+ }
+}
+
impl SegmentCollector for TopSegmentCollector {
type Fruit = Vec<SegmentDoc>;
@@ -178,15 +192,13 @@ impl SegmentCollector for TopSegmentCollector {
self.num_docs_taken += 1;
}
- let mut keys = Vec::new();
- self.key_reader.get_vals(doc, &mut keys);
- debug_assert_eq!(keys.len(), 2);
-
- let keys = [keys[0], keys[1]];
- let key = combine_u64s(keys);
self.bucket_collector.insert(SegmentDoc {
- key: Prehashed(key),
+ hashes: Hashes {
+ site: self.get_hash(&doc, &FastField::SiteHash),
+ title: self.get_hash(&doc, &FastField::TitleHash),
+ url: self.get_hash(&doc, &FastField::UrlHash),
+ },
id: doc,
segment: self.segment_ord,
score: score as f64,
@@ -198,254 +210,157 @@ impl SegmentCollector for TopSegmentCollector {
}
}
-pub struct BucketCollector<T: Doc> {
- buckets: PrehashMap<Bucket<T>>,
- heads: MinMaxHeap<BucketHead>,
- top_n: usize,
+struct ScoredDoc<T: Doc> {
+ doc: T,
+ adjusted_score: f64,
}
-impl<T: Doc> BucketCollector<T> {
- pub fn new(top_n: usize) -> Self {
- assert!(top_n > 0);
-
- Self {
- top_n,
- heads: MinMaxHeap::with_capacity(top_n + 1),
- buckets: PrehashMap::new(),
- }
+impl<T: Doc> PartialOrd for ScoredDoc<T> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.adjusted_score.partial_cmp(&other.adjusted_score)
}
+}
- pub fn insert(&mut self, doc: T) {
- if let Some(bucket) = self.buckets.get_mut(doc.key()) {
- bucket.insert(doc);
- } else {
- let mut bucket = Bucket::new(self.top_n);
- bucket.insert(doc.clone());
-
- self.buckets.insert(doc.key().clone(), bucket);
-
- self.heads.push(BucketHead {
- key: doc.key().clone(),
- tweaked_score: *doc.score(),
- });
- }
-
- if self.buckets.len() > self.top_n + 1 {
- self.prune_buckets()
- }
+impl<T: Doc> PartialEq for ScoredDoc<T> {
+ fn eq(&self, other: &Self) -> bool {
+ self.adjusted_score == other.adjusted_score
}
+}
- fn prune_buckets(&mut self) {
- self.update_worst_head();
- let worst_head = self.heads.pop_min().unwrap();
- self.buckets.remove(&worst_head.key);
- }
-
- fn update_worst_head(&mut self) {
- loop {
- let mut worst_head = self.heads.peek_min_mut().unwrap();
- let current_score = self
- .buckets
- .get(&worst_head.key)
- .unwrap()
- .get_best()
- .unwrap()
- .tweaked_score;
-
- if worst_head.tweaked_score != current_score {
- worst_head.tweaked_score = current_score;
- } else {
- break;
- }
- }
+impl<T: Doc> Ord for ScoredDoc<T> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
}
+}
- fn build_heads(&self) -> MinMaxHeap<BucketHead> {
- let mut bucket_heads: MinMaxHeap<BucketHead> = MinMaxHeap::with_capacity(self.top_n);
-
- for (key, bucket) in self.buckets.iter() {
- let best_in_bucket = bucket.get_best().unwrap();
- if bucket_heads.len() >= self.top_n {
- let mut worst_head = bucket_heads.peek_min_mut().unwrap();
+impl<T: Doc> Eq for ScoredDoc<T> {}
- if best_in_bucket.tweaked_score > worst_head.tweaked_score {
- worst_head.key = key.clone();
- worst_head.tweaked_score = best_in_bucket.tweaked_score
- }
- } else {
- bucket_heads.push(BucketHead {
- key: key.clone(),
- tweaked_score: best_in_bucket.tweaked_score,
- });
- }
+impl<T: Doc> From<T> for ScoredDoc<T> {
+ fn from(doc: T) -> Self {
+ Self {
+ adjusted_score: *doc.score(),
+ doc,
}
-
- bucket_heads
}
+}
- pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {
- let mut res = Vec::new();
-
- let mut bucket_heads = self.build_heads();
+#[derive(Default)]
+struct BucketCount {
+ buckets: HashMap<Prehashed, usize>,
+}
- while let Some(mut head) = bucket_heads.pop_max() {
- let bucket = self.buckets.get_mut(&head.key).unwrap();
+impl BucketCount {
+ pub fn adjust_score<T: Doc>(&self, doc: &mut ScoredDoc<T>) {
+ let hashes = doc.doc.hashes();
- if let Some(mut doc) = bucket.pop_best() {
- if de_rank_similar {
- doc.set_score(adjust_score(bucket.num_taken - 1, *doc.score()));
- }
+ let mut adjuster = 1.0;
- res.push(doc);
+ let taken_sites = self.buckets.get(&hashes.site).unwrap_or(&0);
- if let Some(new_best) = bucket.get_best() {
- head.tweaked_score = new_best.tweaked_score;
- }
+ adjuster *= SITE_SCALE / (SITE_SCALE + (*taken_sites as f64));
- bucket_heads.push(head);
- }
+ let taken_urls = self.buckets.get(&hashes.url).unwrap_or(&0);
+ adjuster *= URL_SCALE / (URL_SCALE + (*taken_urls as f64));
- if res.len() == self.top_n {
- break;
- }
- }
-
- res
- }
-}
-
-struct BucketHead {
- key: Prehashed,
- tweaked_score: f64,
-}
+ let taken_titles = self.buckets.get(&hashes.title).unwrap_or(&0);
+ adjuster *= TITLE_SCALE / (TITLE_SCALE + (*taken_titles as f64));
-impl PartialEq for BucketHead {
- fn eq(&self, other: &Self) -> bool {
- self.tweaked_score == other.tweaked_score
+ doc.adjusted_score = *doc.doc.score() * adjuster;
}
-}
-impl Eq for BucketHead {}
-
-impl PartialOrd for BucketHead {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- self.tweaked_score.partial_cmp(&other.tweaked_score)
- }
-}
+ fn update_counts<T: Doc>(&mut self, doc: &ScoredDoc<T>) {
+ let hashes = doc.doc.hashes();
-impl Ord for BucketHead {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
+ *self.buckets.entry(hashes.site).or_default() += 1;
+ *self.buckets.entry(hashes.url).or_default() += 1;
+ *self.buckets.entry(hashes.title).or_default() += 1;
}
}
-struct Bucket<T: Doc> {
- num_taken: usize,
- docs: MinMaxHeap<T>,
+pub struct BucketCollector<T: Doc> {
+ count: BucketCount,
+ documents: MinMaxHeap<ScoredDoc<T>>,
top_n: usize,
}
-impl<T: Doc> Bucket<T> {
+impl<T: Doc> BucketCollector<T> {
pub fn new(top_n: usize) -> Self {
assert!(top_n > 0);
Self {
top_n,
- num_taken: 0,
- docs: MinMaxHeap::with_capacity(top_n),
+ count: BucketCount::default(),
+ documents: MinMaxHeap::with_capacity(top_n + 1),
}
}
- pub fn pop_best(&mut self) -> Option<T> {
- let res = self.docs.pop_max();
+ pub fn insert(&mut self, doc: T) {
+ let scored_doc = doc.into();
- self.num_taken += 1;
+ self.documents.push(scored_doc);
- res
+ self.prune_buckets()
}
- pub fn get_best(&self) -> Option<TweakedDoc<'_>> {
- self.docs.peek_max().map(|doc| TweakedDoc {
- tweaked_score: adjust_score(self.num_taken, *doc.score()),
- _doc_id: doc.id(),
- })
+ fn prune_buckets(&mut self) {
+ while self.documents.len() > self.top_n + 1 {
+ self.documents.pop_min().unwrap();
+ }
}
- pub fn insert(&mut self, doc: T) {
- if self.docs.len() >= self.top_n {
- let mut worst = self.docs.peek_min_mut().unwrap();
+ fn update_best_doc(&mut self) {
+ if self.documents.len() <= 1 {
+ return;
+ }
- worst.update_values_from(&doc);
- } else {
- self.docs.push(doc);
+ while let Some(mut best_doc) = self.documents.peek_max_mut() {
+ let current_score = best_doc.adjusted_score;
+ self.count.adjust_score(&mut *best_doc);
+
+ if best_doc.adjusted_score == current_score {
+ break;
+ }
}
}
-}
-struct TweakedDoc<'a> {
- tweaked_score: f64,
- _doc_id: &'a DocId,
+ pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {
+ let mut res = Vec::new();
+
+ while let Some(best_doc) = self.documents.pop_max() {
+ if de_rank_similar {
+ self.count.update_counts(&best_doc);
+ self.update_best_doc();
+ }
+ res.push(best_doc.doc);
+
+ if res.len() == self.top_n {
+ break;
+ }
+ }
+
+ res
+ }
}
#[derive(Debug, Clone)]
pub struct SegmentDoc {
- key: Prehashed,
+ hashes: Hashes,
id: DocId,
segment: SegmentOrdinal,
score: f64,
}
-impl PartialEq for SegmentDoc {
- fn eq(&self, other: &Self) -> bool {
- self.score == other.score
- }
-}
-
-impl Eq for SegmentDoc {}
-
-impl PartialOrd for SegmentDoc {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- self.score.partial_cmp(&other.score)
- }
-}
-
-impl Ord for SegmentDoc {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
- }
-}
-
impl Doc for SegmentDoc {
fn score(&self) -> &f64 {
&self.score
}
- fn key(&self) -> &Prehashed {
- &self.key
- }
-
fn id(&self) -> &DocId {
&self.id
}
- fn set_score(&mut self, score: f64) {
- self.score = score;
- }
-
- fn set_key(&mut self, key: Prehashed) {
- self.key = key;
- }
-
- fn set_id(&mut self, id: DocId) {
- self.id = id;
- }
-
- fn update_values_from(&mut self, other: &Self) {
- self.set_id(*other.id());
- self.set_key(other.key().clone());
- self.set_score(*other.score());
-
- self.segment = other.segment;
+ fn hashes(&self) -> Hashes {
+ self.hashes
}
}
@@ -530,12 +445,12 @@ where
mod tests {
use super::*;
- fn test(top_n: usize, docs: &[(u128, DocId, f64)], expected: &[(f64, DocId)]) {
+ fn test(top_n: usize, docs: &[(Hashes, DocId, f64)], expected: &[(f64, DocId)]) {
let mut collector = BucketCollector::new(top_n);
for doc in docs {
collector.insert(SegmentDoc {
- key: Prehashed(doc.0),
+ hashes: doc.0,
id: doc.1,
score: doc.2,
segment: 0,
@@ -556,11 +471,51 @@ mod tests {
test(
3,
&[
- (1, 123, 1.0),
- (2, 124, 2.0),
- (3, 125, 3.0),
- (4, 126, 4.0),
- (5, 127, 5.0),
+ (
+ Hashes {
+ site: 1.into(),
+ title: 1.into(),
+ url: 1.into(),
+ },
+ 123,
+ 1.0,
+ ),
+ (
+ Hashes {
+ site: 2.into(),
+ title: 2.into(),
+ url: 2.into(),
+ },
+ 124,
+ 2.0,
+ ),
+ (
+ Hashes {
+ site: 3.into(),
+ title: 3.into(),
+ url: 3.into(),
+ },
+ 125,
+ 3.0,
+ ),
+ (
+ Hashes {
+ site: 4.into(),
+ title: 4.into(),
+ url: 4.into(),
+ },
+ 126,
+ 4.0,
+ ),
+ (
+ Hashes {
+ site: 5.into(),
+ title: 5.into(),
+ url: 5.into(),
+ },
+ 127,
+ 5.0,
+ ),
],
&[(5.0, 127), (4.0, 126), (3.0, 125)],
);
@@ -570,7 +525,35 @@ mod tests {
fn less_than_topn() {
test(
10,
- &[(3, 125, 3.0), (4, 126, 4.0), (5, 127, 5.0)],
+ &[
+ (
+ Hashes {
+ site: 3.into(),
+ title: 3.into(),
+ url: 3.into(),
+ },
+ 125,
+ 3.0,
+ ),
+ (
+ Hashes {
+ site: 4.into(),
+ title: 4.into(),
+ url: 4.into(),
+ },
+ 126,
+ 4.0,
+ ),
+ (
+ Hashes {
+ site: 5.into(),
+ title: 5.into(),
+ url: 5.into(),
+ },
+ 127,
+ 5.0,
+ ),
+ ],
&[(5.0, 127), (4.0, 126), (3.0, 125)],
);
}
@@ -579,18 +562,70 @@ mod tests {
fn same_key_de_prioritised() {
test(
10,
- &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],
&[
- (adjust_score(0, 5.0), 127),
- (adjust_score(0, 3.0), 125),
- (adjust_score(1, 3.1), 126),
+ (
+ Hashes {
+ site: 1.into(),
+ title: 1.into(),
+ url: 1.into(),
+ },
+ 125,
+ 3.0,
+ ),
+ (
+ Hashes {
+ site: 2.into(),
+ title: 2.into(),
+ url: 2.into(),
+ },
+ 126,
+ 3.1,
+ ),
+ (
+ Hashes {
+ site: 2.into(),
+ title: 2.into(),
+ url: 2.into(),
+ },
+ 127,
+ 5.0,
+ ),
],
+ &[(5.0, 127), (3.0, 125), (3.1, 126)],
);
test(
2,
- &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],
- &[(adjust_score(0, 5.0), 127), (adjust_score(0, 3.0), 125)],
+ &[
+ (
+ Hashes {
+ site: 1.into(),
+ title: 1.into(),
+ url: 1.into(),
+ },
+ 125,
+ 3.0,
+ ),
+ (
+ Hashes {
+ site: 2.into(),
+ title: 2.into(),
+ url: 2.into(),
+ },
+ 126,
+ 3.1,
+ ),
+ (
+ Hashes {
+ site: 2.into(),
+ title: 2.into(),
+ url: 2.into(),
+ },
+ 127,
+ 5.0,
+ ),
+ ],
+ &[(5.0, 127), (3.0, 125)],
);
}
}
diff --git a/src/fastfield_cache.rs b/src/fastfield_cache.rs
index 57b39917..de245512 100644
--- a/src/fastfield_cache.rs
+++ b/src/fastfield_cache.rs
@@ -130,6 +130,13 @@ impl DocCache {
_ => None,
}
}
+
+ pub fn get_u64s(&self, doc: &DocId) -> Option<&[u64]> {
+ match self.get(doc) {
+ FieldValue::U64s(res) => Some(res),
+ _ => None,
+ }
+ }
}
pub struct SegmentCache {
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 9d35a2b3..a90752f8 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -23,9 +23,9 @@ use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
+use crate::collector::Hashes;
use crate::fastfield_cache::FastFieldCache;
use crate::image_store::Image;
-use crate::prehashed::Prehashed;
use crate::query::Query;
use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
use crate::snippet;
@@ -47,7 +47,7 @@ pub struct InitialSearchResult {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct WebsitePointer {
pub score: f64,
- pub site_hash: Prehashed,
+ pub hashes: Hashes,
pub address: DocAddress,
}
diff --git a/src/prehashed.rs b/src/prehashed.rs
index 3165d756..0863a4f9 100644
--- a/src/prehashed.rs
+++ b/src/prehashed.rs
@@ -47,9 +47,15 @@ impl Hasher for Prehasher {
}
}
-#[derive(Debug, Eq, Clone, Serialize, Deserialize)]
+#[derive(Debug, Eq, Clone, Copy, Serialize, Deserialize)]
pub struct Prehashed(pub u128);
+impl From<u128> for Prehashed {
+ fn from(val: u128) -> Self {
+ Self(val)
+ }
+}
+
impl Hash for Prehashed {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
state.write_u128(self.0);
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index eda19d72..53c1691d 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -93,7 +93,8 @@ impl Ranker {
Arc::clone(&self.fastfield_cache),
);
- let mut collector = TopDocs::with_limit(NUM_RESULTS_PER_PAGE);
+ let mut collector =
+ TopDocs::with_limit(NUM_RESULTS_PER_PAGE, Arc::clone(&self.fastfield_cache));
if self.de_rank_similar {
collector = collector.and_de_rank_similar()
diff --git a/src/searcher/distributed.rs b/src/searcher/distributed.rs
index 745464eb..7408c27a 100644
--- a/src/searcher/distributed.rs
+++ b/src/searcher/distributed.rs
@@ -22,7 +22,7 @@ use crate::{
searcher::{PrettifiedWebsitesResult, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE},
};
-use std::{cmp::Ordering, net::SocketAddr, time::Instant};
+use std::{net::SocketAddr, time::Instant};
use futures::stream::FuturesUnordered;
use futures::StreamExt;
@@ -269,57 +269,15 @@ impl collector::Doc for ScoredWebsitePointer {
&self.local_pointer.score
}
- fn key(&self) -> &crate::prehashed::Prehashed {
- &self.local_pointer.site_hash
- }
-
fn id(&self) -> &tantivy::DocId {
&self.local_pointer.address.doc_id
}
- fn set_score(&mut self, score: f64) {
- self.local_pointer.score = score;
- }
-
- fn set_key(&mut self, key: crate::prehashed::Prehashed) {
- self.local_pointer.site_hash = key;
- }
-
- fn set_id(&mut self, id: tantivy::DocId) {
- self.local_pointer.address.doc_id = id;
- }
-
- fn update_values_from(&mut self, other: &Self) {
- self.set_score(*other.score());
- self.set_key(other.key().clone());
- self.set_id(*other.id());
-
- self.shard = other.shard.clone();
+ fn hashes(&self) -> collector::Hashes {
+ self.local_pointer.hashes
}
}
-impl PartialOrd for ScoredWebsitePointer {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- self.local_pointer
- .score
- .partial_cmp(&other.local_pointer.score)
- }
-}
-
-impl Ord for ScoredWebsitePointer {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.partial_cmp(other).unwrap_or(Ordering::Equal)
- }
-}
-
-impl PartialEq for ScoredWebsitePointer {
- fn eq(&self, other: &Self) -> bool {
- self.local_pointer.score == other.local_pointer.score
- }
-}
-
-impl Eq for ScoredWebsitePointer {}
-
impl DistributedSearcher {
pub fn new(shards: Vec<Shard>) -> Self {
Self { shards }
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 59e75976..483eebbb 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -271,13 +271,15 @@ impl Html {
pub fn parse_without_text(html: &str, url: &str) -> Self {
let root = kuchiki::parse_html().one(html);
+ let mut url: Url = url.to_string().into();
+ url = url.full_without_id_tags().into();
Self {
root,
all_text: None,
clean_text: None,
lang: None,
- url: url.to_string().into(),
+ url,
}
}
@@ -498,7 +500,7 @@ impl Html {
}
fn pretokenize_url(&self) -> PreTokenizedString {
- let url = self.url().full_without_id_tags();
+ let url = self.url().full();
self.pretokenize_string(url)
}