web search done right
deduplicate host edges across entire query, not just adjacent documents
Mikkel Denkerba01fc700c07
diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rsindex 6e3f1cb3..91118035 100644--- a/crates/core/src/webgraph/query/raw/host_links.rs+++ b/crates/core/src/webgraph/query/raw/host_links.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use rustc_hash::FxHashSet;use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};use crate::webgraph::{@@ -100,7 +101,7 @@ impl tantivy::query::Weight for HostLinksWeight {struct HostLinksScorer {postings: SegmentPostings,host_id_column: Column<u64>,- last_host_id: Option<u64>,+ seen_host_ids: FxHashSet<u64>,}impl HostLinksScorer {@@ -118,10 +119,18 @@ impl HostLinksScorer {Ok(reader.inverted_index(term.field())?.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?- .map(|postings| Self {- last_host_id: host_id_column.first(postings.doc()),- host_id_column,- postings,+ .map(|postings| {+ let mut seen_host_ids = FxHashSet::default();++ if let Some(host_id) = host_id_column.first(postings.doc()) {+ seen_host_ids.insert(host_id);+ }++ Self {+ seen_host_ids,+ host_id_column,+ postings,+ }}))}}@@ -133,6 +142,12 @@ impl HostLinksScorer {}self.host_id_column.first(doc)}++ fn has_seen_host(&self, doc: tantivy::DocId) -> bool {+ self.host_id(doc)+ .map(|host_id| self.seen_host_ids.contains(&host_id))+ .unwrap_or(false)+ }}impl tantivy::query::Scorer for HostLinksScorer {@@ -144,26 +159,25 @@ impl tantivy::DocSet for HostLinksScorer {fn advance(&mut self) -> tantivy::DocId {self.postings.advance();- while self.last_host_id- == self.host_id(- self.postings- .block_cursor()- .skip_reader()- .last_doc_in_block(),- )- && self.doc() != tantivy::TERMINATED+ while self.has_seen_host(+ self.postings+ .block_cursor()+ .skip_reader()+ .last_doc_in_block(),+ ) && self.doc() != tantivy::TERMINATED{self.postings.mut_block_cursor().advance();self.postings.reset_cursor_start_block();}- while self.host_id(self.postings.doc()) == self.last_host_id- && self.doc() != tantivy::TERMINATED- {+ while self.has_seen_host(self.postings.doc()) && self.doc() != tantivy::TERMINATED {self.postings.advance();}- self.last_host_id = self.host_id(self.postings.doc());+ if let Some(host_id) = self.host_id(self.postings.doc()) {+ self.seen_host_ids.insert(host_id);+ }+self.postings.doc()}