Skip to content

web search done right

only consider simhash duplicates if we don't have enough non-duplicates

Mikkel Denkerc4ea75ad2a8a
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rs
index c1cff34a..7e957465 100644
--- a/crates/core/src/collector/top_docs.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -196,6 +196,7 @@ impl TopSegmentCollector {
}
}
+#[derive(Debug, Clone)]
struct ScoredDoc<T: Doc> {
doc: T,
adjusted_score: f64,
@@ -337,7 +338,11 @@ impl<T: Doc> BucketCollector<T> {
}
}
- res.extend(simhash_dups);
+ res.extend(
+ simhash_dups
+ .into_iter()
+ .take(self.top_n.saturating_sub(res.len())),
+ );
res
}