web search done right
take extra edges to ensure the remote has enough for deduplication
Mikkel Denker9083d600c7a3
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 4d80fd4c..442e2b03 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -29,7 +29,7 @@ pub async fn main() {lambda_model_path: None,dual_encoder_model_path: None,bangs_path: Some("data/bangs.json".to_string()),- query_store_db_host: None,+ query_store_db: None,gossip_seed_nodes: None,gossip_addr: "0.0.0.0:8002".parse().unwrap(),collector: collector_conf.clone(),diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex 9940cbc4..56919e55 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -160,9 +160,14 @@ pub async fn router(None => None,};- let query_store_queue = config.query_store_db_host.clone().map(|db_host| {+ let query_store_queue = config.query_store_db.clone().map(|query_store_config| {let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));- tokio::spawn(store_improvements_loop(query_store_queue.clone(), db_host));+ tokio::spawn(store_improvements_loop(+ query_store_queue.clone(),+ query_store_config.host,+ query_store_config.username,+ query_store_config.password,+ ));query_store_queue});diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex b45c6b9d..f5fd50aa 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -253,6 +253,12 @@ pub struct ApiSpellCheck {#[serde(default)]pub correction_config: CorrectionConfig,}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]+pub struct QueryStoreConfig {+ pub host: String,+ pub username: String,+ pub password: String,+}#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct ApiConfig {@@ -262,7 +268,7 @@ pub struct ApiConfig {pub lambda_model_path: Option<String>,pub dual_encoder_model_path: Option<String>,pub bangs_path: Option<String>,- pub query_store_db_host: Option<String>,+ pub query_store_db: Option<QueryStoreConfig>,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,diff --git a/crates/core/src/improvement.rs b/crates/core/src/improvement.rsindex 41a0904e..ed2a869a 100644--- a/crates/core/src/improvement.rs+++ b/crates/core/src/improvement.rs@@ -86,8 +86,16 @@ async fn dump_queue(queue: &Mutex<LeakyQueue<ImprovementEvent>>) -> Vec<Improvempub async fn store_improvements_loop(queue: Arc<Mutex<LeakyQueue<ImprovementEvent>>>,scylla_host: String,+ scylla_user: String,+ scylla_password: String,) {- let scylla = ScyllaConn::new(scylla_host.as_str()).await.unwrap();+ let scylla = ScyllaConn::new(+ scylla_host.as_str(),+ scylla_user.as_str(),+ scylla_password.as_str(),+ )+ .await+ .unwrap();let mut interval = time::interval(Duration::from_secs(30));loop {@@ -114,12 +122,21 @@ struct ScyllaConn {}impl ScyllaConn {- async fn new(seed_node: &str) -> Result<Self, Error> {- let session = SessionBuilder::new().known_node(seed_node).build().await?;+ async fn new(seed_node: &str, user: &str, password: &str) -> Result<Self, Error> {+ let session = SessionBuilder::new()+ .known_node(seed_node)+ .user(user, password)+ .build()+ .await?;- session.query("CREATE KEYSPACE IF NOT EXISTS ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}", &[]).await?;session- .query(+ .query(+ "CREATE KEYSPACE IF NOT EXISTS ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}",+ &[],+ )+ .await?;+ session+ .query("CREATE TABLE IF NOT EXISTS ks.queries (qid uuid, query text, urls text, timestamp timestamp, primary key (qid, timestamp)) WITH default_time_to_live = 7776000", // ttl 90 days&[],)diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rsindex 78c7d24c..419435e6 100644--- a/crates/core/src/webgraph/query/backlink.rs+++ b/crates/core/src/webgraph/query/backlink.rs@@ -17,7 +17,7 @@use tantivy::query::ShortCircuitQuery;use super::{- collector::{HostDeduplicator, TopDocsCollector},+ collector::{top_docs, HostDeduplicator, TopDocsCollector},document_scorer::DefaultDocumentScorer,raw::{HostLinksQuery, LinksQuery},Query,@@ -124,9 +124,14 @@ impl Query for BacklinksQuery {fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {match self.limit {EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, ToId)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => Box::new(- ShortCircuitQuery::new(Box::new(LinksQuery::new(self.node, ToId)), limit as u64),- ),+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node, ToId)),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node, ToId)),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}@@ -214,17 +219,24 @@ impl Query for HostBacklinksQuery {FromHostId,searcher.warmed_column_fields().clone(),)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {- Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node,- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),- limit as u64,- ))- }+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node,+ ToHostId,+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node,+ ToHostId,+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}@@ -313,12 +325,14 @@ impl Query for FullBacklinksQuery {fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {match self.limit {EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), ToId)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {- Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node.id(), ToId)),- limit as u64,- ))- }+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node.id(), ToId)),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node.id(), ToId)),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}@@ -396,17 +410,24 @@ impl Query for FullHostBacklinksQuery {FromHostId,searcher.warmed_column_fields().clone(),)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {- Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),- limit as u64,- ))- }+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node.clone().into_host().id(),+ ToHostId,+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node.clone().into_host().id(),+ ToHostId,+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rsindex de4a2433..bb50daa8 100644--- a/crates/core/src/webgraph/query/collector/mod.rs+++ b/crates/core/src/webgraph/query/collector/mod.rs@@ -18,7 +18,7 @@ mod fast_count;mod first_doc;mod group_exact;mod group_sketch;-mod top_docs;+pub mod top_docs;pub use fast_count::{FastCountCollector, FastCountValue};pub use first_doc::FirstDocCollector;diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rsindex 15f4dae0..7ac7aa7a 100644--- a/crates/core/src/webgraph/query/collector/top_docs.rs+++ b/crates/core/src/webgraph/query/collector/top_docs.rs@@ -34,6 +34,9 @@ use crate::{distributed::member::ShardId, webgraph::warmed_column_fields::Warmeduse super::Collector;+/// Buffer to ensure remote has enough docs for deduplication+pub const DEDUPLICATION_BUFFER: usize = 128;+pub trait DeduplicatorDocwhereSelf: Send + Sync + serde::Serialize + serde::de::DeserializeOwned + Ord + Clone,@@ -253,9 +256,11 @@ where{fn computer(&self) -> Computer<D> {match (self.offset, self.limit) {- (Some(offset), Some(limit)) => Computer::TopN(TopNComputer::new(limit + offset)),+ (Some(offset), Some(limit)) => {+ Computer::TopN(TopNComputer::new(limit + offset + DEDUPLICATION_BUFFER))+ }(Some(_), None) => Computer::All(AllComputer::new()),- (None, Some(limit)) => Computer::TopN(TopNComputer::new(limit)),+ (None, Some(limit)) => Computer::TopN(TopNComputer::new(limit + DEDUPLICATION_BUFFER)),(None, None) => Computer::All(AllComputer::new()),}}@@ -296,20 +301,33 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDo) -> crate::Result<Self::Fruit> {let mut computer = self.computer();- for (score, doc) in self- .deduplicator- .deduplicate(segment_fruits.into_iter().flatten().collect())- {+ let before_deduplication: Vec<_> = segment_fruits.into_iter().flatten().collect();+ dbg!(&before_deduplication.len());+ let deduplicated = self.deduplicator.deduplicate(before_deduplication);++ for (score, doc) in deduplicated {computer.push(score, doc);}- let mut result = computer.harvest();+ let result = computer.harvest();if self.perform_offset {- result = result.into_iter().skip(self.offset.unwrap_or(0)).collect();+ Ok(result+ .into_iter()+ .skip(self.offset.unwrap_or(0))+ .take(self.limit.unwrap_or(usize::MAX))+ .collect())+ } else {+ Ok(result+ .into_iter()+ .take(+ // offset is only performed on remote. take buffer to ensure remote has enough docs for deduplication+ self.limit+ .unwrap_or(usize::MAX)+ .saturating_add(DEDUPLICATION_BUFFER),+ )+ .collect())}-- Ok(result)}}@@ -373,6 +391,7 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> SegmentCollectortype Fruit = Vec<(tantivy::Score, <D as Deduplicator>::Doc)>;fn collect(&mut self, doc: DocId, _: tantivy::Score) {+ dbg!(doc);if doc == tantivy::TERMINATED {return;}@@ -392,7 +411,7 @@ mod tests {use crate::{webgraph::{query::{BacklinksQuery, HostBacklinksQuery},- Edge, Node, Webgraph,+ Edge, EdgeLimit, Node, Webgraph,},webpage::RelFlags,};@@ -499,4 +518,94 @@ mod tests {assert_eq!(res.len(), 1);}++ #[test]+ fn test_offset_with_deduplication() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/1"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.1,+ })+ .unwrap();+ graph+ .insert(Edge {+ from: Node::from("https://A.com/2"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.1,+ })+ .unwrap();+ graph+ .insert(Edge {+ from: Node::from("https://C.com/1"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph+ .search(+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(+ EdgeLimit::LimitAndOffset {+ limit: 1024,+ offset: 0,+ },+ ),+ )+ .unwrap();++ assert_eq!(res.len(), 2);++ let res = graph+ .search(+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(+ EdgeLimit::LimitAndOffset {+ limit: 1,+ offset: 0,+ },+ ),+ )+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].from, Node::from("https://A.com/").into_host().id());++ let res = graph+ .search(+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(+ EdgeLimit::LimitAndOffset {+ limit: 1,+ offset: 1,+ },+ ),+ )+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].from, Node::from("https://C.com/").into_host().id());++ let res = graph+ .search(+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(+ EdgeLimit::LimitAndOffset {+ limit: 1,+ offset: 2,+ },+ ),+ )+ .unwrap();++ assert_eq!(res.len(), 0);+ }}diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rsindex 54cb89e5..d2e0b1e0 100644--- a/crates/core/src/webgraph/query/forwardlink.rs+++ b/crates/core/src/webgraph/query/forwardlink.rs@@ -16,7 +16,7 @@use super::{backlink::{fetch_edges, fetch_small_edges},- collector::{HostDeduplicator, TopDocsCollector},+ collector::{top_docs, HostDeduplicator, TopDocsCollector},document_scorer::DefaultDocumentScorer,raw::{host_links::HostLinksQuery, links::LinksQuery},Query,@@ -63,9 +63,14 @@ impl Query for ForwardlinksQuery {fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {match self.limit {EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, FromId)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => Box::new(- ShortCircuitQuery::new(Box::new(LinksQuery::new(self.node, FromId)), limit as u64),- ),+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node, FromId)),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node, FromId)),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}@@ -155,17 +160,24 @@ impl Query for HostForwardlinksQuery {ToHostId,searcher.warmed_column_fields().clone(),)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {- Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node,- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),- limit as u64,- ))- }+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node,+ FromHostId,+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node,+ FromHostId,+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}@@ -254,12 +266,14 @@ impl Query for FullForwardlinksQuery {fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {match self.limit {EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), FromId)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {- Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node.id(), FromId)),- limit as u64,- ))- }+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node.id(), FromId)),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(LinksQuery::new(self.node.id(), FromId)),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}@@ -337,17 +351,24 @@ impl Query for FullHostForwardlinksQuery {ToHostId,searcher.warmed_column_fields().clone(),)),- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {- Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),- limit as u64,- ))- }+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node.clone().into_host().id(),+ FromHostId,+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(+ Box::new(HostLinksQuery::new(+ self.node.clone().into_host().id(),+ FromHostId,+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )),+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,+ )),}}diff --git a/frontend/src/routes/search/ResultLink.svelte b/frontend/src/routes/search/ResultLink.svelteindex 125ba4f0..a99170f2 100644--- a/frontend/src/routes/search/ResultLink.svelte+++ b/frontend/src/routes/search/ResultLink.svelte@@ -1,5 +1,3 @@-<!-- Template to simplify the logic of result links, checking the results in new tab setting -->-<script lang="ts">import { resultsInNewTab } from '$lib/stores';import { improvements } from '$lib/improvements';@@ -38,7 +36,7 @@{title}use:improvements={resultIndex}target={$resultsInNewTab ? '_blank' : null}- rel={$resultsInNewTab ? 'noopener' : null}+ rel="noopener nofollow"bind:this={link}><slot />@@ -49,7 +47,7 @@class={_class}{title}target={$resultsInNewTab ? '_blank' : null}- rel={$resultsInNewTab ? 'noopener' : null}+ rel="noopener nofollow"bind:this={link}><slot />