Skip to content

web search done right

take extra edges to ensure the remote has enough for deduplication

Mikkel Denker9083d600c7a3
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index 4d80fd4c..442e2b03 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -29,7 +29,7 @@ pub async fn main() {
lambda_model_path: None,
dual_encoder_model_path: None,
bangs_path: Some("data/bangs.json".to_string()),
- query_store_db_host: None,
+ query_store_db: None,
gossip_seed_nodes: None,
gossip_addr: "0.0.0.0:8002".parse().unwrap(),
collector: collector_conf.clone(),
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index 9940cbc4..56919e55 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -160,9 +160,14 @@ pub async fn router(
None => None,
};
- let query_store_queue = config.query_store_db_host.clone().map(|db_host| {
+ let query_store_queue = config.query_store_db.clone().map(|query_store_config| {
let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));
- tokio::spawn(store_improvements_loop(query_store_queue.clone(), db_host));
+ tokio::spawn(store_improvements_loop(
+ query_store_queue.clone(),
+ query_store_config.host,
+ query_store_config.username,
+ query_store_config.password,
+ ));
query_store_queue
});
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index b45c6b9d..f5fd50aa 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -253,6 +253,12 @@ pub struct ApiSpellCheck {
#[serde(default)]
pub correction_config: CorrectionConfig,
}
+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
+pub struct QueryStoreConfig {
+ pub host: String,
+ pub username: String,
+ pub password: String,
+}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct ApiConfig {
@@ -262,7 +268,7 @@ pub struct ApiConfig {
pub lambda_model_path: Option<String>,
pub dual_encoder_model_path: Option<String>,
pub bangs_path: Option<String>,
- pub query_store_db_host: Option<String>,
+ pub query_store_db: Option<QueryStoreConfig>,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
diff --git a/crates/core/src/improvement.rs b/crates/core/src/improvement.rs
index 41a0904e..ed2a869a 100644
--- a/crates/core/src/improvement.rs
+++ b/crates/core/src/improvement.rs
@@ -86,8 +86,16 @@ async fn dump_queue(queue: &Mutex<LeakyQueue<ImprovementEvent>>) -> Vec<Improvem
pub async fn store_improvements_loop(
queue: Arc<Mutex<LeakyQueue<ImprovementEvent>>>,
scylla_host: String,
+ scylla_user: String,
+ scylla_password: String,
) {
- let scylla = ScyllaConn::new(scylla_host.as_str()).await.unwrap();
+ let scylla = ScyllaConn::new(
+ scylla_host.as_str(),
+ scylla_user.as_str(),
+ scylla_password.as_str(),
+ )
+ .await
+ .unwrap();
let mut interval = time::interval(Duration::from_secs(30));
loop {
@@ -114,12 +122,21 @@ struct ScyllaConn {
}
impl ScyllaConn {
- async fn new(seed_node: &str) -> Result<Self, Error> {
- let session = SessionBuilder::new().known_node(seed_node).build().await?;
+ async fn new(seed_node: &str, user: &str, password: &str) -> Result<Self, Error> {
+ let session = SessionBuilder::new()
+ .known_node(seed_node)
+ .user(user, password)
+ .build()
+ .await?;
- session.query("CREATE KEYSPACE IF NOT EXISTS ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}", &[]).await?;
session
- .query(
+ .query(
+ "CREATE KEYSPACE IF NOT EXISTS ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}",
+ &[],
+ )
+ .await?;
+ session
+ .query(
"CREATE TABLE IF NOT EXISTS ks.queries (qid uuid, query text, urls text, timestamp timestamp, primary key (qid, timestamp)) WITH default_time_to_live = 7776000", // ttl 90 days
&[],
)
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index 78c7d24c..419435e6 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -17,7 +17,7 @@
use tantivy::query::ShortCircuitQuery;
use super::{
- collector::{HostDeduplicator, TopDocsCollector},
+ collector::{top_docs, HostDeduplicator, TopDocsCollector},
document_scorer::DefaultDocumentScorer,
raw::{HostLinksQuery, LinksQuery},
Query,
@@ -124,9 +124,14 @@ impl Query for BacklinksQuery {
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
match self.limit {
EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, ToId)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => Box::new(
- ShortCircuitQuery::new(Box::new(LinksQuery::new(self.node, ToId)), limit as u64),
- ),
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node, ToId)),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node, ToId)),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
@@ -214,17 +219,24 @@ impl Query for HostBacklinksQuery {
FromHostId,
searcher.warmed_column_fields().clone(),
)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
- Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node,
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
- limit as u64,
- ))
- }
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node,
+ ToHostId,
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node,
+ ToHostId,
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
@@ -313,12 +325,14 @@ impl Query for FullBacklinksQuery {
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
match self.limit {
EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), ToId)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
- Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node.id(), ToId)),
- limit as u64,
- ))
- }
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node.id(), ToId)),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node.id(), ToId)),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
@@ -396,17 +410,24 @@ impl Query for FullHostBacklinksQuery {
FromHostId,
searcher.warmed_column_fields().clone(),
)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
- Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
- limit as u64,
- ))
- }
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ ToHostId,
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ ToHostId,
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rs
index de4a2433..bb50daa8 100644
--- a/crates/core/src/webgraph/query/collector/mod.rs
+++ b/crates/core/src/webgraph/query/collector/mod.rs
@@ -18,7 +18,7 @@ mod fast_count;
mod first_doc;
mod group_exact;
mod group_sketch;
-mod top_docs;
+pub mod top_docs;
pub use fast_count::{FastCountCollector, FastCountValue};
pub use first_doc::FirstDocCollector;
diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rs
index 15f4dae0..7ac7aa7a 100644
--- a/crates/core/src/webgraph/query/collector/top_docs.rs
+++ b/crates/core/src/webgraph/query/collector/top_docs.rs
@@ -34,6 +34,9 @@ use crate::{distributed::member::ShardId, webgraph::warmed_column_fields::Warmed
use super::Collector;
+/// Buffer to ensure remote has enough docs for deduplication
+pub const DEDUPLICATION_BUFFER: usize = 128;
+
pub trait DeduplicatorDoc
where
Self: Send + Sync + serde::Serialize + serde::de::DeserializeOwned + Ord + Clone,
@@ -253,9 +256,11 @@ where
{
fn computer(&self) -> Computer<D> {
match (self.offset, self.limit) {
- (Some(offset), Some(limit)) => Computer::TopN(TopNComputer::new(limit + offset)),
+ (Some(offset), Some(limit)) => {
+ Computer::TopN(TopNComputer::new(limit + offset + DEDUPLICATION_BUFFER))
+ }
(Some(_), None) => Computer::All(AllComputer::new()),
- (None, Some(limit)) => Computer::TopN(TopNComputer::new(limit)),
+ (None, Some(limit)) => Computer::TopN(TopNComputer::new(limit + DEDUPLICATION_BUFFER)),
(None, None) => Computer::All(AllComputer::new()),
}
}
@@ -296,20 +301,33 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDo
) -> crate::Result<Self::Fruit> {
let mut computer = self.computer();
- for (score, doc) in self
- .deduplicator
- .deduplicate(segment_fruits.into_iter().flatten().collect())
- {
+ let before_deduplication: Vec<_> = segment_fruits.into_iter().flatten().collect();
+ dbg!(&before_deduplication.len());
+ let deduplicated = self.deduplicator.deduplicate(before_deduplication);
+
+ for (score, doc) in deduplicated {
computer.push(score, doc);
}
- let mut result = computer.harvest();
+ let result = computer.harvest();
if self.perform_offset {
- result = result.into_iter().skip(self.offset.unwrap_or(0)).collect();
+ Ok(result
+ .into_iter()
+ .skip(self.offset.unwrap_or(0))
+ .take(self.limit.unwrap_or(usize::MAX))
+ .collect())
+ } else {
+ Ok(result
+ .into_iter()
+ .take(
+ // offset is only performed on remote. take buffer to ensure remote has enough docs for deduplication
+ self.limit
+ .unwrap_or(usize::MAX)
+ .saturating_add(DEDUPLICATION_BUFFER),
+ )
+ .collect())
}
-
- Ok(result)
}
}
@@ -373,6 +391,7 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> SegmentCollector
type Fruit = Vec<(tantivy::Score, <D as Deduplicator>::Doc)>;
fn collect(&mut self, doc: DocId, _: tantivy::Score) {
+ dbg!(doc);
if doc == tantivy::TERMINATED {
return;
}
@@ -392,7 +411,7 @@ mod tests {
use crate::{
webgraph::{
query::{BacklinksQuery, HostBacklinksQuery},
- Edge, Node, Webgraph,
+ Edge, EdgeLimit, Node, Webgraph,
},
webpage::RelFlags,
};
@@ -499,4 +518,94 @@ mod tests {
assert_eq!(res.len(), 1);
}
+
+ #[test]
+ fn test_offset_with_deduplication() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/1"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.1,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/2"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.1,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://C.com/1"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph
+ .search(
+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(
+ EdgeLimit::LimitAndOffset {
+ limit: 1024,
+ offset: 0,
+ },
+ ),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 2);
+
+ let res = graph
+ .search(
+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(
+ EdgeLimit::LimitAndOffset {
+ limit: 1,
+ offset: 0,
+ },
+ ),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("https://A.com/").into_host().id());
+
+ let res = graph
+ .search(
+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(
+ EdgeLimit::LimitAndOffset {
+ limit: 1,
+ offset: 1,
+ },
+ ),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("https://C.com/").into_host().id());
+
+ let res = graph
+ .search(
+ &HostBacklinksQuery::new(Node::from("https://B.com/").into_host().id()).with_limit(
+ EdgeLimit::LimitAndOffset {
+ limit: 1,
+ offset: 2,
+ },
+ ),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 0);
+ }
}
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
index 54cb89e5..d2e0b1e0 100644
--- a/crates/core/src/webgraph/query/forwardlink.rs
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -16,7 +16,7 @@
use super::{
backlink::{fetch_edges, fetch_small_edges},
- collector::{HostDeduplicator, TopDocsCollector},
+ collector::{top_docs, HostDeduplicator, TopDocsCollector},
document_scorer::DefaultDocumentScorer,
raw::{host_links::HostLinksQuery, links::LinksQuery},
Query,
@@ -63,9 +63,14 @@ impl Query for ForwardlinksQuery {
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
match self.limit {
EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, FromId)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => Box::new(
- ShortCircuitQuery::new(Box::new(LinksQuery::new(self.node, FromId)), limit as u64),
- ),
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node, FromId)),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node, FromId)),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
@@ -155,17 +160,24 @@ impl Query for HostForwardlinksQuery {
ToHostId,
searcher.warmed_column_fields().clone(),
)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
- Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node,
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
- limit as u64,
- ))
- }
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node,
+ FromHostId,
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node,
+ FromHostId,
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
@@ -254,12 +266,14 @@ impl Query for FullForwardlinksQuery {
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
match self.limit {
EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), FromId)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
- Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node.id(), FromId)),
- limit as u64,
- ))
- }
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node.id(), FromId)),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node.id(), FromId)),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
@@ -337,17 +351,24 @@ impl Query for FullHostForwardlinksQuery {
ToHostId,
searcher.warmed_column_fields().clone(),
)),
- EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
- Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
- limit as u64,
- ))
- }
+ EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ FromHostId,
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ FromHostId,
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )),
+ (limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
+ )),
}
}
diff --git a/frontend/src/routes/search/ResultLink.svelte b/frontend/src/routes/search/ResultLink.svelte
index 125ba4f0..a99170f2 100644
--- a/frontend/src/routes/search/ResultLink.svelte
+++ b/frontend/src/routes/search/ResultLink.svelte
@@ -1,5 +1,3 @@
-<!-- Template to simplify the logic of result links, checking the results in new tab setting -->
-
<script lang="ts">
import { resultsInNewTab } from '$lib/stores';
import { improvements } from '$lib/improvements';
@@ -38,7 +36,7 @@
{title}
use:improvements={resultIndex}
target={$resultsInNewTab ? '_blank' : null}
- rel={$resultsInNewTab ? 'noopener' : null}
+ rel="noopener nofollow"
bind:this={link}
>
<slot />
@@ -49,7 +47,7 @@
class={_class}
{title}
target={$resultsInNewTab ? '_blank' : null}
- rel={$resultsInNewTab ? 'noopener' : null}
+ rel="noopener nofollow"
bind:this={link}
>
<slot />