Skip to content

web search done right

[webgraph] query to find all edges between two nodes

Mikkel Denker2fd7f31baee4
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 09c0291a..6ac81727 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -44,6 +44,7 @@ pub mod schema;
mod searcher;
mod shortest_path;
mod store;
+pub mod tokenizer;
mod warmed_column_fields;
#[cfg(test)]
diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rs
new file mode 100644
index 00000000..25847f58
--- /dev/null
+++ b/crates/core/src/webgraph/query/between.rs
@@ -0,0 +1,136 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::query::BooleanQuery;
+
+use crate::{
+ ampc::dht::ShardId,
+ webgraph::{
+ schema::{FromUrl, ToUrl},
+ searcher::Searcher,
+ Edge, EdgeLimit, Node, Query,
+ },
+ Result,
+};
+
+use super::{collector::TopDocsCollector, fetch_edges, raw::PhraseOrTermQuery};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct FullLinksBetweenQuery {
+ from: Node,
+ to: Node,
+ limit: EdgeLimit,
+}
+
+impl FullLinksBetweenQuery {
+ pub fn new(from: Node, to: Node) -> Self {
+ Self {
+ from,
+ to,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(mut self, limit: EdgeLimit) -> Self {
+ self.limit = limit;
+ self
+ }
+}
+
+impl Query for FullLinksBetweenQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = BooleanQuery;
+ type IntermediateOutput = Vec<Edge>;
+ type Output = Vec<Edge>;
+
+ fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
+ let from_query = PhraseOrTermQuery::new(self.from.as_str().to_string(), FromUrl);
+ let to_query = PhraseOrTermQuery::new(self.to.as_str().to_string(), ToUrl);
+
+ BooleanQuery::intersection(vec![Box::new(from_query), Box::new(to_query)])
+ }
+
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(searcher.shard())
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc_address)| doc_address.shard_id == shard_id)
+ .collect()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
+ let edges = fetch_edges(searcher, docs)?;
+ Ok(edges)
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|a, b| b.sort_score.total_cmp(&a.sort_score));
+ edges
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{webgraph::Webgraph, webpage::RelFlags};
+
+ use super::*;
+
+ #[test]
+ fn test_between() {
+ let from = Node::from("https://example.com");
+ let to = Node::from("https://example.org");
+
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ graph
+ .insert(Edge {
+ from: from.clone(),
+ to: to.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+
+ let res = graph
+ .search(&FullLinksBetweenQuery::new(from.clone(), to.clone()))
+ .unwrap();
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, from);
+ assert_eq!(res[0].to, to);
+ }
+}
diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rs
index bc294125..01a287e4 100644
--- a/crates/core/src/webgraph/query/mod.rs
+++ b/crates/core/src/webgraph/query/mod.rs
@@ -30,6 +30,9 @@ pub use id2node::*;
pub mod group_by;
pub use group_by::*;
+pub mod between;
+pub use between::*;
+
pub mod collector;
mod degree;
mod document_scorer;
diff --git a/crates/core/src/webgraph/query/raw/mod.rs b/crates/core/src/webgraph/query/raw/mod.rs
index 5087efa2..5c940356 100644
--- a/crates/core/src/webgraph/query/raw/mod.rs
+++ b/crates/core/src/webgraph/query/raw/mod.rs
@@ -20,7 +20,9 @@ pub use dummy::DummyQuery;
pub mod host_links;
pub mod id2node;
pub mod links;
+pub mod phrase_or_term;
pub use host_links::HostLinksQuery;
pub use id2node::Id2NodeQuery;
pub use links::LinksQuery;
+pub use phrase_or_term::PhraseOrTermQuery;
diff --git a/crates/core/src/webgraph/query/raw/phrase_or_term.rs b/crates/core/src/webgraph/query/raw/phrase_or_term.rs
new file mode 100644
index 00000000..718c8578
--- /dev/null
+++ b/crates/core/src/webgraph/query/raw/phrase_or_term.rs
@@ -0,0 +1,94 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::query::{BooleanQuery, Query, Weight};
+
+use crate::webgraph::{
+ schema::{Field, FieldEnum},
+ tokenizer::Tokenizer,
+};
+
+#[derive(Clone, Debug)]
+pub struct PhraseOrTermQuery {
+ text: String,
+ field: FieldEnum,
+}
+
+impl PhraseOrTermQuery {
+ pub fn new<F: Field>(text: String, field: F) -> Self {
+ Self {
+ text,
+ field: field.into(),
+ }
+ }
+}
+
+impl Query for PhraseOrTermQuery {
+ fn weight(
+ &self,
+ scoring: tantivy::query::EnableScoring<'_>,
+ ) -> tantivy::Result<Box<dyn Weight>> {
+ let schema = scoring.schema();
+
+ let tv_field = schema.get_field(self.field.name())?;
+ let mut tokenizer = self.field.tokenizer();
+ let mut token_stream = tokenizer.token_stream(&self.text);
+
+ let mut terms = Vec::new();
+
+ while token_stream.advance() {
+ terms.push(tantivy::Term::from_field_text(
+ tv_field,
+ &token_stream.token().text,
+ ));
+ }
+
+ if terms.is_empty() {
+ Ok(Box::new(tantivy::query::EmptyWeight))
+ } else if terms.len() == 1 {
+ tantivy::query::TermQuery::new(
+ terms.remove(0),
+ tantivy::schema::IndexRecordOption::Basic,
+ )
+ .weight(scoring)
+ } else {
+ let index_record_option = schema
+ .get_field_entry(tv_field)
+ .field_type()
+ .index_record_option()
+ .unwrap();
+
+ match index_record_option {
+ tantivy::schema::IndexRecordOption::WithFreqsAndPositions => {
+ tantivy::query::PhraseQuery::new(terms).weight(scoring)
+ }
+ _ => {
+ let queries = terms
+ .into_iter()
+ .map(|term| {
+ Box::new(tantivy::query::TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ )) as Box<dyn Query>
+ })
+ .collect();
+
+ BooleanQuery::intersection(queries).weight(scoring)
+ }
+ }
+ }
+ }
+}
diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rs
index 1e3bcb63..baaef16a 100644
--- a/crates/core/src/webgraph/schema.rs
+++ b/crates/core/src/webgraph/schema.rs
@@ -19,6 +19,7 @@ use strum::{EnumDiscriminants, VariantArray};
use tantivy::schema::FieldType;
use tantivy::schema::NumericOptions;
use tantivy::schema::OwnedValue;
+use tantivy::schema::TextFieldIndexing;
use tantivy::schema::TextOptions;
use tantivy::schema::Value;
@@ -26,6 +27,8 @@ use crate::enum_dispatch_from_discriminant;
use crate::Result;
use super::document::{Edge, ReferenceValue};
+use super::tokenizer::Tokenizer;
+use super::tokenizer::TokenizerEnum;
use super::Node;
pub fn create_schema() -> tantivy::schema::Schema {
@@ -49,6 +52,9 @@ pub trait Field:
fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a>;
fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()>;
fn field_type(&self) -> tantivy::schema::FieldType;
+ fn tokenizer(&self) -> TokenizerEnum {
+ TokenizerEnum::default()
+ }
}
#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
@@ -72,7 +78,13 @@ impl Field for FromUrl {
}
fn field_type(&self) -> FieldType {
- FieldType::Str(TextOptions::default().set_stored())
+ FieldType::Str(
+ TextOptions::default().set_stored().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(self.tokenizer().name())
+ .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions),
+ ),
+ )
}
}
@@ -97,7 +109,13 @@ impl Field for ToUrl {
}
fn field_type(&self) -> FieldType {
- FieldType::Str(TextOptions::default().set_stored())
+ FieldType::Str(
+ TextOptions::default().set_stored().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(self.tokenizer().name())
+ .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions),
+ ),
+ )
}
}
@@ -253,7 +271,13 @@ impl Field for Label {
}
fn field_type(&self) -> FieldType {
- FieldType::Str(TextOptions::default().set_stored())
+ FieldType::Str(
+ TextOptions::default().set_stored().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(self.tokenizer().name())
+ .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions),
+ ),
+ )
}
}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index ce55660a..f2c32c07 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -26,6 +26,7 @@ use super::{
query::collector::{Collector, TantivyCollector},
schema::{self, create_schema, Field, FromHostId, FromId, ToHostId, ToId},
searcher::Searcher,
+ tokenizer::{Tokenizer, TokenizerEnum},
warmed_column_fields::WarmedColumnFields,
Edge,
};
@@ -33,11 +34,18 @@ use crate::{ampc::dht::ShardId, webpage::html::links::RelFlags, Result};
use itertools::Itertools;
use rustc_hash::FxHashSet;
use tantivy::{
- columnar::Column, directory::MmapDirectory, indexer::NoMergePolicy, DocId, SegmentReader,
+ columnar::Column, directory::MmapDirectory, indexer::NoMergePolicy,
+ tokenizer::TokenizerManager, DocId, SegmentReader,
};
use super::{query::Query, NodeID};
+fn register_tokenizers(manager: &TokenizerManager) {
+ for tokenizer in TokenizerEnum::iter() {
+ manager.register(tokenizer.name(), tokenizer.into_tantivy());
+ }
+}
+
pub struct EdgeStore {
index: tantivy::Index,
writer: Option<tantivy::IndexWriter<Edge>>,
@@ -65,6 +73,8 @@ impl EdgeStore {
})
.open_or_create(MmapDirectory::open(&path)?)?;
+ register_tokenizers(index.tokenizers());
+
let warmed_column_fields = WarmedColumnFields::new(&index.reader()?.searcher())?;
Ok(Self {
diff --git a/crates/core/src/webgraph/tokenizer.rs b/crates/core/src/webgraph/tokenizer.rs
new file mode 100644
index 00000000..45d393d2
--- /dev/null
+++ b/crates/core/src/webgraph/tokenizer.rs
@@ -0,0 +1,91 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use enum_dispatch::enum_dispatch;
+use strum::{EnumDiscriminants, VariantArray};
+use tantivy::tokenizer::{BoxTokenStream, Tokenizer as _};
+
+#[enum_dispatch]
+pub trait Tokenizer: Into<TokenizerEnum> + Clone {
+ fn name(&self) -> &'static str;
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a>;
+
+ fn into_tantivy(self) -> TantivyBridge {
+ TantivyBridge::from(self.into())
+ }
+}
+
+#[derive(Clone, Default)]
+pub struct SimpleTokenizer(tantivy::tokenizer::SimpleTokenizer);
+
+impl Tokenizer for SimpleTokenizer {
+ fn name(&self) -> &'static str {
+ "simple"
+ }
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
+ BoxTokenStream::new(self.0.token_stream(text))
+ }
+}
+
+#[enum_dispatch(Tokenizer)]
+#[derive(Clone, EnumDiscriminants)]
+#[strum_discriminants(derive(VariantArray))]
+pub enum TokenizerEnum {
+ SimpleTokenizer,
+}
+
+impl TokenizerEnum {
+ pub fn iter() -> impl Iterator<Item = Self> {
+ TokenizerEnumDiscriminants::VARIANTS
+ .iter()
+ .map(|d| (*d).into())
+ }
+}
+
+impl Default for TokenizerEnum {
+ fn default() -> Self {
+ TokenizerEnum::SimpleTokenizer(SimpleTokenizer::default())
+ }
+}
+
+impl From<TokenizerEnumDiscriminants> for TokenizerEnum {
+ fn from(value: TokenizerEnumDiscriminants) -> Self {
+ match value {
+ TokenizerEnumDiscriminants::SimpleTokenizer => SimpleTokenizer::default().into(),
+ }
+ }
+}
+
+#[derive(Clone)]
+pub struct TantivyBridge(TokenizerEnum);
+
+impl<T> From<T> for TantivyBridge
+where
+ T: Tokenizer + Sync + Send + 'static,
+{
+ fn from(value: T) -> Self {
+ Self(value.into())
+ }
+}
+
+impl tantivy::tokenizer::Tokenizer for TantivyBridge {
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+ self.0.token_stream(text)
+ }
+}