web search done right
[webgraph] query to find all edges between two nodes
Mikkel Denker2fd7f31baee4
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 09c0291a..6ac81727 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -44,6 +44,7 @@ pub mod schema;mod searcher;mod shortest_path;mod store;+pub mod tokenizer;mod warmed_column_fields;#[cfg(test)]diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rsnew file mode 100644index 00000000..25847f58--- /dev/null+++ b/crates/core/src/webgraph/query/between.rs@@ -0,0 +1,136 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::query::BooleanQuery;++use crate::{+ ampc::dht::ShardId,+ webgraph::{+ schema::{FromUrl, ToUrl},+ searcher::Searcher,+ Edge, EdgeLimit, Node, Query,+ },+ Result,+};++use super::{collector::TopDocsCollector, fetch_edges, raw::PhraseOrTermQuery};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct FullLinksBetweenQuery {+ from: Node,+ to: Node,+ limit: EdgeLimit,+}++impl FullLinksBetweenQuery {+ pub fn new(from: Node, to: Node) -> Self {+ Self {+ from,+ to,+ limit: EdgeLimit::Unlimited,+ }+ }++ pub fn with_limit(mut self, limit: EdgeLimit) -> Self {+ self.limit = limit;+ self+ }+}++impl Query for FullLinksBetweenQuery {+ type Collector = TopDocsCollector;+ type TantivyQuery = BooleanQuery;+ type IntermediateOutput = Vec<Edge>;+ type Output = Vec<Edge>;++ fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {+ let from_query = PhraseOrTermQuery::new(self.from.as_str().to_string(), FromUrl);+ let to_query = PhraseOrTermQuery::new(self.to.as_str().to_string(), ToUrl);++ BooleanQuery::intersection(vec![Box::new(from_query), Box::new(to_query)])+ }++ fn collector(&self, searcher: &Searcher) -> Self::Collector {+ TopDocsCollector::from(self.limit)+ .with_shard_id(searcher.shard())+ .disable_offset()+ }++ fn remote_collector(&self) -> Self::Collector {+ TopDocsCollector::from(self.limit).enable_offset()+ }++ fn filter_fruit_shards(+ &self,+ shard_id: ShardId,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> <Self::Collector as super::Collector>::Fruit {+ fruit+ .into_iter()+ .filter(|(_, doc_address)| doc_address.shard_id == shard_id)+ .collect()+ }++ fn retrieve(+ &self,+ searcher: &Searcher,+ fruit: <Self::Collector as super::collector::Collector>::Fruit,+ ) -> Result<Self::IntermediateOutput> {+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();+ let edges = fetch_edges(searcher, docs)?;+ Ok(edges)+ }++ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {+ let mut edges: Vec<_> = results.into_iter().flatten().collect();+ edges.sort_by(|a, b| b.sort_score.total_cmp(&a.sort_score));+ edges+ }+}++#[cfg(test)]+mod tests {+ use crate::{webgraph::Webgraph, webpage::RelFlags};++ use super::*;++ #[test]+ fn test_between() {+ let from = Node::from("https://example.com");+ let to = Node::from("https://example.org");++ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ graph+ .insert(Edge {+ from: from.clone(),+ to: to.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();+ graph.commit().unwrap();++ let res = graph+ .search(&FullLinksBetweenQuery::new(from.clone(), to.clone()))+ .unwrap();+ assert_eq!(res.len(), 1);+ assert_eq!(res[0].from, from);+ assert_eq!(res[0].to, to);+ }+}diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rsindex bc294125..01a287e4 100644--- a/crates/core/src/webgraph/query/mod.rs+++ b/crates/core/src/webgraph/query/mod.rs@@ -30,6 +30,9 @@ pub use id2node::*;pub mod group_by;pub use group_by::*;+pub mod between;+pub use between::*;+pub mod collector;mod degree;mod document_scorer;diff --git a/crates/core/src/webgraph/query/raw/mod.rs b/crates/core/src/webgraph/query/raw/mod.rsindex 5087efa2..5c940356 100644--- a/crates/core/src/webgraph/query/raw/mod.rs+++ b/crates/core/src/webgraph/query/raw/mod.rs@@ -20,7 +20,9 @@ pub use dummy::DummyQuery;pub mod host_links;pub mod id2node;pub mod links;+pub mod phrase_or_term;pub use host_links::HostLinksQuery;pub use id2node::Id2NodeQuery;pub use links::LinksQuery;+pub use phrase_or_term::PhraseOrTermQuery;diff --git a/crates/core/src/webgraph/query/raw/phrase_or_term.rs b/crates/core/src/webgraph/query/raw/phrase_or_term.rsnew file mode 100644index 00000000..718c8578--- /dev/null+++ b/crates/core/src/webgraph/query/raw/phrase_or_term.rs@@ -0,0 +1,94 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::query::{BooleanQuery, Query, Weight};++use crate::webgraph::{+ schema::{Field, FieldEnum},+ tokenizer::Tokenizer,+};++#[derive(Clone, Debug)]+pub struct PhraseOrTermQuery {+ text: String,+ field: FieldEnum,+}++impl PhraseOrTermQuery {+ pub fn new<F: Field>(text: String, field: F) -> Self {+ Self {+ text,+ field: field.into(),+ }+ }+}++impl Query for PhraseOrTermQuery {+ fn weight(+ &self,+ scoring: tantivy::query::EnableScoring<'_>,+ ) -> tantivy::Result<Box<dyn Weight>> {+ let schema = scoring.schema();++ let tv_field = schema.get_field(self.field.name())?;+ let mut tokenizer = self.field.tokenizer();+ let mut token_stream = tokenizer.token_stream(&self.text);++ let mut terms = Vec::new();++ while token_stream.advance() {+ terms.push(tantivy::Term::from_field_text(+ tv_field,+ &token_stream.token().text,+ ));+ }++ if terms.is_empty() {+ Ok(Box::new(tantivy::query::EmptyWeight))+ } else if terms.len() == 1 {+ tantivy::query::TermQuery::new(+ terms.remove(0),+ tantivy::schema::IndexRecordOption::Basic,+ )+ .weight(scoring)+ } else {+ let index_record_option = schema+ .get_field_entry(tv_field)+ .field_type()+ .index_record_option()+ .unwrap();++ match index_record_option {+ tantivy::schema::IndexRecordOption::WithFreqsAndPositions => {+ tantivy::query::PhraseQuery::new(terms).weight(scoring)+ }+ _ => {+ let queries = terms+ .into_iter()+ .map(|term| {+ Box::new(tantivy::query::TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ )) as Box<dyn Query>+ })+ .collect();++ BooleanQuery::intersection(queries).weight(scoring)+ }+ }+ }+ }+}diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rsindex 1e3bcb63..baaef16a 100644--- a/crates/core/src/webgraph/schema.rs+++ b/crates/core/src/webgraph/schema.rs@@ -19,6 +19,7 @@ use strum::{EnumDiscriminants, VariantArray};use tantivy::schema::FieldType;use tantivy::schema::NumericOptions;use tantivy::schema::OwnedValue;+use tantivy::schema::TextFieldIndexing;use tantivy::schema::TextOptions;use tantivy::schema::Value;@@ -26,6 +27,8 @@ use crate::enum_dispatch_from_discriminant;use crate::Result;use super::document::{Edge, ReferenceValue};+use super::tokenizer::Tokenizer;+use super::tokenizer::TokenizerEnum;use super::Node;pub fn create_schema() -> tantivy::schema::Schema {@@ -49,6 +52,9 @@ pub trait Field:fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a>;fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()>;fn field_type(&self) -> tantivy::schema::FieldType;+ fn tokenizer(&self) -> TokenizerEnum {+ TokenizerEnum::default()+ }}#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]@@ -72,7 +78,13 @@ impl Field for FromUrl {}fn field_type(&self) -> FieldType {- FieldType::Str(TextOptions::default().set_stored())+ FieldType::Str(+ TextOptions::default().set_stored().set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer(self.tokenizer().name())+ .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions),+ ),+ )}}@@ -97,7 +109,13 @@ impl Field for ToUrl {}fn field_type(&self) -> FieldType {- FieldType::Str(TextOptions::default().set_stored())+ FieldType::Str(+ TextOptions::default().set_stored().set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer(self.tokenizer().name())+ .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions),+ ),+ )}}@@ -253,7 +271,13 @@ impl Field for Label {}fn field_type(&self) -> FieldType {- FieldType::Str(TextOptions::default().set_stored())+ FieldType::Str(+ TextOptions::default().set_stored().set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer(self.tokenizer().name())+ .set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions),+ ),+ )}}diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex ce55660a..f2c32c07 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -26,6 +26,7 @@ use super::{query::collector::{Collector, TantivyCollector},schema::{self, create_schema, Field, FromHostId, FromId, ToHostId, ToId},searcher::Searcher,+ tokenizer::{Tokenizer, TokenizerEnum},warmed_column_fields::WarmedColumnFields,Edge,};@@ -33,11 +34,18 @@ use crate::{ampc::dht::ShardId, webpage::html::links::RelFlags, Result};use itertools::Itertools;use rustc_hash::FxHashSet;use tantivy::{- columnar::Column, directory::MmapDirectory, indexer::NoMergePolicy, DocId, SegmentReader,+ columnar::Column, directory::MmapDirectory, indexer::NoMergePolicy,+ tokenizer::TokenizerManager, DocId, SegmentReader,};use super::{query::Query, NodeID};+fn register_tokenizers(manager: &TokenizerManager) {+ for tokenizer in TokenizerEnum::iter() {+ manager.register(tokenizer.name(), tokenizer.into_tantivy());+ }+}+pub struct EdgeStore {index: tantivy::Index,writer: Option<tantivy::IndexWriter<Edge>>,@@ -65,6 +73,8 @@ impl EdgeStore {}).open_or_create(MmapDirectory::open(&path)?)?;+ register_tokenizers(index.tokenizers());+let warmed_column_fields = WarmedColumnFields::new(&index.reader()?.searcher())?;Ok(Self {diff --git a/crates/core/src/webgraph/tokenizer.rs b/crates/core/src/webgraph/tokenizer.rsnew file mode 100644index 00000000..45d393d2--- /dev/null+++ b/crates/core/src/webgraph/tokenizer.rs@@ -0,0 +1,91 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use enum_dispatch::enum_dispatch;+use strum::{EnumDiscriminants, VariantArray};+use tantivy::tokenizer::{BoxTokenStream, Tokenizer as _};++#[enum_dispatch]+pub trait Tokenizer: Into<TokenizerEnum> + Clone {+ fn name(&self) -> &'static str;+ fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a>;++ fn into_tantivy(self) -> TantivyBridge {+ TantivyBridge::from(self.into())+ }+}++#[derive(Clone, Default)]+pub struct SimpleTokenizer(tantivy::tokenizer::SimpleTokenizer);++impl Tokenizer for SimpleTokenizer {+ fn name(&self) -> &'static str {+ "simple"+ }++ fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {+ BoxTokenStream::new(self.0.token_stream(text))+ }+}++#[enum_dispatch(Tokenizer)]+#[derive(Clone, EnumDiscriminants)]+#[strum_discriminants(derive(VariantArray))]+pub enum TokenizerEnum {+ SimpleTokenizer,+}++impl TokenizerEnum {+ pub fn iter() -> impl Iterator<Item = Self> {+ TokenizerEnumDiscriminants::VARIANTS+ .iter()+ .map(|d| (*d).into())+ }+}++impl Default for TokenizerEnum {+ fn default() -> Self {+ TokenizerEnum::SimpleTokenizer(SimpleTokenizer::default())+ }+}++impl From<TokenizerEnumDiscriminants> for TokenizerEnum {+ fn from(value: TokenizerEnumDiscriminants) -> Self {+ match value {+ TokenizerEnumDiscriminants::SimpleTokenizer => SimpleTokenizer::default().into(),+ }+ }+}++#[derive(Clone)]+pub struct TantivyBridge(TokenizerEnum);++impl<T> From<T> for TantivyBridge+where+ T: Tokenizer + Sync + Send + 'static,+{+ fn from(value: T) -> Self {+ Self(value.into())+ }+}++impl tantivy::tokenizer::Tokenizer for TantivyBridge {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ self.0.token_stream(text)+ }+}