web search done right
correctly name host_centrality_store_path
Mikkel Denker73a46ae0d845
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex aa388cdb..b2a85aea 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -63,7 +63,7 @@ impl SearchService {.entity_index_path.map(|path| EntityIndex::open(path).unwrap());let centrality_store = config- .centrality_store_path+ .host_centrality_store_path.map(|p| InboundSimilarity::open(Path::new(&p).join("inbound_similarity")).unwrap());let search_index = Index::open(config.index_path)?;diff --git a/core/src/feed/mod.rs b/core/src/feed/mod.rsnew file mode 100644index 00000000..7b6ee52c--- /dev/null+++ b/core/src/feed/mod.rs@@ -0,0 +1,215 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::path::Path;++use crate::tokenizer::{SiteOperatorUrlTokenizer, Tokenizer};+use anyhow::Result;+use tantivy::{+ query::{PhraseQuery, TermQuery},+ schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},+ tokenizer::Tokenizer as TantivyTokenizer,+};+use url::Url;++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]+pub enum FeedKind {+ Atom,+ Rss,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]+pub struct Feed {+ url: Url,+ kind: FeedKind,+}++struct Index {+ tantivy_index: tantivy::Index,+ writer: tantivy::IndexWriter,+ reader: tantivy::IndexReader,+ schema: tantivy::schema::Schema,+}++impl Index {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }++ let url_tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);+ let kind_tokenizer = Tokenizer::default();++ let mut builder = tantivy::schema::Schema::builder();++ builder.add_text_field(+ "url",+ TextOptions::default()+ .set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer(url_tokenizer.as_str())+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ )+ .set_stored(),+ );+ builder.add_text_field(+ "kind",+ TextOptions::default()+ .set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer(kind_tokenizer.as_str())+ .set_index_option(IndexRecordOption::Basic),+ )+ .set_stored(),+ );++ let schema = builder.build();++ let tv_index = tantivy::Index::create_in_dir(path, schema.clone())?;+ tv_index+ .tokenizers()+ .register(url_tokenizer.as_str(), url_tokenizer);++ tv_index+ .tokenizers()+ .register(kind_tokenizer.as_str(), kind_tokenizer);++ let writer = tv_index.writer(50_000_000)?;++ let reader = tv_index.reader()?;++ Ok(Self {+ tantivy_index: tv_index,+ writer,+ reader,+ schema,+ })+ }++ pub fn insert(&mut self, feed: &Feed) -> Result<()> {+ let mut doc = tantivy::TantivyDocument::default();++ doc.add_text(self.schema.get_field("url")?, feed.url.as_str());+ doc.add_text(+ self.schema.get_field("kind")?,+ &serde_json::to_string(&feed.kind)?,+ );++ self.writer.add_document(doc)?;++ Ok(())+ }++ pub fn commit(&mut self) -> Result<()> {+ self.writer.commit()?;+ self.reader.reload()?;++ Ok(())+ }++ fn terms(&self, query: &str) -> Vec<tantivy::Term> {+ let mut tokenizer = SiteOperatorUrlTokenizer;+ let mut res = Vec::new();+ let tv_field = self.schema.get_field("url").unwrap();++ let mut stream = tokenizer.token_stream(query);++ while let Some(token) = stream.next() {+ res.push(tantivy::Term::from_field_text(tv_field, &token.text));+ }++ res+ }++ pub fn search(&self, query: &str) -> Result<Vec<Feed>> {+ let searcher = self.reader.searcher();++ let terms = self.terms(query);+ if terms.is_empty() {+ return Ok(Vec::new());+ }++ let query = if terms.len() == 1 {+ let term = terms.get(0).unwrap().clone();+ Box::new(TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ )) as Box<dyn tantivy::query::Query>+ } else {+ Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>+ };++ let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;++ let mut res = Vec::new();++ for address in docs {+ let doc: tantivy::TantivyDocument = searcher.doc(address)?;++ let url = doc+ .get_first(self.schema.get_field("url")?)+ .unwrap()+ .as_ref()+ .as_str()+ .unwrap();++ let url = Url::parse(url)?;++ let kind = doc+ .get_first(self.schema.get_field("kind")?)+ .unwrap()+ .as_ref()+ .as_str()+ .unwrap();++ let kind: FeedKind = serde_json::from_str(kind)?;++ res.push(Feed { url, kind });+ }++ Ok(res)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn feed_index() {+ let mut index = Index::open(crate::gen_temp_path()).unwrap();++ let a = Feed {+ url: Url::parse("https://a.com/feed.xml").unwrap(),+ kind: FeedKind::Atom,+ };+ index.insert(&a).unwrap();++ let b = Feed {+ url: Url::parse("https://b.com/another/feed").unwrap(),+ kind: FeedKind::Rss,+ };+ index.insert(&b).unwrap();++ index.commit().unwrap();++ let res = index.search("a.com").unwrap();+ assert_eq!(res, vec![a]);++ let res = index.search("b.com").unwrap();+ assert_eq!(res, vec![b]);+ }+}diff --git a/core/src/lib.rs b/core/src/lib.rsindex 69bc9cd0..7069c4ca 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -46,6 +46,7 @@ mod entity_index;mod enum_map;mod executor;mod fastfield_reader;+pub mod feed;mod human_website_annotations;pub mod hyperloglog;mod image_downloader;diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 1511b8be..08e282c4 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -667,7 +667,7 @@ pub struct SignalAggregator {query: Option<QueryData>,query_signal_coefficients: Option<SignalCoefficient>,segment_reader: Option<SegmentReader>,- inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,+ inbound_similarity: Option<Arc<inbound_similarity::Scorer>>,fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,query_centrality: Option<Arc<query_centrality::Scorer>>,@@ -683,7 +683,7 @@ impl Clone for SignalAggregator {query: self.query.clone(),query_signal_coefficients: self.query_signal_coefficients.clone(),segment_reader: None,- inbound_similariy: self.inbound_similariy.clone(),+ inbound_similarity: self.inbound_similarity.clone(),fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),update_time_cache: self.update_time_cache.clone(),query_centrality: self.query_centrality.clone(),@@ -738,7 +738,7 @@ impl SignalAggregator {Self {segment_reader: None,- inbound_similariy: None,+ inbound_similarity: None,query_signal_coefficients,fetch_time_ms_cache,update_time_cache,@@ -870,7 +870,7 @@ impl SignalAggregator {}pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {- self.inbound_similariy = Some(Arc::new(scorer));+ self.inbound_similarity = Some(Arc::new(scorer));}pub fn set_region_count(&mut self, region_count: RegionCount) {@@ -896,7 +896,7 @@ impl SignalAggregator {}pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {- self.inbound_similariy+ self.inbound_similarity.as_ref().map(|scorer| scorer.score(&host_id)).unwrap_or_default()diff --git a/core/src/warc.rs b/core/src/warc.rsindex 06af44eb..200429a0 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -250,6 +250,7 @@ impl FromStr for PayloadType {fn from_str(s: &str) -> Result<Self, Self::Err> {match s {"application/html" => Ok(Self::Html),+ "text/html" => Ok(Self::Html),"application/pdf" => Ok(Self::Pdf),"application/rss" => Ok(Self::Rss),"application/atom" => Ok(Self::Atom),@@ -261,7 +262,7 @@ impl FromStr for PayloadType {impl ToString for PayloadType {fn to_string(&self) -> String {match self {- Self::Html => "application/html".to_string(),+ Self::Html => "text/html".to_string(),Self::Pdf => "application/pdf".to_string(),Self::Rss => "application/rss".to_string(),Self::Atom => "application/atom".to_string(),