Skip to content

web search done right

correctly name host_centrality_store_path

Mikkel Denker73a46ae0d845
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index aa388cdb..b2a85aea 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -63,7 +63,7 @@ impl SearchService {
.entity_index_path
.map(|path| EntityIndex::open(path).unwrap());
let centrality_store = config
- .centrality_store_path
+ .host_centrality_store_path
.map(|p| InboundSimilarity::open(Path::new(&p).join("inbound_similarity")).unwrap());
let search_index = Index::open(config.index_path)?;
diff --git a/core/src/feed/mod.rs b/core/src/feed/mod.rs
new file mode 100644
index 00000000..7b6ee52c
--- /dev/null
+++ b/core/src/feed/mod.rs
@@ -0,0 +1,215 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::path::Path;
+
+use crate::tokenizer::{SiteOperatorUrlTokenizer, Tokenizer};
+use anyhow::Result;
+use tantivy::{
+ query::{PhraseQuery, TermQuery},
+ schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},
+ tokenizer::Tokenizer as TantivyTokenizer,
+};
+use url::Url;
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
+pub enum FeedKind {
+ Atom,
+ Rss,
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
+pub struct Feed {
+ url: Url,
+ kind: FeedKind,
+}
+
+struct Index {
+ tantivy_index: tantivy::Index,
+ writer: tantivy::IndexWriter,
+ reader: tantivy::IndexReader,
+ schema: tantivy::schema::Schema,
+}
+
+impl Index {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ if !path.as_ref().exists() {
+ std::fs::create_dir_all(path.as_ref())?;
+ }
+
+ let url_tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);
+ let kind_tokenizer = Tokenizer::default();
+
+ let mut builder = tantivy::schema::Schema::builder();
+
+ builder.add_text_field(
+ "url",
+ TextOptions::default()
+ .set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(url_tokenizer.as_str())
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ )
+ .set_stored(),
+ );
+ builder.add_text_field(
+ "kind",
+ TextOptions::default()
+ .set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(kind_tokenizer.as_str())
+ .set_index_option(IndexRecordOption::Basic),
+ )
+ .set_stored(),
+ );
+
+ let schema = builder.build();
+
+ let tv_index = tantivy::Index::create_in_dir(path, schema.clone())?;
+ tv_index
+ .tokenizers()
+ .register(url_tokenizer.as_str(), url_tokenizer);
+
+ tv_index
+ .tokenizers()
+ .register(kind_tokenizer.as_str(), kind_tokenizer);
+
+ let writer = tv_index.writer(50_000_000)?;
+
+ let reader = tv_index.reader()?;
+
+ Ok(Self {
+ tantivy_index: tv_index,
+ writer,
+ reader,
+ schema,
+ })
+ }
+
+ pub fn insert(&mut self, feed: &Feed) -> Result<()> {
+ let mut doc = tantivy::TantivyDocument::default();
+
+ doc.add_text(self.schema.get_field("url")?, feed.url.as_str());
+ doc.add_text(
+ self.schema.get_field("kind")?,
+ &serde_json::to_string(&feed.kind)?,
+ );
+
+ self.writer.add_document(doc)?;
+
+ Ok(())
+ }
+
+ pub fn commit(&mut self) -> Result<()> {
+ self.writer.commit()?;
+ self.reader.reload()?;
+
+ Ok(())
+ }
+
+ fn terms(&self, query: &str) -> Vec<tantivy::Term> {
+ let mut tokenizer = SiteOperatorUrlTokenizer;
+ let mut res = Vec::new();
+ let tv_field = self.schema.get_field("url").unwrap();
+
+ let mut stream = tokenizer.token_stream(query);
+
+ while let Some(token) = stream.next() {
+ res.push(tantivy::Term::from_field_text(tv_field, &token.text));
+ }
+
+ res
+ }
+
+ pub fn search(&self, query: &str) -> Result<Vec<Feed>> {
+ let searcher = self.reader.searcher();
+
+ let terms = self.terms(query);
+ if terms.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let query = if terms.len() == 1 {
+ let term = terms.get(0).unwrap().clone();
+ Box::new(TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ )) as Box<dyn tantivy::query::Query>
+ } else {
+ Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>
+ };
+
+ let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;
+
+ let mut res = Vec::new();
+
+ for address in docs {
+ let doc: tantivy::TantivyDocument = searcher.doc(address)?;
+
+ let url = doc
+ .get_first(self.schema.get_field("url")?)
+ .unwrap()
+ .as_ref()
+ .as_str()
+ .unwrap();
+
+ let url = Url::parse(url)?;
+
+ let kind = doc
+ .get_first(self.schema.get_field("kind")?)
+ .unwrap()
+ .as_ref()
+ .as_str()
+ .unwrap();
+
+ let kind: FeedKind = serde_json::from_str(kind)?;
+
+ res.push(Feed { url, kind });
+ }
+
+ Ok(res)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn feed_index() {
+ let mut index = Index::open(crate::gen_temp_path()).unwrap();
+
+ let a = Feed {
+ url: Url::parse("https://a.com/feed.xml").unwrap(),
+ kind: FeedKind::Atom,
+ };
+ index.insert(&a).unwrap();
+
+ let b = Feed {
+ url: Url::parse("https://b.com/another/feed").unwrap(),
+ kind: FeedKind::Rss,
+ };
+ index.insert(&b).unwrap();
+
+ index.commit().unwrap();
+
+ let res = index.search("a.com").unwrap();
+ assert_eq!(res, vec![a]);
+
+ let res = index.search("b.com").unwrap();
+ assert_eq!(res, vec![b]);
+ }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 69bc9cd0..7069c4ca 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -46,6 +46,7 @@ mod entity_index;
mod enum_map;
mod executor;
mod fastfield_reader;
+pub mod feed;
mod human_website_annotations;
pub mod hyperloglog;
mod image_downloader;
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 1511b8be..08e282c4 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -667,7 +667,7 @@ pub struct SignalAggregator {
query: Option<QueryData>,
query_signal_coefficients: Option<SignalCoefficient>,
segment_reader: Option<SegmentReader>,
- inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,
+ inbound_similarity: Option<Arc<inbound_similarity::Scorer>>,
fetch_time_ms_cache: Vec<f64>,
update_time_cache: Vec<f64>,
query_centrality: Option<Arc<query_centrality::Scorer>>,
@@ -683,7 +683,7 @@ impl Clone for SignalAggregator {
query: self.query.clone(),
query_signal_coefficients: self.query_signal_coefficients.clone(),
segment_reader: None,
- inbound_similariy: self.inbound_similariy.clone(),
+ inbound_similarity: self.inbound_similarity.clone(),
fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),
update_time_cache: self.update_time_cache.clone(),
query_centrality: self.query_centrality.clone(),
@@ -738,7 +738,7 @@ impl SignalAggregator {
Self {
segment_reader: None,
- inbound_similariy: None,
+ inbound_similarity: None,
query_signal_coefficients,
fetch_time_ms_cache,
update_time_cache,
@@ -870,7 +870,7 @@ impl SignalAggregator {
}
pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {
- self.inbound_similariy = Some(Arc::new(scorer));
+ self.inbound_similarity = Some(Arc::new(scorer));
}
pub fn set_region_count(&mut self, region_count: RegionCount) {
@@ -896,7 +896,7 @@ impl SignalAggregator {
}
pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {
- self.inbound_similariy
+ self.inbound_similarity
.as_ref()
.map(|scorer| scorer.score(&host_id))
.unwrap_or_default()
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 06af44eb..200429a0 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -250,6 +250,7 @@ impl FromStr for PayloadType {
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"application/html" => Ok(Self::Html),
+ "text/html" => Ok(Self::Html),
"application/pdf" => Ok(Self::Pdf),
"application/rss" => Ok(Self::Rss),
"application/atom" => Ok(Self::Atom),
@@ -261,7 +262,7 @@ impl FromStr for PayloadType {
impl ToString for PayloadType {
fn to_string(&self) -> String {
match self {
- Self::Html => "application/html".to_string(),
+ Self::Html => "text/html".to_string(),
Self::Pdf => "application/pdf".to_string(),
Self::Rss => "application/rss".to_string(),
Self::Atom => "application/atom".to_string(),