Skip to content

web search done right

parse rss and atom feeds

Mikkel Denker0cfd7701fa37
diff --git a/core/src/entrypoint/api.rs b/core/src/entrypoint/api.rs
index 9a6231fb..56b88f19 100644
--- a/core/src/entrypoint/api.rs
+++ b/core/src/entrypoint/api.rs
@@ -34,7 +34,7 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {
let group = registry
.new_group(
- "search_requests".to_string(),
+ "stract_search_requests".to_string(),
Some("Total number of incoming search requests.".to_string()),
)
.unwrap();
@@ -56,7 +56,7 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {
let group = registry
.new_group(
- "explore_requests".to_string(),
+ "stract_explore_requests".to_string(),
Some("Total number of incoming requests to explore api.".to_string()),
)
.unwrap();
@@ -64,7 +64,7 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {
let group = registry
.new_group(
- "daily_active_users".to_string(),
+ "stract_daily_active_users".to_string(),
Some("Approximate number of unique daily active users.".to_string()),
)
.unwrap();
diff --git a/core/src/feed/index.rs b/core/src/feed/index.rs
new file mode 100644
index 00000000..4547f493
--- /dev/null
+++ b/core/src/feed/index.rs
@@ -0,0 +1,203 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::path::Path;
+
+use crate::tokenizer::{SiteOperatorUrlTokenizer, Tokenizer};
+use anyhow::Result;
+use tantivy::{
+ query::{PhraseQuery, TermQuery},
+ schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},
+ tokenizer::Tokenizer as TantivyTokenizer,
+};
+use url::Url;
+
+use super::{Feed, FeedKind};
+
+pub struct FeedIndex {
+ writer: tantivy::IndexWriter,
+ reader: tantivy::IndexReader,
+ schema: tantivy::schema::Schema,
+}
+
+impl FeedIndex {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ if !path.as_ref().exists() {
+ std::fs::create_dir_all(path.as_ref())?;
+ }
+
+ let url_tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);
+ let kind_tokenizer = Tokenizer::default();
+
+ let mut builder = tantivy::schema::Schema::builder();
+
+ builder.add_text_field(
+ "url",
+ TextOptions::default()
+ .set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(url_tokenizer.as_str())
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ )
+ .set_stored(),
+ );
+ builder.add_text_field(
+ "kind",
+ TextOptions::default()
+ .set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(kind_tokenizer.as_str())
+ .set_index_option(IndexRecordOption::Basic),
+ )
+ .set_stored(),
+ );
+
+ let schema = builder.build();
+
+ let tv_index = tantivy::Index::create_in_dir(path, schema.clone())?;
+ tv_index
+ .tokenizers()
+ .register(url_tokenizer.as_str(), url_tokenizer);
+
+ tv_index
+ .tokenizers()
+ .register(kind_tokenizer.as_str(), kind_tokenizer);
+
+ let writer = tv_index.writer(50_000_000)?;
+
+ let reader = tv_index.reader()?;
+
+ Ok(Self {
+ writer,
+ reader,
+ schema,
+ })
+ }
+
+ pub fn insert(&mut self, feed: &Feed) -> Result<()> {
+ let mut doc = tantivy::TantivyDocument::default();
+
+ doc.add_text(self.schema.get_field("url")?, feed.url.as_str());
+ doc.add_text(
+ self.schema.get_field("kind")?,
+ &serde_json::to_string(&feed.kind)?,
+ );
+
+ self.writer.add_document(doc)?;
+
+ Ok(())
+ }
+
+ pub fn commit(&mut self) -> Result<()> {
+ self.writer.commit()?;
+ self.reader.reload()?;
+
+ Ok(())
+ }
+
+ fn terms(&self, query: &str) -> Vec<tantivy::Term> {
+ let mut tokenizer = SiteOperatorUrlTokenizer;
+ let mut res = Vec::new();
+ let tv_field = self.schema.get_field("url").unwrap();
+
+ let mut stream = tokenizer.token_stream(query);
+
+ while let Some(token) = stream.next() {
+ res.push(tantivy::Term::from_field_text(tv_field, &token.text));
+ }
+
+ res
+ }
+
+ pub fn search(&self, query: &str) -> Result<Vec<Feed>> {
+ let searcher = self.reader.searcher();
+
+ let terms = self.terms(query);
+ if terms.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let query = if terms.len() == 1 {
+ let term = terms.first().unwrap().clone();
+ Box::new(TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ )) as Box<dyn tantivy::query::Query>
+ } else {
+ Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>
+ };
+
+ let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;
+
+ let mut res = Vec::new();
+
+ for address in docs {
+ let doc: tantivy::TantivyDocument = searcher.doc(address)?;
+
+ let url = doc
+ .get_first(self.schema.get_field("url")?)
+ .unwrap()
+ .as_ref()
+ .as_str()
+ .unwrap();
+
+ let url = Url::parse(url)?;
+
+ let kind = doc
+ .get_first(self.schema.get_field("kind")?)
+ .unwrap()
+ .as_ref()
+ .as_str()
+ .unwrap();
+
+ let kind: FeedKind = serde_json::from_str(kind)?;
+
+ res.push(Feed { url, kind });
+ }
+
+ Ok(res)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn feed_index() {
+ let mut index = FeedIndex::open(crate::gen_temp_path()).unwrap();
+
+ let a = Feed {
+ url: Url::parse("https://a.com/feed.xml").unwrap(),
+ kind: FeedKind::Atom,
+ };
+ index.insert(&a).unwrap();
+
+ let b = Feed {
+ url: Url::parse("https://b.com/another/feed").unwrap(),
+ kind: FeedKind::Rss,
+ };
+ index.insert(&b).unwrap();
+
+ index.commit().unwrap();
+
+ let res = index.search("a.com").unwrap();
+ assert_eq!(res, vec![a]);
+
+ let res = index.search("b.com").unwrap();
+ assert_eq!(res, vec![b]);
+ }
+}
diff --git a/core/src/feed/mod.rs b/core/src/feed/mod.rs
index e7e69c28..284c2451 100644
--- a/core/src/feed/mod.rs
+++ b/core/src/feed/mod.rs
@@ -14,20 +14,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::path::Path;
-
-use crate::tokenizer::{SiteOperatorUrlTokenizer, Tokenizer};
-use anyhow::Result;
-use tantivy::{
- query::{PhraseQuery, TermQuery},
- schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},
- tokenizer::Tokenizer as TantivyTokenizer,
-};
use url::Url;
+pub mod index;
+mod parser;
pub mod scheduler;
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
+pub use parser::parse;
+
+#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
pub enum FeedKind {
Atom,
Rss,
@@ -39,177 +34,6 @@ pub struct Feed {
kind: FeedKind,
}
-pub struct FeedIndex {
- writer: tantivy::IndexWriter,
- reader: tantivy::IndexReader,
- schema: tantivy::schema::Schema,
-}
-
-impl FeedIndex {
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- if !path.as_ref().exists() {
- std::fs::create_dir_all(path.as_ref())?;
- }
-
- let url_tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);
- let kind_tokenizer = Tokenizer::default();
-
- let mut builder = tantivy::schema::Schema::builder();
-
- builder.add_text_field(
- "url",
- TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(url_tokenizer.as_str())
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- )
- .set_stored(),
- );
- builder.add_text_field(
- "kind",
- TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(kind_tokenizer.as_str())
- .set_index_option(IndexRecordOption::Basic),
- )
- .set_stored(),
- );
-
- let schema = builder.build();
-
- let tv_index = tantivy::Index::create_in_dir(path, schema.clone())?;
- tv_index
- .tokenizers()
- .register(url_tokenizer.as_str(), url_tokenizer);
-
- tv_index
- .tokenizers()
- .register(kind_tokenizer.as_str(), kind_tokenizer);
-
- let writer = tv_index.writer(50_000_000)?;
-
- let reader = tv_index.reader()?;
-
- Ok(Self {
- writer,
- reader,
- schema,
- })
- }
-
- pub fn insert(&mut self, feed: &Feed) -> Result<()> {
- let mut doc = tantivy::TantivyDocument::default();
-
- doc.add_text(self.schema.get_field("url")?, feed.url.as_str());
- doc.add_text(
- self.schema.get_field("kind")?,
- &serde_json::to_string(&feed.kind)?,
- );
-
- self.writer.add_document(doc)?;
-
- Ok(())
- }
-
- pub fn commit(&mut self) -> Result<()> {
- self.writer.commit()?;
- self.reader.reload()?;
-
- Ok(())
- }
-
- fn terms(&self, query: &str) -> Vec<tantivy::Term> {
- let mut tokenizer = SiteOperatorUrlTokenizer;
- let mut res = Vec::new();
- let tv_field = self.schema.get_field("url").unwrap();
-
- let mut stream = tokenizer.token_stream(query);
-
- while let Some(token) = stream.next() {
- res.push(tantivy::Term::from_field_text(tv_field, &token.text));
- }
-
- res
- }
-
- pub fn search(&self, query: &str) -> Result<Vec<Feed>> {
- let searcher = self.reader.searcher();
-
- let terms = self.terms(query);
- if terms.is_empty() {
- return Ok(Vec::new());
- }
-
- let query = if terms.len() == 1 {
- let term = terms.get(0).unwrap().clone();
- Box::new(TermQuery::new(
- term,
- tantivy::schema::IndexRecordOption::Basic,
- )) as Box<dyn tantivy::query::Query>
- } else {
- Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>
- };
-
- let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;
-
- let mut res = Vec::new();
-
- for address in docs {
- let doc: tantivy::TantivyDocument = searcher.doc(address)?;
-
- let url = doc
- .get_first(self.schema.get_field("url")?)
- .unwrap()
- .as_ref()
- .as_str()
- .unwrap();
-
- let url = Url::parse(url)?;
-
- let kind = doc
- .get_first(self.schema.get_field("kind")?)
- .unwrap()
- .as_ref()
- .as_str()
- .unwrap();
-
- let kind: FeedKind = serde_json::from_str(kind)?;
-
- res.push(Feed { url, kind });
- }
-
- Ok(res)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn feed_index() {
- let mut index = FeedIndex::open(crate::gen_temp_path()).unwrap();
-
- let a = Feed {
- url: Url::parse("https://a.com/feed.xml").unwrap(),
- kind: FeedKind::Atom,
- };
- index.insert(&a).unwrap();
-
- let b = Feed {
- url: Url::parse("https://b.com/another/feed").unwrap(),
- kind: FeedKind::Rss,
- };
- index.insert(&b).unwrap();
-
- index.commit().unwrap();
-
- let res = index.search("a.com").unwrap();
- assert_eq!(res, vec![a]);
-
- let res = index.search("b.com").unwrap();
- assert_eq!(res, vec![b]);
- }
+pub struct ParsedFeed {
+ pub links: Vec<Url>,
}
diff --git a/core/src/feed/parser.rs b/core/src/feed/parser.rs
new file mode 100644
index 00000000..de3734fa
--- /dev/null
+++ b/core/src/feed/parser.rs
@@ -0,0 +1,170 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::{anyhow, Result};
+use quick_xml::events::Event;
+use url::Url;
+
+use super::{FeedKind, ParsedFeed};
+
+fn parse_rss(feed: &str) -> Result<ParsedFeed> {
+ let mut reader = quick_xml::Reader::from_str(feed);
+
+ let mut buf = Vec::new();
+ let mut links = Vec::new();
+ let mut inside_link = false;
+ let mut inside_item = false;
+
+ loop {
+ match reader.read_event_into(&mut buf) {
+ Ok(Event::Start(ref e)) if e.name().as_ref() == b"link" => {
+ inside_link = true;
+ }
+ Ok(Event::End(ref e)) if e.name().as_ref() == b"link" => {
+ inside_link = false;
+ }
+ Ok(Event::Start(ref e)) if e.name().as_ref() == b"item" => {
+ inside_item = true;
+ }
+ Ok(Event::End(ref e)) if e.name().as_ref() == b"item" => {
+ inside_item = false;
+ }
+ Ok(Event::Text(e)) if inside_item && inside_link => {
+ let link = e.unescape()?;
+ if let Ok(link) = Url::parse(&link) {
+ links.push(link);
+ }
+ }
+ Err(e) => {
+ tracing::warn!("Error parsing feed: {}", e);
+ break;
+ }
+ Ok(Event::Eof) => break,
+ _ => {}
+ }
+ }
+
+ Ok(ParsedFeed { links })
+}
+
+fn parse_atom(feed: &str) -> Result<ParsedFeed> {
+ let mut reader = quick_xml::Reader::from_str(feed);
+
+ let mut buf = Vec::new();
+ let mut links = Vec::new();
+
+ let mut inside_entry = false;
+
+ loop {
+ match reader.read_event_into(&mut buf) {
+ Ok(Event::Start(ref e)) if e.name().as_ref() == b"entry" => {
+ inside_entry = true;
+ }
+ Ok(Event::End(ref e)) if e.name().as_ref() == b"entry" => {
+ inside_entry = false;
+ }
+ Ok(Event::Start(ref e)) | Ok(Event::Empty(ref e)) => {
+ if inside_entry && e.name().as_ref() == b"link" {
+ if let Some(Ok(href)) = e
+ .attributes()
+ .filter(std::result::Result::is_ok)
+ .find(|attr| attr.as_ref().unwrap().key.as_ref() == b"href")
+ {
+ dbg!(&href);
+ if let Ok(href) = href
+ .unescape_value()
+ .map_err(|e| anyhow!(e))
+ .and_then(|v| Url::parse(&v).map_err(|e| anyhow!(e)))
+ {
+ links.push(href);
+ }
+ }
+ }
+ }
+ Err(e) => {
+ tracing::warn!("Error parsing feed: {}", e);
+ break;
+ }
+ Ok(Event::Eof) => break,
+ _ => {}
+ }
+ }
+
+ Ok(ParsedFeed { links })
+}
+
+pub fn parse(feed: &str, kind: FeedKind) -> Result<ParsedFeed> {
+ // remember to only crawl urls that are not already in the index
+ // and on the same root_domain as the feed.
+ match kind {
+ FeedKind::Atom => parse_atom(feed),
+ FeedKind::Rss => parse_rss(feed),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_parse_rss() {
+ let feed = r#"
+ <?xml version="1.0" encoding="UTF-8"?>
+ <rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" version="2.0">
+ <channel>
+ <title>Test title</title>
+ <description>this is a description</description>
+ <link>https://example.com/</link>
+ <lastBuildDate>Mon, 30 Oct 2023 08:59:01 GMT</lastBuildDate>
+ <item>
+ <title>First title</title>
+ <description>First desc></description>
+ <link>https://example.com/a</link>
+ <pubDate>Mon, 30 Oct 2023 08:55:00 GMT</pubDate>
+ </item>
+ </channel>
+ </rss>
+ "#;
+
+ let parsed = parse_rss(feed).unwrap();
+
+ assert_eq!(
+ parsed.links,
+ vec![Url::parse("https://example.com/a").unwrap()]
+ );
+ }
+
+ #[test]
+ fn test_parse_atom() {
+ let feed = r#"
+ <?xml version="1.0" encoding="UTF-8"?>
+ <feed xmlns="http://www.w3.org/2005/Atom">
+ <title>Example Feed</title>
+ <entry>
+ <link href="https://example.com/a"/>
+ <updated>2003-12-13T18:30:02Z</updated>
+ </entry>
+ </feed>
+ "#;
+
+ let parsed = parse_atom(feed).unwrap();
+
+ assert_eq!(
+ parsed.links,
+ vec![Url::parse("https://example.com/a").unwrap()]
+ );
+ }
+}
diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rs
index d6fdf516..1d0795f3 100644
--- a/core/src/feed/scheduler.rs
+++ b/core/src/feed/scheduler.rs
@@ -26,7 +26,7 @@ use crate::{
webpage::url_ext::UrlExt,
};
-use super::{Feed, FeedIndex};
+use super::{index::FeedIndex, Feed};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Domain(String);
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 9c1450c6..f0e9aa33 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -202,7 +202,7 @@ impl InvertedIndex {
Self::open(path)
}
- pub fn insert(&mut self, webpage: Webpage) -> Result<()> {
+ pub fn insert(&self, webpage: Webpage) -> Result<()> {
self.writer
.add_document(webpage.into_tantivy(&self.schema)?)?;
Ok(())
@@ -1070,7 +1070,7 @@ mod tests {
#[test]
fn merge() {
- let mut index1 = InvertedIndex::temporary().expect("Unable to open index");
+ let index1 = InvertedIndex::temporary().expect("Unable to open index");
index1
.insert(
@@ -1094,7 +1094,7 @@ mod tests {
)
.expect("failed to insert webpage");
- let mut index2 = InvertedIndex::temporary().expect("Unable to open index");
+ let index2 = InvertedIndex::temporary().expect("Unable to open index");
index2
.insert(