web search done right
parse rss and atom feeds
Mikkel Denker0cfd7701fa37
diff --git a/core/src/entrypoint/api.rs b/core/src/entrypoint/api.rsindex 9a6231fb..56b88f19 100644--- a/core/src/entrypoint/api.rs+++ b/core/src/entrypoint/api.rs@@ -34,7 +34,7 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {let group = registry.new_group(- "search_requests".to_string(),+ "stract_search_requests".to_string(),Some("Total number of incoming search requests.".to_string()),).unwrap();@@ -56,7 +56,7 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {let group = registry.new_group(- "explore_requests".to_string(),+ "stract_explore_requests".to_string(),Some("Total number of incoming requests to explore api.".to_string()),).unwrap();@@ -64,7 +64,7 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {let group = registry.new_group(- "daily_active_users".to_string(),+ "stract_daily_active_users".to_string(),Some("Approximate number of unique daily active users.".to_string()),).unwrap();diff --git a/core/src/feed/index.rs b/core/src/feed/index.rsnew file mode 100644index 00000000..4547f493--- /dev/null+++ b/core/src/feed/index.rs@@ -0,0 +1,203 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::path::Path;++use crate::tokenizer::{SiteOperatorUrlTokenizer, Tokenizer};+use anyhow::Result;+use tantivy::{+ query::{PhraseQuery, TermQuery},+ schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},+ tokenizer::Tokenizer as TantivyTokenizer,+};+use url::Url;++use super::{Feed, FeedKind};++pub struct FeedIndex {+ writer: tantivy::IndexWriter,+ reader: tantivy::IndexReader,+ schema: tantivy::schema::Schema,+}++impl FeedIndex {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(path.as_ref())?;+ }++ let url_tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);+ let kind_tokenizer = Tokenizer::default();++ let mut builder = tantivy::schema::Schema::builder();++ builder.add_text_field(+ "url",+ TextOptions::default()+ .set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer(url_tokenizer.as_str())+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ )+ .set_stored(),+ );+ builder.add_text_field(+ "kind",+ TextOptions::default()+ .set_indexing_options(+ TextFieldIndexing::default()+ .set_tokenizer(kind_tokenizer.as_str())+ .set_index_option(IndexRecordOption::Basic),+ )+ .set_stored(),+ );++ let schema = builder.build();++ let tv_index = tantivy::Index::create_in_dir(path, schema.clone())?;+ tv_index+ .tokenizers()+ .register(url_tokenizer.as_str(), url_tokenizer);++ tv_index+ .tokenizers()+ .register(kind_tokenizer.as_str(), kind_tokenizer);++ let writer = tv_index.writer(50_000_000)?;++ let reader = tv_index.reader()?;++ Ok(Self {+ writer,+ reader,+ schema,+ })+ }++ pub fn insert(&mut self, feed: &Feed) -> Result<()> {+ let mut doc = tantivy::TantivyDocument::default();++ doc.add_text(self.schema.get_field("url")?, feed.url.as_str());+ doc.add_text(+ self.schema.get_field("kind")?,+ &serde_json::to_string(&feed.kind)?,+ );++ self.writer.add_document(doc)?;++ Ok(())+ }++ pub fn commit(&mut self) -> Result<()> {+ self.writer.commit()?;+ self.reader.reload()?;++ Ok(())+ }++ fn terms(&self, query: &str) -> Vec<tantivy::Term> {+ let mut tokenizer = SiteOperatorUrlTokenizer;+ let mut res = Vec::new();+ let tv_field = self.schema.get_field("url").unwrap();++ let mut stream = tokenizer.token_stream(query);++ while let Some(token) = stream.next() {+ res.push(tantivy::Term::from_field_text(tv_field, &token.text));+ }++ res+ }++ pub fn search(&self, query: &str) -> Result<Vec<Feed>> {+ let searcher = self.reader.searcher();++ let terms = self.terms(query);+ if terms.is_empty() {+ return Ok(Vec::new());+ }++ let query = if terms.len() == 1 {+ let term = terms.first().unwrap().clone();+ Box::new(TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ )) as Box<dyn tantivy::query::Query>+ } else {+ Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>+ };++ let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;++ let mut res = Vec::new();++ for address in docs {+ let doc: tantivy::TantivyDocument = searcher.doc(address)?;++ let url = doc+ .get_first(self.schema.get_field("url")?)+ .unwrap()+ .as_ref()+ .as_str()+ .unwrap();++ let url = Url::parse(url)?;++ let kind = doc+ .get_first(self.schema.get_field("kind")?)+ .unwrap()+ .as_ref()+ .as_str()+ .unwrap();++ let kind: FeedKind = serde_json::from_str(kind)?;++ res.push(Feed { url, kind });+ }++ Ok(res)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn feed_index() {+ let mut index = FeedIndex::open(crate::gen_temp_path()).unwrap();++ let a = Feed {+ url: Url::parse("https://a.com/feed.xml").unwrap(),+ kind: FeedKind::Atom,+ };+ index.insert(&a).unwrap();++ let b = Feed {+ url: Url::parse("https://b.com/another/feed").unwrap(),+ kind: FeedKind::Rss,+ };+ index.insert(&b).unwrap();++ index.commit().unwrap();++ let res = index.search("a.com").unwrap();+ assert_eq!(res, vec![a]);++ let res = index.search("b.com").unwrap();+ assert_eq!(res, vec![b]);+ }+}diff --git a/core/src/feed/mod.rs b/core/src/feed/mod.rsindex e7e69c28..284c2451 100644--- a/core/src/feed/mod.rs+++ b/core/src/feed/mod.rs@@ -14,20 +14,15 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::Path;--use crate::tokenizer::{SiteOperatorUrlTokenizer, Tokenizer};-use anyhow::Result;-use tantivy::{- query::{PhraseQuery, TermQuery},- schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},- tokenizer::Tokenizer as TantivyTokenizer,-};use url::Url;+pub mod index;+mod parser;pub mod scheduler;-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]+pub use parser::parse;++#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub enum FeedKind {Atom,Rss,@@ -39,177 +34,6 @@ pub struct Feed {kind: FeedKind,}-pub struct FeedIndex {- writer: tantivy::IndexWriter,- reader: tantivy::IndexReader,- schema: tantivy::schema::Schema,-}--impl FeedIndex {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- if !path.as_ref().exists() {- std::fs::create_dir_all(path.as_ref())?;- }-- let url_tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);- let kind_tokenizer = Tokenizer::default();-- let mut builder = tantivy::schema::Schema::builder();-- builder.add_text_field(- "url",- TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default()- .set_tokenizer(url_tokenizer.as_str())- .set_index_option(IndexRecordOption::WithFreqsAndPositions),- )- .set_stored(),- );- builder.add_text_field(- "kind",- TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default()- .set_tokenizer(kind_tokenizer.as_str())- .set_index_option(IndexRecordOption::Basic),- )- .set_stored(),- );-- let schema = builder.build();-- let tv_index = tantivy::Index::create_in_dir(path, schema.clone())?;- tv_index- .tokenizers()- .register(url_tokenizer.as_str(), url_tokenizer);-- tv_index- .tokenizers()- .register(kind_tokenizer.as_str(), kind_tokenizer);-- let writer = tv_index.writer(50_000_000)?;-- let reader = tv_index.reader()?;-- Ok(Self {- writer,- reader,- schema,- })- }-- pub fn insert(&mut self, feed: &Feed) -> Result<()> {- let mut doc = tantivy::TantivyDocument::default();-- doc.add_text(self.schema.get_field("url")?, feed.url.as_str());- doc.add_text(- self.schema.get_field("kind")?,- &serde_json::to_string(&feed.kind)?,- );-- self.writer.add_document(doc)?;-- Ok(())- }-- pub fn commit(&mut self) -> Result<()> {- self.writer.commit()?;- self.reader.reload()?;-- Ok(())- }-- fn terms(&self, query: &str) -> Vec<tantivy::Term> {- let mut tokenizer = SiteOperatorUrlTokenizer;- let mut res = Vec::new();- let tv_field = self.schema.get_field("url").unwrap();-- let mut stream = tokenizer.token_stream(query);-- while let Some(token) = stream.next() {- res.push(tantivy::Term::from_field_text(tv_field, &token.text));- }-- res- }-- pub fn search(&self, query: &str) -> Result<Vec<Feed>> {- let searcher = self.reader.searcher();-- let terms = self.terms(query);- if terms.is_empty() {- return Ok(Vec::new());- }-- let query = if terms.len() == 1 {- let term = terms.get(0).unwrap().clone();- Box::new(TermQuery::new(- term,- tantivy::schema::IndexRecordOption::Basic,- )) as Box<dyn tantivy::query::Query>- } else {- Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>- };-- let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;-- let mut res = Vec::new();-- for address in docs {- let doc: tantivy::TantivyDocument = searcher.doc(address)?;-- let url = doc- .get_first(self.schema.get_field("url")?)- .unwrap()- .as_ref()- .as_str()- .unwrap();-- let url = Url::parse(url)?;-- let kind = doc- .get_first(self.schema.get_field("kind")?)- .unwrap()- .as_ref()- .as_str()- .unwrap();-- let kind: FeedKind = serde_json::from_str(kind)?;-- res.push(Feed { url, kind });- }-- Ok(res)- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn feed_index() {- let mut index = FeedIndex::open(crate::gen_temp_path()).unwrap();-- let a = Feed {- url: Url::parse("https://a.com/feed.xml").unwrap(),- kind: FeedKind::Atom,- };- index.insert(&a).unwrap();-- let b = Feed {- url: Url::parse("https://b.com/another/feed").unwrap(),- kind: FeedKind::Rss,- };- index.insert(&b).unwrap();-- index.commit().unwrap();-- let res = index.search("a.com").unwrap();- assert_eq!(res, vec![a]);-- let res = index.search("b.com").unwrap();- assert_eq!(res, vec![b]);- }+pub struct ParsedFeed {+ pub links: Vec<Url>,}diff --git a/core/src/feed/parser.rs b/core/src/feed/parser.rsnew file mode 100644index 00000000..de3734fa--- /dev/null+++ b/core/src/feed/parser.rs@@ -0,0 +1,170 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use anyhow::{anyhow, Result};+use quick_xml::events::Event;+use url::Url;++use super::{FeedKind, ParsedFeed};++fn parse_rss(feed: &str) -> Result<ParsedFeed> {+ let mut reader = quick_xml::Reader::from_str(feed);++ let mut buf = Vec::new();+ let mut links = Vec::new();+ let mut inside_link = false;+ let mut inside_item = false;++ loop {+ match reader.read_event_into(&mut buf) {+ Ok(Event::Start(ref e)) if e.name().as_ref() == b"link" => {+ inside_link = true;+ }+ Ok(Event::End(ref e)) if e.name().as_ref() == b"link" => {+ inside_link = false;+ }+ Ok(Event::Start(ref e)) if e.name().as_ref() == b"item" => {+ inside_item = true;+ }+ Ok(Event::End(ref e)) if e.name().as_ref() == b"item" => {+ inside_item = false;+ }+ Ok(Event::Text(e)) if inside_item && inside_link => {+ let link = e.unescape()?;+ if let Ok(link) = Url::parse(&link) {+ links.push(link);+ }+ }+ Err(e) => {+ tracing::warn!("Error parsing feed: {}", e);+ break;+ }+ Ok(Event::Eof) => break,+ _ => {}+ }+ }++ Ok(ParsedFeed { links })+}++fn parse_atom(feed: &str) -> Result<ParsedFeed> {+ let mut reader = quick_xml::Reader::from_str(feed);++ let mut buf = Vec::new();+ let mut links = Vec::new();++ let mut inside_entry = false;++ loop {+ match reader.read_event_into(&mut buf) {+ Ok(Event::Start(ref e)) if e.name().as_ref() == b"entry" => {+ inside_entry = true;+ }+ Ok(Event::End(ref e)) if e.name().as_ref() == b"entry" => {+ inside_entry = false;+ }+ Ok(Event::Start(ref e)) | Ok(Event::Empty(ref e)) => {+ if inside_entry && e.name().as_ref() == b"link" {+ if let Some(Ok(href)) = e+ .attributes()+ .filter(std::result::Result::is_ok)+ .find(|attr| attr.as_ref().unwrap().key.as_ref() == b"href")+ {+ dbg!(&href);+ if let Ok(href) = href+ .unescape_value()+ .map_err(|e| anyhow!(e))+ .and_then(|v| Url::parse(&v).map_err(|e| anyhow!(e)))+ {+ links.push(href);+ }+ }+ }+ }+ Err(e) => {+ tracing::warn!("Error parsing feed: {}", e);+ break;+ }+ Ok(Event::Eof) => break,+ _ => {}+ }+ }++ Ok(ParsedFeed { links })+}++pub fn parse(feed: &str, kind: FeedKind) -> Result<ParsedFeed> {+ // remember to only crawl urls that are not already in the index+ // and on the same root_domain as the feed.+ match kind {+ FeedKind::Atom => parse_atom(feed),+ FeedKind::Rss => parse_rss(feed),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_parse_rss() {+ let feed = r#"+ <?xml version="1.0" encoding="UTF-8"?>+ <rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" version="2.0">+ <channel>+ <title>Test title</title>+ <description>this is a description</description>+ <link>https://example.com/</link>+ <lastBuildDate>Mon, 30 Oct 2023 08:59:01 GMT</lastBuildDate>+ <item>+ <title>First title</title>+ <description>First desc></description>+ <link>https://example.com/a</link>+ <pubDate>Mon, 30 Oct 2023 08:55:00 GMT</pubDate>+ </item>+ </channel>+ </rss>+ "#;++ let parsed = parse_rss(feed).unwrap();++ assert_eq!(+ parsed.links,+ vec![Url::parse("https://example.com/a").unwrap()]+ );+ }++ #[test]+ fn test_parse_atom() {+ let feed = r#"+ <?xml version="1.0" encoding="UTF-8"?>+ <feed xmlns="http://www.w3.org/2005/Atom">+ <title>Example Feed</title>+ <entry>+ <link href="https://example.com/a"/>+ <updated>2003-12-13T18:30:02Z</updated>+ </entry>+ </feed>+ "#;++ let parsed = parse_atom(feed).unwrap();++ assert_eq!(+ parsed.links,+ vec![Url::parse("https://example.com/a").unwrap()]+ );+ }+}diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rsindex d6fdf516..1d0795f3 100644--- a/core/src/feed/scheduler.rs+++ b/core/src/feed/scheduler.rs@@ -26,7 +26,7 @@ use crate::{webpage::url_ext::UrlExt,};-use super::{Feed, FeedIndex};+use super::{index::FeedIndex, Feed};#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct Domain(String);diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 9c1450c6..f0e9aa33 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -202,7 +202,7 @@ impl InvertedIndex {Self::open(path)}- pub fn insert(&mut self, webpage: Webpage) -> Result<()> {+ pub fn insert(&self, webpage: Webpage) -> Result<()> {self.writer.add_document(webpage.into_tantivy(&self.schema)?)?;Ok(())@@ -1070,7 +1070,7 @@ mod tests {#[test]fn merge() {- let mut index1 = InvertedIndex::temporary().expect("Unable to open index");+ let index1 = InvertedIndex::temporary().expect("Unable to open index");index1.insert(@@ -1094,7 +1094,7 @@ mod tests {).expect("failed to insert webpage");- let mut index2 = InvertedIndex::temporary().expect("Unable to open index");+ let index2 = InvertedIndex::temporary().expect("Unable to open index");index2.insert(