web search done right
Support indieweb only optics. Still need to update the quickstart and blogroll optic to include the new match location.
Mikkel Denker7dbd14907581
diff --git a/core/src/enum_map.rs b/core/src/enum_map.rsindex 07d06c2d..e325152b 100644--- a/core/src/enum_map.rs+++ b/core/src/enum_map.rs@@ -86,6 +86,18 @@ where}}+impl<K, V> EnumMap<K, V>+where+ K: TryFrom<usize> + Into<usize>,+{+ pub fn keys(&self) -> impl Iterator<Item = K> + '_ {+ self.inner+ .iter()+ .enumerate()+ .filter_map(|(key, value)| value.as_ref().and_then(|_| K::try_from(key).ok()))+ }+}+impl<K, V> FromIterator<(K, V)> for EnumMap<K, V>whereK: Into<usize>,@@ -132,6 +144,15 @@ impl<K: Into<usize>> EnumSet<K> {}}+impl<K> EnumSet<K>+where+ K: TryFrom<usize> + Into<usize>,+{+ pub fn iter(&self) -> impl Iterator<Item = K> + '_ {+ self.map.keys()+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex fc68c310..98e1c78d 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -209,6 +209,15 @@ impl AsTantivyQuery for Matching {)),1.0,)),+ MatchLocation::MicroformatTag => Box::new(ConstQuery::new(+ Box::new(PatternQuery::new(+ self.pattern.clone(),+ TextField::MicroformatTags,+ schema,+ fastfield_reader.clone(),+ )),+ 1.0,+ )),MatchLocation::Schema => Box::new(ConstQuery::new(Box::new(PatternQuery::new(self.pattern.clone(),@@ -1646,4 +1655,97 @@ mod tests {.webpages;assert_eq!(res.len(), 1);}++ #[test]+ fn indieweb_search() {+ let mut index = Index::temporary().expect("Unable to open index");++ let mut page = Webpage {+ html: Html::parse(+ r#"+ <html>+ <head>+ <title>This is an example indie website</title>+ </head>+ <body>+ <article class="h-entry">+ <h1 class="p-name">Microformats are amazing</h1>+ <p class="e-content">This is the content of the article</p>+ <a class="u-url" href="https://example.com/microformats">Permalink</a>+ <a class="u-author" href="https://example.com">Author</a>+ <time class="dt-published" datetime="2021-01-01T00:00:00+00:00">2021-01-01</time>+ </article>+ </body>+ </html>+ "#,+ "https://example.com/",+ ).unwrap(),+ backlink_labels: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,++ node_id: None,+ dmoz_description: None,+ };++ page.html.set_clean_text("".to_string());++ index.insert(page).expect("failed to insert webpage");++ let mut page = Webpage {+ html: Html::parse(+ r#"+ <html>+ <head>+ <title>This is an example non-indie website</title>+ </head>+ <body>+ example example example+ </body>+ </html>+ "#,+ "https://non-indie-example.com/",+ ).unwrap(),+ backlink_labels: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,++ node_id: None,+ dmoz_description: None,+ };++ page.html.set_clean_text("".to_string());++ index.insert(page).expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let searcher = LocalSearcher::from(index);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 2);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ Optic::parse("DiscardNonMatching; Rule { Matches { MicroformatTag(\"|h-*\") } }")+ .unwrap(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);+ assert_eq!(res[0].domain, "example.com");+ }}diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex 95c50fe4..a11b6019 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -322,6 +322,7 @@ impl PatternWeight {Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),Field::Text(TextField::SiteWithout) => Ok(FastField::NumSiteTokens),Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),+ Field::Text(TextField::MicroformatTags) => Ok(FastField::NumMicroformatTagsTokens),Field::Text(TextField::FlattenedSchemaOrgJson) => {Ok(FastField::NumFlattenedSchemaTokens)}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 02a7e9f5..50eaf3d9 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -51,6 +51,7 @@ pub enum TextField {TitleBigrams,CleanBodyTrigrams,TitleTrigrams,+ MicroformatTags,}impl From<TextField> for usize {@@ -86,6 +87,7 @@ impl TextField {TextField::TitleBigrams => Tokenizer::Bigram(BigramTokenizer::default()),TextField::CleanBodyTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),TextField::TitleTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),+ TextField::MicroformatTags => Tokenizer::default(),}}@@ -123,6 +125,7 @@ impl TextField {TextField::TitleBigrams => false,TextField::CleanBodyTrigrams => false,TextField::TitleTrigrams => false,+ TextField::MicroformatTags => true,}}@@ -152,6 +155,7 @@ impl TextField {TextField::TitleBigrams => "title_bigrams",TextField::CleanBodyTrigrams => "clean_body_trigrams",TextField::TitleTrigrams => "title_trigrams",+ TextField::MicroformatTags => "microformat_tags",}}}@@ -171,6 +175,7 @@ pub enum FastField {NumDescriptionTokens,NumSiteTokens,NumDomainTokens,+ NumMicroformatTagsTokens,SiteHash1,SiteHash2,UrlWithoutQueryHash1,@@ -207,6 +212,7 @@ impl FastField {FastField::NumDomainTokens => "num_domain_tokens",FastField::NumSiteTokens => "num_site_tokens",FastField::NumFlattenedSchemaTokens => "num_flattened_schema_tokens",+ FastField::NumMicroformatTagsTokens => "num_microformat_tags_tokens",FastField::SiteHash1 => "site_hash1",FastField::SiteHash2 => "site_hash2",FastField::UrlWithoutQueryHash1 => "url_without_query_hash1",@@ -239,7 +245,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 54] = [+pub static ALL_FIELDS: [Field; 56] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -264,6 +270,7 @@ pub static ALL_FIELDS: [Field; 54] = [Field::Text(TextField::TitleBigrams),Field::Text(TextField::CleanBodyTrigrams),Field::Text(TextField::TitleTrigrams),+ Field::Text(TextField::MicroformatTags),// FAST FIELDSField::Fast(FastField::IsHomepage),Field::Fast(FastField::HostCentrality),@@ -279,6 +286,7 @@ pub static ALL_FIELDS: [Field; 54] = [Field::Fast(FastField::NumDomainTokens),Field::Fast(FastField::NumSiteTokens),Field::Fast(FastField::NumFlattenedSchemaTokens),+ Field::Fast(FastField::NumMicroformatTagsTokens),Field::Fast(FastField::SiteHash1),Field::Fast(FastField::SiteHash2),Field::Fast(FastField::UrlWithoutQueryHash1),@@ -384,6 +392,9 @@ impl Field {Field::Text(TextField::TitleTrigrams) => {IndexingOption::Text(self.default_text_options())}+ Field::Text(TextField::MicroformatTags) => {+ IndexingOption::Text(self.default_text_options())+ }Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue)@@ -436,6 +447,11 @@ impl Field {.set_fast(Cardinality::SingleValue).set_indexed(),),+ Field::Fast(FastField::NumMicroformatTagsTokens) => IndexingOption::Integer(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed(),+ ),Field::Fast(FastField::NumUrlTokens) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue)@@ -532,6 +548,11 @@ impl Field {}}+ /// Whether or not the field should be included+ /// in the fields that the `Query` searches.+ ///+ /// The fields can still be searched by manually+ /// constructing a tantivy query.pub fn is_searchable(&self) -> bool {!matches!(self,@@ -541,6 +562,7 @@ impl Field {| Field::Text(TextField::TitleBigrams)| Field::Text(TextField::CleanBodyTrigrams)| Field::Text(TextField::TitleTrigrams)+ | Field::Text(TextField::MicroformatTags)) && !self.is_fast()}@@ -600,6 +622,7 @@ impl FastField {FastField::Region => DataType::U64,FastField::NumUrlTokens => DataType::U64,FastField::NumTitleTokens => DataType::U64,+ FastField::NumMicroformatTagsTokens => DataType::U64,FastField::NumCleanBodyTokens => DataType::U64,FastField::NumDescriptionTokens => DataType::U64,FastField::NumDomainTokens => DataType::U64,diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex c3c0642f..b50d8760 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -293,7 +293,7 @@ impl From<RobotsMeta> for usize {}#[allow(clippy::enum_variant_names)]-enum MicroFormat {+enum Microformat {HCard,HEvent,HEntry,@@ -302,39 +302,55 @@ enum MicroFormat {HProduct,}-impl MicroFormat {+impl Microformat {fn as_str(&self) -> &str {match self {- MicroFormat::HCard => "h-card",- MicroFormat::HEvent => "h-event",- MicroFormat::HEntry => "h-entry",- MicroFormat::HRecipe => "h-recipe",- MicroFormat::HReview => "h-review",- MicroFormat::HProduct => "h-product",+ Microformat::HCard => "h-card",+ Microformat::HEvent => "h-event",+ Microformat::HEntry => "h-entry",+ Microformat::HRecipe => "h-recipe",+ Microformat::HReview => "h-review",+ Microformat::HProduct => "h-product",}}}-impl From<MicroFormat> for usize {- fn from(value: MicroFormat) -> Self {+impl From<Microformat> for usize {+ fn from(value: Microformat) -> Self {match value {- MicroFormat::HCard => 0,- MicroFormat::HEvent => 1,- MicroFormat::HEntry => 2,- MicroFormat::HRecipe => 3,- MicroFormat::HReview => 4,- MicroFormat::HProduct => 5,+ Microformat::HCard => 0,+ Microformat::HEvent => 1,+ Microformat::HEntry => 2,+ Microformat::HRecipe => 3,+ Microformat::HReview => 4,+ Microformat::HProduct => 5,}}}-const ALL_MICROFORMATS: [MicroFormat; 6] = [- MicroFormat::HCard,- MicroFormat::HEvent,- MicroFormat::HEntry,- MicroFormat::HRecipe,- MicroFormat::HReview,- MicroFormat::HProduct,+impl TryFrom<usize> for Microformat {+ type Error = anyhow::Error;++ fn try_from(value: usize) -> Result<Self> {+ match value {+ 0 => Ok(Microformat::HCard),+ 1 => Ok(Microformat::HEvent),+ 2 => Ok(Microformat::HEntry),+ 3 => Ok(Microformat::HRecipe),+ 4 => Ok(Microformat::HReview),+ 5 => Ok(Microformat::HProduct),+ _ => Err(anyhow::anyhow!("Unknown microformat")),+ }+ }+}++const ALL_MICROFORMATS: [Microformat; 6] = [+ Microformat::HCard,+ Microformat::HEvent,+ Microformat::HEntry,+ Microformat::HRecipe,+ Microformat::HReview,+ Microformat::HProduct,];#[derive(Debug)]@@ -765,7 +781,7 @@ impl Html {}}- fn microformats(&self) -> EnumSet<MicroFormat> {+ fn microformats(&self) -> EnumSet<Microformat> {let mut microformats = EnumSet::new();for node in self.root.inclusive_descendants() {@@ -861,6 +877,17 @@ impl Html {self.pretokenize_string(text)}+ fn pretokenize_microformats(&self) -> PreTokenizedString {+ let mut text = String::new();++ for microformat in self.microformats().iter() {+ text.push_str(microformat.as_str());+ text.push(' ');+ }++ self.pretokenize_string(text)+ }+fn pretokenize_string(&self, text: String) -> PreTokenizedString {let mut tokens = Vec::new();@@ -884,6 +911,7 @@ impl Html {let domain = self.pretokenize_domain();let site = self.pretokenize_site();let description = self.pretokenize_description();+ let microformats = self.pretokenize_microformats();let schemas: Vec<_> = self.schema_org();@@ -1102,6 +1130,9 @@ impl Html {Field::Text(TextField::FlattenedSchemaOrgJson) => {doc.add_pre_tokenized_text(tantivy_field, pretokenized_schema_json.clone());}+ Field::Text(TextField::MicroformatTags) => {+ doc.add_pre_tokenized_text(tantivy_field, microformats.clone());+ }Field::Fast(FastField::IsHomepage) => {doc.add_u64(tantivy_field, (self.is_homepage()).into());}@@ -1116,6 +1147,9 @@ impl Html {Field::Fast(FastField::NumUrlTokens) => {doc.add_u64(tantivy_field, url.tokens.len() as u64)}+ Field::Fast(FastField::NumMicroformatTagsTokens) => {+ doc.add_u64(tantivy_field, microformats.tokens.len() as u64)+ }Field::Fast(FastField::NumTitleTokens) => {doc.add_u64(tantivy_field, title.tokens.len() as u64)}@@ -2330,7 +2364,7 @@ mod tests {<time class="dt-published" datetime="2021-01-01T00:00:00+00:00">2021-01-01</time></article>- <div class="h-RECIPE>+ <div class="h-RECIPE">For some reason this site also has a recipe</div></body>@@ -2341,8 +2375,8 @@ mod tests {let microformats = html.microformats();- assert!(microformats.contains(MicroFormat::HEntry));- assert!(microformats.contains(MicroFormat::HRecipe));- assert!(!microformats.contains(MicroFormat::HCard));+ assert!(microformats.contains(Microformat::HEntry));+ assert!(microformats.contains(Microformat::HRecipe));+ assert!(!microformats.contains(Microformat::HCard));}}diff --git a/optics/src/ast.rs b/optics/src/ast.rsindex c9c6ab95..ce7d84bb 100644--- a/optics/src/ast.rs+++ b/optics/src/ast.rs@@ -122,6 +122,7 @@ pub enum RawMatchPart {Title(String),Description(String),Content(String),+ MicroformatTag(String),Schema(String),}diff --git a/optics/src/lexer.rs b/optics/src/lexer.rsindex ab02a610..2cbfe227 100644--- a/optics/src/lexer.rs+++ b/optics/src/lexer.rs@@ -41,6 +41,7 @@ pub enum Token<'a> {Title,Description,Content,+ MicroformatTag,Schema,Action,Boost,@@ -76,6 +77,7 @@ impl<'a> ToString for Token<'a> {Token::Title => "Title".to_string(),Token::Description => "Description".to_string(),Token::Content => "Content".to_string(),+ Token::MicroformatTag => "MicroformatTag".to_string(),Token::Schema => "Schema".to_string(),Token::Action => "Action".to_string(),Token::Boost => "Boost".to_string(),@@ -143,6 +145,8 @@ enum Outer<'a> {Description,#[token("Content")]Content,+ #[token("MicroformatTag")]+ MicroformatTag,#[token("Schema")]Schema,#[token("Action")]@@ -314,6 +318,7 @@ impl<'source> Iterator for LexerBridge<'source> {Outer::Title => Some(Ok((s.start, Token::Title, s.end))),Outer::Description => Some(Ok((s.start, Token::Description, s.end))),Outer::Content => Some(Ok((s.start, Token::Content, s.end))),+ Outer::MicroformatTag => Some(Ok((s.start, Token::MicroformatTag, s.end))),Outer::Schema => Some(Ok((s.start, Token::Schema, s.end))),Outer::Action => Some(Ok((s.start, Token::Action, s.end))),Outer::Boost => Some(Ok((s.start, Token::Boost, s.end))),diff --git a/optics/src/lib.rs b/optics/src/lib.rsindex b9853f34..5b07d225 100644--- a/optics/src/lib.rs+++ b/optics/src/lib.rs@@ -147,6 +147,7 @@ impl ToString for Matching {MatchLocation::Title => s.push_str("Title"),MatchLocation::Description => s.push_str("Description"),MatchLocation::Content => s.push_str("Content"),+ MatchLocation::MicroformatTag => s.push_str("MicroformatTag"),MatchLocation::Schema => s.push_str("Schema"),}s.push('(');@@ -174,6 +175,7 @@ impl TryFrom<RawMatchPart> for Matching {RawMatchPart::Title(s) => (s, MatchLocation::Title),RawMatchPart::Description(s) => (s, MatchLocation::Description),RawMatchPart::Content(s) => (s, MatchLocation::Content),+ RawMatchPart::MicroformatTag(s) => (s, MatchLocation::MicroformatTag),RawMatchPart::Schema(s) => (s, MatchLocation::Schema),};@@ -282,6 +284,7 @@ pub enum MatchLocation {Title,Description,Content,+ MicroformatTag,Schema,}diff --git a/optics/src/parser.lalrpop b/optics/src/parser.lalrpopindex a9ac04f2..f0d8a34e 100644--- a/optics/src/parser.lalrpop+++ b/optics/src/parser.lalrpop@@ -41,6 +41,7 @@ RawMatchPart: RawMatchPart = {"Title" "(" <StringLiteral> ")" => RawMatchPart::Title(<>.to_string()),"Description" "(" <StringLiteral> ")" => RawMatchPart::Description(<>.to_string()),"Content" "(" <StringLiteral> ")" => RawMatchPart::Content(<>.to_string()),+ "MicroformatTag" "(" <StringLiteral> ")" => RawMatchPart::MicroformatTag(<>.to_string()),"Schema" "(" <StringLiteral> ")" => RawMatchPart::Schema(<>.to_string()),}@@ -116,6 +117,7 @@ extern {"Title" => Token::Title,"Description" => Token::Description,"Content" => Token::Content,+ "MicroformatTag" => Token::MicroformatTag,"Schema" => Token::Schema,"Action" => Token::Action,"Boost" => Token::Boost,diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_modelindex bb584ead..6855a1c2 100755--- a/scripts/export_abstractive_summary_model+++ b/scripts/export_abstractive_summary_model@@ -8,7 +8,6 @@ import sysif os.path.exists("data/summarizer/abstractive"):print("data/summarizer/abstractive already exists. Exiting...")sys.exit()-os.makedirs("data/summarizer/abstractive")MODEL = "lidiya/bart-large-xsum-samsum"