Skip to content

web search done right

Support indieweb only optics. Still need to update the quickstart and blogroll optic to include the new match location.

Mikkel Denker7dbd14907581
diff --git a/core/src/enum_map.rs b/core/src/enum_map.rs
index 07d06c2d..e325152b 100644
--- a/core/src/enum_map.rs
+++ b/core/src/enum_map.rs
@@ -86,6 +86,18 @@ where
}
}
+impl<K, V> EnumMap<K, V>
+where
+ K: TryFrom<usize> + Into<usize>,
+{
+ pub fn keys(&self) -> impl Iterator<Item = K> + '_ {
+ self.inner
+ .iter()
+ .enumerate()
+ .filter_map(|(key, value)| value.as_ref().and_then(|_| K::try_from(key).ok()))
+ }
+}
+
impl<K, V> FromIterator<(K, V)> for EnumMap<K, V>
where
K: Into<usize>,
@@ -132,6 +144,15 @@ impl<K: Into<usize>> EnumSet<K> {
}
}
+impl<K> EnumSet<K>
+where
+ K: TryFrom<usize> + Into<usize>,
+{
+ pub fn iter(&self) -> impl Iterator<Item = K> + '_ {
+ self.map.keys()
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index fc68c310..98e1c78d 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -209,6 +209,15 @@ impl AsTantivyQuery for Matching {
)),
1.0,
)),
+ MatchLocation::MicroformatTag => Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ self.pattern.clone(),
+ TextField::MicroformatTags,
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ )),
MatchLocation::Schema => Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
@@ -1646,4 +1655,97 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
}
+
+ #[test]
+ fn indieweb_search() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ let mut page = Webpage {
+ html: Html::parse(
+ r#"
+ <html>
+ <head>
+ <title>This is an example indie website</title>
+ </head>
+ <body>
+ <article class="h-entry">
+ <h1 class="p-name">Microformats are amazing</h1>
+ <p class="e-content">This is the content of the article</p>
+ <a class="u-url" href="https://example.com/microformats">Permalink</a>
+ <a class="u-author" href="https://example.com">Author</a>
+ <time class="dt-published" datetime="2021-01-01T00:00:00+00:00">2021-01-01</time>
+ </article>
+ </body>
+ </html>
+ "#,
+ "https://example.com/",
+ ).unwrap(),
+ backlink_labels: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ };
+
+ page.html.set_clean_text("".to_string());
+
+ index.insert(page).expect("failed to insert webpage");
+
+ let mut page = Webpage {
+ html: Html::parse(
+ r#"
+ <html>
+ <head>
+ <title>This is an example non-indie website</title>
+ </head>
+ <body>
+ example example example
+ </body>
+ </html>
+ "#,
+ "https://non-indie-example.com/",
+ ).unwrap(),
+ backlink_labels: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ };
+
+ page.html.set_clean_text("".to_string());
+
+ index.insert(page).expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let searcher = LocalSearcher::from(index);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 2);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ Optic::parse("DiscardNonMatching; Rule { Matches { MicroformatTag(\"|h-*\") } }")
+ .unwrap(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].domain, "example.com");
+ }
}
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 95c50fe4..a11b6019 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -322,6 +322,7 @@ impl PatternWeight {
Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
Field::Text(TextField::SiteWithout) => Ok(FastField::NumSiteTokens),
Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
+ Field::Text(TextField::MicroformatTags) => Ok(FastField::NumMicroformatTagsTokens),
Field::Text(TextField::FlattenedSchemaOrgJson) => {
Ok(FastField::NumFlattenedSchemaTokens)
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 02a7e9f5..50eaf3d9 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -51,6 +51,7 @@ pub enum TextField {
TitleBigrams,
CleanBodyTrigrams,
TitleTrigrams,
+ MicroformatTags,
}
impl From<TextField> for usize {
@@ -86,6 +87,7 @@ impl TextField {
TextField::TitleBigrams => Tokenizer::Bigram(BigramTokenizer::default()),
TextField::CleanBodyTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),
TextField::TitleTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),
+ TextField::MicroformatTags => Tokenizer::default(),
}
}
@@ -123,6 +125,7 @@ impl TextField {
TextField::TitleBigrams => false,
TextField::CleanBodyTrigrams => false,
TextField::TitleTrigrams => false,
+ TextField::MicroformatTags => true,
}
}
@@ -152,6 +155,7 @@ impl TextField {
TextField::TitleBigrams => "title_bigrams",
TextField::CleanBodyTrigrams => "clean_body_trigrams",
TextField::TitleTrigrams => "title_trigrams",
+ TextField::MicroformatTags => "microformat_tags",
}
}
}
@@ -171,6 +175,7 @@ pub enum FastField {
NumDescriptionTokens,
NumSiteTokens,
NumDomainTokens,
+ NumMicroformatTagsTokens,
SiteHash1,
SiteHash2,
UrlWithoutQueryHash1,
@@ -207,6 +212,7 @@ impl FastField {
FastField::NumDomainTokens => "num_domain_tokens",
FastField::NumSiteTokens => "num_site_tokens",
FastField::NumFlattenedSchemaTokens => "num_flattened_schema_tokens",
+ FastField::NumMicroformatTagsTokens => "num_microformat_tags_tokens",
FastField::SiteHash1 => "site_hash1",
FastField::SiteHash2 => "site_hash2",
FastField::UrlWithoutQueryHash1 => "url_without_query_hash1",
@@ -239,7 +245,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 54] = [
+pub static ALL_FIELDS: [Field; 56] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -264,6 +270,7 @@ pub static ALL_FIELDS: [Field; 54] = [
Field::Text(TextField::TitleBigrams),
Field::Text(TextField::CleanBodyTrigrams),
Field::Text(TextField::TitleTrigrams),
+ Field::Text(TextField::MicroformatTags),
// FAST FIELDS
Field::Fast(FastField::IsHomepage),
Field::Fast(FastField::HostCentrality),
@@ -279,6 +286,7 @@ pub static ALL_FIELDS: [Field; 54] = [
Field::Fast(FastField::NumDomainTokens),
Field::Fast(FastField::NumSiteTokens),
Field::Fast(FastField::NumFlattenedSchemaTokens),
+ Field::Fast(FastField::NumMicroformatTagsTokens),
Field::Fast(FastField::SiteHash1),
Field::Fast(FastField::SiteHash2),
Field::Fast(FastField::UrlWithoutQueryHash1),
@@ -384,6 +392,9 @@ impl Field {
Field::Text(TextField::TitleTrigrams) => {
IndexingOption::Text(self.default_text_options())
}
+ Field::Text(TextField::MicroformatTags) => {
+ IndexingOption::Text(self.default_text_options())
+ }
Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
@@ -436,6 +447,11 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
+ Field::Fast(FastField::NumMicroformatTagsTokens) => IndexingOption::Integer(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed(),
+ ),
Field::Fast(FastField::NumUrlTokens) => IndexingOption::Integer(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
@@ -532,6 +548,11 @@ impl Field {
}
}
+ /// Whether or not the field should be included
+ /// in the fields that the `Query` searches.
+ ///
+ /// The fields can still be searched by manually
+ /// constructing a tantivy query.
pub fn is_searchable(&self) -> bool {
!matches!(
self,
@@ -541,6 +562,7 @@ impl Field {
| Field::Text(TextField::TitleBigrams)
| Field::Text(TextField::CleanBodyTrigrams)
| Field::Text(TextField::TitleTrigrams)
+ | Field::Text(TextField::MicroformatTags)
) && !self.is_fast()
}
@@ -600,6 +622,7 @@ impl FastField {
FastField::Region => DataType::U64,
FastField::NumUrlTokens => DataType::U64,
FastField::NumTitleTokens => DataType::U64,
+ FastField::NumMicroformatTagsTokens => DataType::U64,
FastField::NumCleanBodyTokens => DataType::U64,
FastField::NumDescriptionTokens => DataType::U64,
FastField::NumDomainTokens => DataType::U64,
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index c3c0642f..b50d8760 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -293,7 +293,7 @@ impl From<RobotsMeta> for usize {
}
#[allow(clippy::enum_variant_names)]
-enum MicroFormat {
+enum Microformat {
HCard,
HEvent,
HEntry,
@@ -302,39 +302,55 @@ enum MicroFormat {
HProduct,
}
-impl MicroFormat {
+impl Microformat {
fn as_str(&self) -> &str {
match self {
- MicroFormat::HCard => "h-card",
- MicroFormat::HEvent => "h-event",
- MicroFormat::HEntry => "h-entry",
- MicroFormat::HRecipe => "h-recipe",
- MicroFormat::HReview => "h-review",
- MicroFormat::HProduct => "h-product",
+ Microformat::HCard => "h-card",
+ Microformat::HEvent => "h-event",
+ Microformat::HEntry => "h-entry",
+ Microformat::HRecipe => "h-recipe",
+ Microformat::HReview => "h-review",
+ Microformat::HProduct => "h-product",
}
}
}
-impl From<MicroFormat> for usize {
- fn from(value: MicroFormat) -> Self {
+impl From<Microformat> for usize {
+ fn from(value: Microformat) -> Self {
match value {
- MicroFormat::HCard => 0,
- MicroFormat::HEvent => 1,
- MicroFormat::HEntry => 2,
- MicroFormat::HRecipe => 3,
- MicroFormat::HReview => 4,
- MicroFormat::HProduct => 5,
+ Microformat::HCard => 0,
+ Microformat::HEvent => 1,
+ Microformat::HEntry => 2,
+ Microformat::HRecipe => 3,
+ Microformat::HReview => 4,
+ Microformat::HProduct => 5,
}
}
}
-const ALL_MICROFORMATS: [MicroFormat; 6] = [
- MicroFormat::HCard,
- MicroFormat::HEvent,
- MicroFormat::HEntry,
- MicroFormat::HRecipe,
- MicroFormat::HReview,
- MicroFormat::HProduct,
+impl TryFrom<usize> for Microformat {
+ type Error = anyhow::Error;
+
+ fn try_from(value: usize) -> Result<Self> {
+ match value {
+ 0 => Ok(Microformat::HCard),
+ 1 => Ok(Microformat::HEvent),
+ 2 => Ok(Microformat::HEntry),
+ 3 => Ok(Microformat::HRecipe),
+ 4 => Ok(Microformat::HReview),
+ 5 => Ok(Microformat::HProduct),
+ _ => Err(anyhow::anyhow!("Unknown microformat")),
+ }
+ }
+}
+
+const ALL_MICROFORMATS: [Microformat; 6] = [
+ Microformat::HCard,
+ Microformat::HEvent,
+ Microformat::HEntry,
+ Microformat::HRecipe,
+ Microformat::HReview,
+ Microformat::HProduct,
];
#[derive(Debug)]
@@ -765,7 +781,7 @@ impl Html {
}
}
- fn microformats(&self) -> EnumSet<MicroFormat> {
+ fn microformats(&self) -> EnumSet<Microformat> {
let mut microformats = EnumSet::new();
for node in self.root.inclusive_descendants() {
@@ -861,6 +877,17 @@ impl Html {
self.pretokenize_string(text)
}
+ fn pretokenize_microformats(&self) -> PreTokenizedString {
+ let mut text = String::new();
+
+ for microformat in self.microformats().iter() {
+ text.push_str(microformat.as_str());
+ text.push(' ');
+ }
+
+ self.pretokenize_string(text)
+ }
+
fn pretokenize_string(&self, text: String) -> PreTokenizedString {
let mut tokens = Vec::new();
@@ -884,6 +911,7 @@ impl Html {
let domain = self.pretokenize_domain();
let site = self.pretokenize_site();
let description = self.pretokenize_description();
+ let microformats = self.pretokenize_microformats();
let schemas: Vec<_> = self.schema_org();
@@ -1102,6 +1130,9 @@ impl Html {
Field::Text(TextField::FlattenedSchemaOrgJson) => {
doc.add_pre_tokenized_text(tantivy_field, pretokenized_schema_json.clone());
}
+ Field::Text(TextField::MicroformatTags) => {
+ doc.add_pre_tokenized_text(tantivy_field, microformats.clone());
+ }
Field::Fast(FastField::IsHomepage) => {
doc.add_u64(tantivy_field, (self.is_homepage()).into());
}
@@ -1116,6 +1147,9 @@ impl Html {
Field::Fast(FastField::NumUrlTokens) => {
doc.add_u64(tantivy_field, url.tokens.len() as u64)
}
+ Field::Fast(FastField::NumMicroformatTagsTokens) => {
+ doc.add_u64(tantivy_field, microformats.tokens.len() as u64)
+ }
Field::Fast(FastField::NumTitleTokens) => {
doc.add_u64(tantivy_field, title.tokens.len() as u64)
}
@@ -2330,7 +2364,7 @@ mod tests {
<time class="dt-published" datetime="2021-01-01T00:00:00+00:00">2021-01-01</time>
</article>
- <div class="h-RECIPE>
+ <div class="h-RECIPE">
For some reason this site also has a recipe
</div>
</body>
@@ -2341,8 +2375,8 @@ mod tests {
let microformats = html.microformats();
- assert!(microformats.contains(MicroFormat::HEntry));
- assert!(microformats.contains(MicroFormat::HRecipe));
- assert!(!microformats.contains(MicroFormat::HCard));
+ assert!(microformats.contains(Microformat::HEntry));
+ assert!(microformats.contains(Microformat::HRecipe));
+ assert!(!microformats.contains(Microformat::HCard));
}
}
diff --git a/optics/src/ast.rs b/optics/src/ast.rs
index c9c6ab95..ce7d84bb 100644
--- a/optics/src/ast.rs
+++ b/optics/src/ast.rs
@@ -122,6 +122,7 @@ pub enum RawMatchPart {
Title(String),
Description(String),
Content(String),
+ MicroformatTag(String),
Schema(String),
}
diff --git a/optics/src/lexer.rs b/optics/src/lexer.rs
index ab02a610..2cbfe227 100644
--- a/optics/src/lexer.rs
+++ b/optics/src/lexer.rs
@@ -41,6 +41,7 @@ pub enum Token<'a> {
Title,
Description,
Content,
+ MicroformatTag,
Schema,
Action,
Boost,
@@ -76,6 +77,7 @@ impl<'a> ToString for Token<'a> {
Token::Title => "Title".to_string(),
Token::Description => "Description".to_string(),
Token::Content => "Content".to_string(),
+ Token::MicroformatTag => "MicroformatTag".to_string(),
Token::Schema => "Schema".to_string(),
Token::Action => "Action".to_string(),
Token::Boost => "Boost".to_string(),
@@ -143,6 +145,8 @@ enum Outer<'a> {
Description,
#[token("Content")]
Content,
+ #[token("MicroformatTag")]
+ MicroformatTag,
#[token("Schema")]
Schema,
#[token("Action")]
@@ -314,6 +318,7 @@ impl<'source> Iterator for LexerBridge<'source> {
Outer::Title => Some(Ok((s.start, Token::Title, s.end))),
Outer::Description => Some(Ok((s.start, Token::Description, s.end))),
Outer::Content => Some(Ok((s.start, Token::Content, s.end))),
+ Outer::MicroformatTag => Some(Ok((s.start, Token::MicroformatTag, s.end))),
Outer::Schema => Some(Ok((s.start, Token::Schema, s.end))),
Outer::Action => Some(Ok((s.start, Token::Action, s.end))),
Outer::Boost => Some(Ok((s.start, Token::Boost, s.end))),
diff --git a/optics/src/lib.rs b/optics/src/lib.rs
index b9853f34..5b07d225 100644
--- a/optics/src/lib.rs
+++ b/optics/src/lib.rs
@@ -147,6 +147,7 @@ impl ToString for Matching {
MatchLocation::Title => s.push_str("Title"),
MatchLocation::Description => s.push_str("Description"),
MatchLocation::Content => s.push_str("Content"),
+ MatchLocation::MicroformatTag => s.push_str("MicroformatTag"),
MatchLocation::Schema => s.push_str("Schema"),
}
s.push('(');
@@ -174,6 +175,7 @@ impl TryFrom<RawMatchPart> for Matching {
RawMatchPart::Title(s) => (s, MatchLocation::Title),
RawMatchPart::Description(s) => (s, MatchLocation::Description),
RawMatchPart::Content(s) => (s, MatchLocation::Content),
+ RawMatchPart::MicroformatTag(s) => (s, MatchLocation::MicroformatTag),
RawMatchPart::Schema(s) => (s, MatchLocation::Schema),
};
@@ -282,6 +284,7 @@ pub enum MatchLocation {
Title,
Description,
Content,
+ MicroformatTag,
Schema,
}
diff --git a/optics/src/parser.lalrpop b/optics/src/parser.lalrpop
index a9ac04f2..f0d8a34e 100644
--- a/optics/src/parser.lalrpop
+++ b/optics/src/parser.lalrpop
@@ -41,6 +41,7 @@ RawMatchPart: RawMatchPart = {
"Title" "(" <StringLiteral> ")" => RawMatchPart::Title(<>.to_string()),
"Description" "(" <StringLiteral> ")" => RawMatchPart::Description(<>.to_string()),
"Content" "(" <StringLiteral> ")" => RawMatchPart::Content(<>.to_string()),
+ "MicroformatTag" "(" <StringLiteral> ")" => RawMatchPart::MicroformatTag(<>.to_string()),
"Schema" "(" <StringLiteral> ")" => RawMatchPart::Schema(<>.to_string()),
}
@@ -116,6 +117,7 @@ extern {
"Title" => Token::Title,
"Description" => Token::Description,
"Content" => Token::Content,
+ "MicroformatTag" => Token::MicroformatTag,
"Schema" => Token::Schema,
"Action" => Token::Action,
"Boost" => Token::Boost,
diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_model
index bb584ead..6855a1c2 100755
--- a/scripts/export_abstractive_summary_model
+++ b/scripts/export_abstractive_summary_model
@@ -8,7 +8,6 @@ import sys
if os.path.exists("data/summarizer/abstractive"):
print("data/summarizer/abstractive already exists. Exiting...")
sys.exit()
-
os.makedirs("data/summarizer/abstractive")
MODEL = "lidiya/bart-large-xsum-samsum"