Skip to content

web search done right

add first h1, all h2 and all h3 tags as fields in index

Mikkel Denker69ad95e4e849
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 8b9aee30..d430b00c 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -172,15 +172,15 @@ pub enum TextFieldEnum {
SiteNoTokenizer,
DomainNoTokenizer,
DomainNameNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during ranking
SiteIfHomepageNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during ranking
DomainIfHomepage,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during ranking
DomainNameIfHomepageNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during ranking
DomainIfHomepageNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during ranking
TitleIfHomepage,
BacklinkText,
Description,
@@ -209,6 +209,9 @@ pub enum TextFieldEnum {
BacklinkLabelsGroup7,
BacklinkLabelsGroup8,
BacklinkLabelsGroup9,
+ FirstH1,
+ AllH2,
+ AllH3,
}
enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
@@ -257,6 +260,9 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
BacklinkLabelsGroup7,
BacklinkLabelsGroup8,
BacklinkLabelsGroup9,
+ FirstH1,
+ AllH2,
+ AllH3,
]);
impl TextFieldEnum {
@@ -1735,3 +1741,75 @@ impl_backlink_group!(BacklinkLabelsGroup6, 6);
impl_backlink_group!(BacklinkLabelsGroup7, 7);
impl_backlink_group!(BacklinkLabelsGroup8, 8);
impl_backlink_group!(BacklinkLabelsGroup9, 9);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct FirstH1;
+impl TextField for FirstH1 {
+ fn name(&self) -> &str {
+ "first_h1"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _: &mut FnCache,
+ doc: &mut TantivyDocument,
+ index: &crate::inverted_index::InvertedIndex,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(index.schema_ref())
+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
+ html.h1().next().unwrap_or_default(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct AllH2;
+impl TextField for AllH2 {
+ fn name(&self) -> &str {
+ "all_h2"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _: &mut FnCache,
+ doc: &mut TantivyDocument,
+ index: &crate::inverted_index::InvertedIndex,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(index.schema_ref())
+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
+ html.h2().join("\n"),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct AllH3;
+impl TextField for AllH3 {
+ fn name(&self) -> &str {
+ "all_h3"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _: &mut FnCache,
+ doc: &mut TantivyDocument,
+ index: &crate::inverted_index::InvertedIndex,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(index.schema_ref())
+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
+ html.h3().join("\n"),
+ );
+
+ Ok(())
+ }
+}
diff --git a/crates/core/src/webpage/html/headings.rs b/crates/core/src/webpage/html/headings.rs
new file mode 100644
index 00000000..854dd557
--- /dev/null
+++ b/crates/core/src/webpage/html/headings.rs
@@ -0,0 +1,68 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::Html;
+
+impl Html {
+ pub fn h1(&self) -> impl Iterator<Item = String> {
+ self.root
+ .select("h1")
+ .expect("css selector should be valid")
+ .map(|node| node.as_node().text_contents().trim().to_string())
+ }
+
+ pub fn h2(&self) -> impl Iterator<Item = String> {
+ self.root
+ .select("h2")
+ .expect("css selector should be valid")
+ .map(|node| node.as_node().text_contents().trim().to_string())
+ }
+
+ pub fn h3(&self) -> impl Iterator<Item = String> {
+ self.root
+ .select("h3")
+ .expect("css selector should be valid")
+ .map(|node| node.as_node().text_contents().trim().to_string())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use itertools::Itertools;
+
+ #[test]
+ fn test_h1() {
+ let html = Html::parse("<h1>Hello</h1><h2>World</h2>", "https://example.com").unwrap();
+ assert_eq!(html.h1().collect_vec(), ["Hello"]);
+ }
+
+ #[test]
+ fn test_h2() {
+ let html = Html::parse("<h1>Hello</h1><h2>World</h2>", "https://example.com").unwrap();
+ assert_eq!(html.h2().collect_vec(), ["World"]);
+ }
+
+ #[test]
+ fn test_h3() {
+ let html = Html::parse(
+ "<h1>Hello</h1><h2>World</h2><h3>!</h3>",
+ "https://example.com",
+ )
+ .unwrap();
+ assert_eq!(html.h3().collect_vec(), ["!"]);
+ }
+}
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index daf0ab01..c5a1e6f6 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -33,6 +33,7 @@ pub use fn_cache::FnCache;
mod feeds;
mod fn_cache;
+mod headings;
mod into_tantivy;
pub mod links;
mod microformats;