web search done right
add first h1, all h2 and all h3 tags as fields in index
Mikkel Denker69ad95e4e849
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 8b9aee30..d430b00c 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -172,15 +172,15 @@ pub enum TextFieldEnum {SiteNoTokenizer,DomainNoTokenizer,DomainNameNoTokenizer,- /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during rankingSiteIfHomepageNoTokenizer,- /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during rankingDomainIfHomepage,- /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during rankingDomainNameIfHomepageNoTokenizer,- /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during rankingDomainIfHomepageNoTokenizer,- /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost during rankingTitleIfHomepage,BacklinkText,Description,@@ -209,6 +209,9 @@ pub enum TextFieldEnum {BacklinkLabelsGroup7,BacklinkLabelsGroup8,BacklinkLabelsGroup9,+ FirstH1,+ AllH2,+ AllH3,}enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,@@ -257,6 +260,9 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,BacklinkLabelsGroup7,BacklinkLabelsGroup8,BacklinkLabelsGroup9,+ FirstH1,+ AllH2,+ AllH3,]);impl TextFieldEnum {@@ -1735,3 +1741,75 @@ impl_backlink_group!(BacklinkLabelsGroup6, 6);impl_backlink_group!(BacklinkLabelsGroup7, 7);impl_backlink_group!(BacklinkLabelsGroup8, 8);impl_backlink_group!(BacklinkLabelsGroup9, 9);++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct FirstH1;+impl TextField for FirstH1 {+ fn name(&self) -> &str {+ "first_h1"+ }++ fn add_html_tantivy(+ &self,+ html: &Html,+ _: &mut FnCache,+ doc: &mut TantivyDocument,+ index: &crate::inverted_index::InvertedIndex,+ ) -> Result<()> {+ doc.add_text(+ self.tantivy_field(index.schema_ref())+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),+ html.h1().next().unwrap_or_default(),+ );++ Ok(())+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct AllH2;+impl TextField for AllH2 {+ fn name(&self) -> &str {+ "all_h2"+ }++ fn add_html_tantivy(+ &self,+ html: &Html,+ _: &mut FnCache,+ doc: &mut TantivyDocument,+ index: &crate::inverted_index::InvertedIndex,+ ) -> Result<()> {+ doc.add_text(+ self.tantivy_field(index.schema_ref())+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),+ html.h2().join("\n"),+ );++ Ok(())+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct AllH3;+impl TextField for AllH3 {+ fn name(&self) -> &str {+ "all_h3"+ }++ fn add_html_tantivy(+ &self,+ html: &Html,+ _: &mut FnCache,+ doc: &mut TantivyDocument,+ index: &crate::inverted_index::InvertedIndex,+ ) -> Result<()> {+ doc.add_text(+ self.tantivy_field(index.schema_ref())+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),+ html.h3().join("\n"),+ );++ Ok(())+ }+}diff --git a/crates/core/src/webpage/html/headings.rs b/crates/core/src/webpage/html/headings.rsnew file mode 100644index 00000000..854dd557--- /dev/null+++ b/crates/core/src/webpage/html/headings.rs@@ -0,0 +1,68 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::Html;++impl Html {+ pub fn h1(&self) -> impl Iterator<Item = String> {+ self.root+ .select("h1")+ .expect("css selector should be valid")+ .map(|node| node.as_node().text_contents().trim().to_string())+ }++ pub fn h2(&self) -> impl Iterator<Item = String> {+ self.root+ .select("h2")+ .expect("css selector should be valid")+ .map(|node| node.as_node().text_contents().trim().to_string())+ }++ pub fn h3(&self) -> impl Iterator<Item = String> {+ self.root+ .select("h3")+ .expect("css selector should be valid")+ .map(|node| node.as_node().text_contents().trim().to_string())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use itertools::Itertools;++ #[test]+ fn test_h1() {+ let html = Html::parse("<h1>Hello</h1><h2>World</h2>", "https://example.com").unwrap();+ assert_eq!(html.h1().collect_vec(), ["Hello"]);+ }++ #[test]+ fn test_h2() {+ let html = Html::parse("<h1>Hello</h1><h2>World</h2>", "https://example.com").unwrap();+ assert_eq!(html.h2().collect_vec(), ["World"]);+ }++ #[test]+ fn test_h3() {+ let html = Html::parse(+ "<h1>Hello</h1><h2>World</h2><h3>!</h3>",+ "https://example.com",+ )+ .unwrap();+ assert_eq!(html.h3().collect_vec(), ["!"]);+ }+}diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex daf0ab01..c5a1e6f6 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -33,6 +33,7 @@ pub use fn_cache::FnCache;mod feeds;mod fn_cache;+mod headings;mod into_tantivy;pub mod links;mod microformats;