web search done right
specialized updated_time function
Mikkel Denker82e7be6d20ba
diff --git a/Cargo.toml b/Cargo.tomlindex 1dfc2e25..36650937 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -45,6 +45,7 @@ fst = "0.4.7"lazy_static = "1.4.0"rocksdb = "0.18.0"image = "0.24.3"+chrono = "0.4.20"[dev-dependencies]maplit = "1.0.2"diff --git a/src/query/mod.rs b/src/query/mod.rsindex 2d80e406..249de3ee 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -102,6 +102,7 @@ impl tantivy::query::Query for Query {let fields: Vec<_> = ALL_FIELDS.iter()+ .filter(|field| field.is_searchable()).map(|field| {let tantivy_field = searcher.schema().get_field(field.as_str()).unwrap();let analyzer = searcher.index().tokenizer_for_field(tantivy_field).ok();diff --git a/src/schema.rs b/src/schema.rsindex b370911c..29b51511 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -134,6 +134,12 @@ impl Field {| Field::FetchTimeMs => None,}}++ pub fn is_searchable(&self) -> bool {+ match self {+ _ => true,+ }+ }}pub fn create_schema() -> tantivy::schema::Schema {diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex a1045538..52cf2014 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{image_store::Image, schema_org::SchemaOrg, Error, Result};+use chrono::{DateTime, FixedOffset};use logos::{Lexer, Logos};use std::{collections::HashMap, time::Duration};@@ -495,6 +496,23 @@ impl<'a> Html<'a> {schemas}++ pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {+ self.metadata()+ .into_iter()+ .find(|metadata| {+ if let Some(property) = metadata.get("property") {+ property == &String::from("og:updated_time")+ } else {+ false+ }+ })+ .and_then(|metadata| {+ metadata+ .get("content")+ .and_then(|time| DateTime::parse_from_rfc3339(time).ok())+ })+ }}#[derive(Debug, PartialEq, Eq)]@@ -868,4 +886,61 @@ mod tests {assert!(html.schema_org().is_empty())}++ #[test]+ fn metadata_updated_time() {+ let html = r#"+ <html>+ <head>+ <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(+ html.updated_time(),+ Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())+ );++ let html = r#"+ <html>+ <head>+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(html.updated_time(), None);++ let html = r#"+ <html>+ <head>+ <meta property="og:whutwhut" content="2022-06-22T19:37:34+00:00" />+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(html.updated_time(), None);++ let html = r#"+ <html>+ <head>+ <meta property="og:updated_time" content="2ss022-06-22T19:37:34+00:00" />+ </head>+ <body>+ </body>+ </html>+ "#;+ let html = Html::parse(html, "example.com");++ assert_eq!(html.updated_time(), None);+ }}