Skip to content

web search done right

specialized updated_time function

Mikkel Denker82e7be6d20ba
diff --git a/Cargo.toml b/Cargo.toml
index 1dfc2e25..36650937 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,6 +45,7 @@ fst = "0.4.7"
lazy_static = "1.4.0"
rocksdb = "0.18.0"
image = "0.24.3"
+chrono = "0.4.20"
[dev-dependencies]
maplit = "1.0.2"
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 2d80e406..249de3ee 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -102,6 +102,7 @@ impl tantivy::query::Query for Query {
let fields: Vec<_> = ALL_FIELDS
.iter()
+ .filter(|field| field.is_searchable())
.map(|field| {
let tantivy_field = searcher.schema().get_field(field.as_str()).unwrap();
let analyzer = searcher.index().tokenizer_for_field(tantivy_field).ok();
diff --git a/src/schema.rs b/src/schema.rs
index b370911c..29b51511 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -134,6 +134,12 @@ impl Field {
| Field::FetchTimeMs => None,
}
}
+
+ pub fn is_searchable(&self) -> bool {
+ match self {
+ _ => true,
+ }
+ }
}
pub fn create_schema() -> tantivy::schema::Schema {
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index a1045538..52cf2014 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{image_store::Image, schema_org::SchemaOrg, Error, Result};
+use chrono::{DateTime, FixedOffset};
use logos::{Lexer, Logos};
use std::{collections::HashMap, time::Duration};
@@ -495,6 +496,23 @@ impl<'a> Html<'a> {
schemas
}
+
+ pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("og:updated_time")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| {
+ metadata
+ .get("content")
+ .and_then(|time| DateTime::parse_from_rfc3339(time).ok())
+ })
+ }
}
#[derive(Debug, PartialEq, Eq)]
@@ -868,4 +886,61 @@ mod tests {
assert!(html.schema_org().is_empty())
}
+
+ #[test]
+ fn metadata_updated_time() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(
+ html.updated_time(),
+ Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())
+ );
+
+ let html = r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(html.updated_time(), None);
+
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:whutwhut" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(html.updated_time(), None);
+
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:updated_time" content="2ss022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "example.com");
+
+ assert_eq!(html.updated_time(), None);
+ }
}