web search done right
distinguish between itemtypes and regular keys in schema flattened json to ensure schema matchings in optics always start their match against an itemtype
Mikkel Denkerd24dce8831a0
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 6c98a103..e4be7350 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -15,14 +15,14 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use itertools::Itertools;-use optics::{Action, MatchLocation, Matching, Optic, Rule};+use optics::{Action, MatchLocation, Matching, Optic, PatternPart, Rule};use std::iter;use tantivy::{query::{BooleanQuery, Occur, QueryClone},schema::Schema,};-use crate::{fastfield_reader::FastFieldReader, schema::text_field};+use crate::{fastfield_reader::FastFieldReader, schema::text_field, webpage::schema_org};use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};@@ -247,15 +247,27 @@ impl AsTantivyQuery for Matching {)),1.0,)),- MatchLocation::Schema => Box::new(ConstQuery::new(- Box::new(PatternQuery::new(- self.pattern.clone(),- text_field::FlattenedSchemaOrgJson.into(),- schema,- fastfield_reader.clone(),- )),- 1.0,- )),+ MatchLocation::Schema => {+ let mut pattern = self.pattern.clone();+ // add TYPE_PREFIX to first term in pattern to ensure+ // we match from the beginning of the path+ if let Some(PatternPart::Raw(first_term)) = pattern+ .iter_mut()+ .find(|p| matches!(p, PatternPart::Raw(_)))+ {+ *first_term = format!("{}{}", schema_org::TYPE_PREFIX, first_term);+ }++ Box::new(ConstQuery::new(+ Box::new(PatternQuery::new(+ pattern,+ text_field::FlattenedSchemaOrgJson.into(),+ schema,+ fastfield_reader.clone(),+ )),+ 1.0,+ ))+ }}}}@@ -794,7 +806,8 @@ mod tests {"contentUrl": "mexico-beach.jpg","datePublished": "2008-01-25","description": "I took this picture while on vacation last year.",- "name": "Beach in Mexico"+ "name": "Beach in Mexico",+ "BlogPosting": "whatever"}}</script></head>diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 8b112306..b283b5a4 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -1159,7 +1159,7 @@ impl TextField for FlattenedSchemaOrgJson {doc.add_pre_tokenized_text(self.tantivy_field(schema).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),- cache.pretokenized_schema_json().clone(),+ dbg!(cache.pretokenized_schema_json().clone()),);Ok(())diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rsindex b03ed771..8985fe7e 100644--- a/crates/core/src/tokenizer/mod.rs+++ b/crates/core/src/tokenizer/mod.rs@@ -576,7 +576,7 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {let mut prev_was_quote = false;while let Some((offset_from, c)) = self.chars.next() {- if c.is_alphanumeric() {+ if !matches!(c, '.' | '\n' | '"') {let offset_to = self.search_token_end(prev_was_quote);self.token.offset_from = offset_from;self.token.offset_to = offset_to;@@ -866,31 +866,19 @@ Breadcrumb.url="https://www.eurotecnicaservice.it/testing\"#[test]fn tokenize_json_field() {assert_eq!(- tokenize_json(- r#"- Test.field="value"- "#- ),+ tokenize_json(r#"Test.field="value""#),vec!["Test", "field", "\"value\"",]);assert_eq!(- tokenize_json(- r#"- Test.field="this is the value"- "#- ),+ tokenize_json(r#"Test.field="this is the value""#),vec!["Test", "field", "\"this is the value\"",]);assert_eq!(- tokenize_json(- r#"- Test.field="this is\" the value"- "#- ),+ tokenize_json(r#"Test.field="this is\" the value""#),vec!["Test", "field", "\"this is\\\" the value\"",]);assert_eq!(- tokenize_json("\n Test.field=\"this*@# is\\\" the\\\" \nvalue\"\n "),+ tokenize_json("Test.field=\"this*@# is\\\" the\\\" \nvalue\""),vec!["Test", "field", "\"this*@# is\\\" the\\\" \nvalue\"",]);}@@ -939,6 +927,19 @@ key1.key3="value2""#;flattened_json_helper(json, expected);+ let json = r#"+ {+ "$key1": {+ "$key2": "value1",+ "key3": "value2"+ }+ }+ "#;+ let expected = r#"$key1.$key2="value1"+$key1.key3="value2""#;++ flattened_json_helper(json, expected);+let json = r#"{"key1": [diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rsindex daf66c59..90d22169 100644--- a/crates/core/src/webpage/html/fn_cache.rs+++ b/crates/core/src/webpage/html/fn_cache.rs@@ -106,7 +106,7 @@ impl<'a> FnCache<'a> {pub fn pretokenized_schema_json(&mut self) -> &PreTokenizedString {if self.pretokenized_schema_json.is_none() {self.pretokenized_schema_json =- match schema_org::flattened_json(self.schema_org().clone()) {+ match schema_org::flattened_json(dbg!(self.schema_org().clone())) {Ok(mut f) => {let mut tokens = Vec::new();diff --git a/crates/core/src/webpage/schema_org/mod.rs b/crates/core/src/webpage/schema_org/mod.rsindex 51b0e952..73c6d635 100644--- a/crates/core/src/webpage/schema_org/mod.rs+++ b/crates/core/src/webpage/schema_org/mod.rs@@ -24,6 +24,11 @@ use crate::{OneOrMany, Result};mod json_ld;mod microdata;+/// All itemtypes will be prefixed with TYPE_PREFIX+/// in the flattened json. This allows us to make sure that+/// the matching during search starts at an itemtype.+pub const TYPE_PREFIX: char = '$';+#[derive(Debug,Clone,@@ -65,15 +70,15 @@ impl Property {bincode::Decode,)]#[serde(untagged)]-enum SingleMap {+enum FlattenedJsonMap {Leaf(String),- Node(HashMap<RawOneOrMany<String>, HashMap<String, RawOneOrMany<SingleMap>>>),+ Node(HashMap<RawOneOrMany<String>, HashMap<String, RawOneOrMany<FlattenedJsonMap>>>),}-impl From<Property> for SingleMap {+impl From<Property> for FlattenedJsonMap {fn from(value: Property) -> Self {match value {- Property::String(s) => SingleMap::Leaf(s),+ Property::String(s) => FlattenedJsonMap::Leaf(s),Property::Item(item) => {let mut res = HashMap::new();@@ -81,22 +86,47 @@ impl From<Property> for SingleMap {let recursive = item.properties.into_iter()+ .map(|(key, val)| {+ (key.chars().skip_while(|c| *c == TYPE_PREFIX).collect(), val)+ }).map(|(key, val)| match val {- OneOrMany::One(one) => (key, RawOneOrMany::One(SingleMap::from(one))),+ OneOrMany::One(one) => {+ (key, RawOneOrMany::One(FlattenedJsonMap::from(one)))+ }OneOrMany::Many(many) => (key,- RawOneOrMany::Many(many.into_iter().map(SingleMap::from).collect()),+ RawOneOrMany::Many(+ many.into_iter().map(FlattenedJsonMap::from).collect(),+ ),),}).collect();match tt {- OneOrMany::One(one) => res.insert(RawOneOrMany::One(one), recursive),- OneOrMany::Many(many) => res.insert(RawOneOrMany::Many(many), recursive),+ OneOrMany::One(one) => {+ let one: String =+ one.chars().skip_while(|c| *c == TYPE_PREFIX).collect();+ let one = TYPE_PREFIX.to_string() + one.as_str();+ res.insert(RawOneOrMany::One(one), recursive)+ }++ OneOrMany::Many(many) => {+ let many = many+ .into_iter()+ .map(|s| {+ s.chars()+ .skip_while(|c| *c == TYPE_PREFIX)+ .collect::<String>()+ })+ .map(|s| TYPE_PREFIX.to_string() + s.as_str())+ .collect();++ res.insert(RawOneOrMany::Many(many), recursive)+ }};}- SingleMap::Node(res)+ FlattenedJsonMap::Node(res)}}}@@ -169,8 +199,8 @@ impl Item {}}- fn into_single_map(self) -> SingleMap {- SingleMap::from(Property::Item(self))+ fn into_flattened_json_map(self) -> FlattenedJsonMap {+ FlattenedJsonMap::from(Property::Item(self))}}@@ -241,7 +271,7 @@ pub fn parse(root: NodeRef) -> Vec<Item> {pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {let single_maps: Vec<_> = schemas.into_iter()- .map(|item| item.into_single_map())+ .map(|item| item.into_flattened_json_map()).collect();FlattenedJson::new(&single_maps)}diff --git a/crates/optics/src/lib.rs b/crates/optics/src/lib.rsindex 1745fe94..21a5fd74 100644--- a/crates/optics/src/lib.rs+++ b/crates/optics/src/lib.rs@@ -198,21 +198,11 @@ impl TryFrom<RawMatchPart> for Matching {let mut pattern = Vec::new();- if matches!(&loc, MatchLocation::Schema) {- for tok in PatternToken::lex(&s) {- match tok {- PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),- PatternToken::Wildcard => return Err(Error::Pattern),- PatternToken::Anchor => return Err(Error::Pattern),- }- }- } else {- for tok in PatternToken::lex(&s) {- match tok {- PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),- PatternToken::Wildcard => pattern.push(PatternPart::Wildcard),- PatternToken::Anchor => pattern.push(PatternPart::Anchor),- }+ for tok in PatternToken::lex(&s) {+ match tok {+ PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),+ PatternToken::Wildcard => pattern.push(PatternPart::Wildcard),+ PatternToken::Anchor => pattern.push(PatternPart::Anchor),}}