Skip to content

web search done right

distinguish between itemtypes and regular keys in schema flattened json to ensure schema matchings in optics always start their match against an itemtype

Mikkel Denkerd24dce8831a0
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 6c98a103..e4be7350 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -15,14 +15,14 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use itertools::Itertools;
-use optics::{Action, MatchLocation, Matching, Optic, Rule};
+use optics::{Action, MatchLocation, Matching, Optic, PatternPart, Rule};
use std::iter;
use tantivy::{
query::{BooleanQuery, Occur, QueryClone},
schema::Schema,
};
-use crate::{fastfield_reader::FastFieldReader, schema::text_field};
+use crate::{fastfield_reader::FastFieldReader, schema::text_field, webpage::schema_org};
use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};
@@ -247,15 +247,27 @@ impl AsTantivyQuery for Matching {
)),
1.0,
)),
- MatchLocation::Schema => Box::new(ConstQuery::new(
- Box::new(PatternQuery::new(
- self.pattern.clone(),
- text_field::FlattenedSchemaOrgJson.into(),
- schema,
- fastfield_reader.clone(),
- )),
- 1.0,
- )),
+ MatchLocation::Schema => {
+ let mut pattern = self.pattern.clone();
+ // add TYPE_PREFIX to first term in pattern to ensure
+ // we match from the beginning of the path
+ if let Some(PatternPart::Raw(first_term)) = pattern
+ .iter_mut()
+ .find(|p| matches!(p, PatternPart::Raw(_)))
+ {
+ *first_term = format!("{}{}", schema_org::TYPE_PREFIX, first_term);
+ }
+
+ Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ pattern,
+ text_field::FlattenedSchemaOrgJson.into(),
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ ))
+ }
}
}
}
@@ -794,7 +806,8 @@ mod tests {
"contentUrl": "mexico-beach.jpg",
"datePublished": "2008-01-25",
"description": "I took this picture while on vacation last year.",
- "name": "Beach in Mexico"
+ "name": "Beach in Mexico",
+ "BlogPosting": "whatever"
}}
</script>
</head>
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 8b112306..b283b5a4 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -1159,7 +1159,7 @@ impl TextField for FlattenedSchemaOrgJson {
doc.add_pre_tokenized_text(
self.tantivy_field(schema)
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
- cache.pretokenized_schema_json().clone(),
+ dbg!(cache.pretokenized_schema_json().clone()),
);
Ok(())
diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rs
index b03ed771..8985fe7e 100644
--- a/crates/core/src/tokenizer/mod.rs
+++ b/crates/core/src/tokenizer/mod.rs
@@ -576,7 +576,7 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {
let mut prev_was_quote = false;
while let Some((offset_from, c)) = self.chars.next() {
- if c.is_alphanumeric() {
+ if !matches!(c, '.' | '\n' | '"') {
let offset_to = self.search_token_end(prev_was_quote);
self.token.offset_from = offset_from;
self.token.offset_to = offset_to;
@@ -866,31 +866,19 @@ Breadcrumb.url="https://www.eurotecnicaservice.it/testing\"
#[test]
fn tokenize_json_field() {
assert_eq!(
- tokenize_json(
- r#"
- Test.field="value"
- "#
- ),
+ tokenize_json(r#"Test.field="value""#),
vec!["Test", "field", "\"value\"",]
);
assert_eq!(
- tokenize_json(
- r#"
- Test.field="this is the value"
- "#
- ),
+ tokenize_json(r#"Test.field="this is the value""#),
vec!["Test", "field", "\"this is the value\"",]
);
assert_eq!(
- tokenize_json(
- r#"
- Test.field="this is\" the value"
- "#
- ),
+ tokenize_json(r#"Test.field="this is\" the value""#),
vec!["Test", "field", "\"this is\\\" the value\"",]
);
assert_eq!(
- tokenize_json("\n Test.field=\"this*@# is\\\" the\\\" \nvalue\"\n "),
+ tokenize_json("Test.field=\"this*@# is\\\" the\\\" \nvalue\""),
vec!["Test", "field", "\"this*@# is\\\" the\\\" \nvalue\"",]
);
}
@@ -939,6 +927,19 @@ key1.key3="value2""#;
flattened_json_helper(json, expected);
+ let json = r#"
+ {
+ "$key1": {
+ "$key2": "value1",
+ "key3": "value2"
+ }
+ }
+ "#;
+ let expected = r#"$key1.$key2="value1"
+$key1.key3="value2""#;
+
+ flattened_json_helper(json, expected);
+
let json = r#"
{
"key1": [
diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rs
index daf66c59..90d22169 100644
--- a/crates/core/src/webpage/html/fn_cache.rs
+++ b/crates/core/src/webpage/html/fn_cache.rs
@@ -106,7 +106,7 @@ impl<'a> FnCache<'a> {
pub fn pretokenized_schema_json(&mut self) -> &PreTokenizedString {
if self.pretokenized_schema_json.is_none() {
self.pretokenized_schema_json =
- match schema_org::flattened_json(self.schema_org().clone()) {
+ match schema_org::flattened_json(dbg!(self.schema_org().clone())) {
Ok(mut f) => {
let mut tokens = Vec::new();
diff --git a/crates/core/src/webpage/schema_org/mod.rs b/crates/core/src/webpage/schema_org/mod.rs
index 51b0e952..73c6d635 100644
--- a/crates/core/src/webpage/schema_org/mod.rs
+++ b/crates/core/src/webpage/schema_org/mod.rs
@@ -24,6 +24,11 @@ use crate::{OneOrMany, Result};
mod json_ld;
mod microdata;
+/// All itemtypes will be prefixed with TYPE_PREFIX
+/// in the flattened json. This allows us to make sure that
+/// the matching during search starts at an itemtype.
+pub const TYPE_PREFIX: char = '$';
+
#[derive(
Debug,
Clone,
@@ -65,15 +70,15 @@ impl Property {
bincode::Decode,
)]
#[serde(untagged)]
-enum SingleMap {
+enum FlattenedJsonMap {
Leaf(String),
- Node(HashMap<RawOneOrMany<String>, HashMap<String, RawOneOrMany<SingleMap>>>),
+ Node(HashMap<RawOneOrMany<String>, HashMap<String, RawOneOrMany<FlattenedJsonMap>>>),
}
-impl From<Property> for SingleMap {
+impl From<Property> for FlattenedJsonMap {
fn from(value: Property) -> Self {
match value {
- Property::String(s) => SingleMap::Leaf(s),
+ Property::String(s) => FlattenedJsonMap::Leaf(s),
Property::Item(item) => {
let mut res = HashMap::new();
@@ -81,22 +86,47 @@ impl From<Property> for SingleMap {
let recursive = item
.properties
.into_iter()
+ .map(|(key, val)| {
+ (key.chars().skip_while(|c| *c == TYPE_PREFIX).collect(), val)
+ })
.map(|(key, val)| match val {
- OneOrMany::One(one) => (key, RawOneOrMany::One(SingleMap::from(one))),
+ OneOrMany::One(one) => {
+ (key, RawOneOrMany::One(FlattenedJsonMap::from(one)))
+ }
OneOrMany::Many(many) => (
key,
- RawOneOrMany::Many(many.into_iter().map(SingleMap::from).collect()),
+ RawOneOrMany::Many(
+ many.into_iter().map(FlattenedJsonMap::from).collect(),
+ ),
),
})
.collect();
match tt {
- OneOrMany::One(one) => res.insert(RawOneOrMany::One(one), recursive),
- OneOrMany::Many(many) => res.insert(RawOneOrMany::Many(many), recursive),
+ OneOrMany::One(one) => {
+ let one: String =
+ one.chars().skip_while(|c| *c == TYPE_PREFIX).collect();
+ let one = TYPE_PREFIX.to_string() + one.as_str();
+ res.insert(RawOneOrMany::One(one), recursive)
+ }
+
+ OneOrMany::Many(many) => {
+ let many = many
+ .into_iter()
+ .map(|s| {
+ s.chars()
+ .skip_while(|c| *c == TYPE_PREFIX)
+ .collect::<String>()
+ })
+ .map(|s| TYPE_PREFIX.to_string() + s.as_str())
+ .collect();
+
+ res.insert(RawOneOrMany::Many(many), recursive)
+ }
};
}
- SingleMap::Node(res)
+ FlattenedJsonMap::Node(res)
}
}
}
@@ -169,8 +199,8 @@ impl Item {
}
}
- fn into_single_map(self) -> SingleMap {
- SingleMap::from(Property::Item(self))
+ fn into_flattened_json_map(self) -> FlattenedJsonMap {
+ FlattenedJsonMap::from(Property::Item(self))
}
}
@@ -241,7 +271,7 @@ pub fn parse(root: NodeRef) -> Vec<Item> {
pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {
let single_maps: Vec<_> = schemas
.into_iter()
- .map(|item| item.into_single_map())
+ .map(|item| item.into_flattened_json_map())
.collect();
FlattenedJson::new(&single_maps)
}
diff --git a/crates/optics/src/lib.rs b/crates/optics/src/lib.rs
index 1745fe94..21a5fd74 100644
--- a/crates/optics/src/lib.rs
+++ b/crates/optics/src/lib.rs
@@ -198,21 +198,11 @@ impl TryFrom<RawMatchPart> for Matching {
let mut pattern = Vec::new();
- if matches!(&loc, MatchLocation::Schema) {
- for tok in PatternToken::lex(&s) {
- match tok {
- PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),
- PatternToken::Wildcard => return Err(Error::Pattern),
- PatternToken::Anchor => return Err(Error::Pattern),
- }
- }
- } else {
- for tok in PatternToken::lex(&s) {
- match tok {
- PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),
- PatternToken::Wildcard => pattern.push(PatternPart::Wildcard),
- PatternToken::Anchor => pattern.push(PatternPart::Anchor),
- }
+ for tok in PatternToken::lex(&s) {
+ match tok {
+ PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),
+ PatternToken::Wildcard => pattern.push(PatternPart::Wildcard),
+ PatternToken::Anchor => pattern.push(PatternPart::Anchor),
}
}