Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow dict field with wildcard definition.

Kishore Nallan4b479291e4fb
diff --git a/src/collection.cpp b/src/collection.cpp
index a3c51725..56e29282 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2320,7 +2320,6 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
if(std::regex_match (kv.key(), std::regex(dynamic_field.name))) {
new_field = dynamic_field;
new_field.name = fname;
- new_field.type = dynamic_field.type;
found_dynamic_field = true;
break;
}
@@ -2357,6 +2356,11 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
continue;
}
+ if(kv.value().is_object()) {
+ kv++;
+ continue;
+ }
+
if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {
return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");
} else {
diff --git a/src/index.cpp b/src/index.cpp
index 19885cba..cbf38bef 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -309,15 +309,6 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3
return coerce_op;
}
}
-
- if(document[field_name].get<int64_t>() > INT32_MAX) {
- if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT)) {
- document.erase(field_name);
- continue;
- } else {
- return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
- }
- }
} else if(a_field.type == field_types::INT64 && !document[field_name].is_number_integer()) {
Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);
if(!coerce_op.ok()) {
@@ -537,6 +528,12 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
for(size_t i = 0; i < iter_batch.size(); i++) {
auto& index_rec = iter_batch[i];
+
+ if(!index_rec.indexed.ok()) {
+ // some records could have been invalidated upstream
+ continue;
+ }
+
if(index_rec.is_update) {
index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);
} else if(index_rec.indexed.ok()) {
@@ -581,6 +578,10 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
if(afield.name == "id") {
for(const auto& record: iter_batch) {
+ if(!record.indexed.ok()) {
+ // some records could have been invalidated upstream
+ continue;
+ }
if(!record.is_update && record.indexed.ok()) {
// for updates, the seq_id will already exist
seq_ids.append(record.seq_id);
@@ -602,6 +603,11 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
int64_t max_score = INT64_MIN;
for(const auto& record: iter_batch) {
+ if(!record.indexed.ok()) {
+ // some records could have been invalidated upstream
+ continue;
+ }
+
const auto& document = record.doc;
const auto seq_id = record.seq_id;
@@ -3615,6 +3621,7 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const f
array_iter = document[field_name].erase(array_iter);
array_ele_erased = true;
}
+
return Option<uint32_t>(200);
}
@@ -3650,6 +3657,14 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const f
}
}
+ if(document.contains(field_name) && document[field_name].get<int64_t>() > INT32_MAX) {
+ if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT)) {
+ document.erase(field_name);
+ } else {
+ return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
+ }
+ }
+
return Option<uint32_t>(200);
}
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 10f8fffa..6ef6103d 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -406,6 +406,7 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {
doc["title"] = "THIRD";
add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `int_values` must be an array of string.", add_op.error());
// singular field coercion
doc["single_int"] = 100;
@@ -413,41 +414,64 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {
add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `single_int` must be a string.", add_op.error());
- // uncoercable field, e.g. nested dict
- doc["dict"] = nlohmann::json::object();
- doc["dict"]["one"] = 1;
- doc["dict"]["two"] = 2;
+ // try with empty array
+ doc["title"] = "FIFTH";
+ doc["int_values"] = {"100"};
+ doc["int_values_2"] = nlohmann::json::array();
+ doc["single_int"] = "200";
add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
ASSERT_FALSE(add_op.ok());
- ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());
+ ASSERT_EQ("Type of field `int_values_2` is invalid.", add_op.error());
- // try with coerce_or_reject
- add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);
- ASSERT_FALSE(add_op.ok());
- ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());
+ collectionManager.drop_collection("coll1");
+}
- // try with drop
- doc["title"] = "FIFTH";
- add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);
+TEST_F(CollectionAllFieldsTest, IntegerAllValues) {
+ Collection *coll1;
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, {}, "", 0, "int32").get();
+ }
+
+ nlohmann::json doc;
+ doc["age"] = 100;
+ doc["year"] = 2000;
+
+ Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0");
ASSERT_TRUE(add_op.ok());
+ auto added_doc = add_op.get();
- results = coll1->search("fifth", {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ("FIFTH", results["hits"][0]["document"]["title"].get<std::string>());
- ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));
+
+ // try with DROP
+ doc["age"] = "SECOND";
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(2, results["hits"].size());
+
+ // try with REJECT
+ doc["age"] = "THIRD";
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `age` must be an int32.", add_op.error());
+
+ // try with coerce_or_reject
+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `age` must be an int32.", add_op.error());
// try with coerce_or_drop
- doc["title"] = "SIXTH";
+ doc["age"] = "FOURTH";
add_op = coll1->add(doc.dump(), CREATE, "66", DIRTY_VALUES::COERCE_OR_DROP);
ASSERT_TRUE(add_op.ok());
- results = coll1->search("sixth", {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
- ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ("SIXTH", results["hits"][0]["document"]["title"].get<std::string>());
- ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));
-
collectionManager.drop_collection("coll1");
}
@@ -1060,6 +1084,54 @@ TEST_F(CollectionAllFieldsTest, BothFallbackAndDynamicFields) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionAllFieldsTest, WildcardFieldAndDictionaryField) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);
+ ASSERT_TRUE(op.ok());
+ coll1 = op.get();
+ }
+
+ nlohmann::json doc;
+ doc["kinds"] = nlohmann::json::object();
+ doc["kinds"]["CGXX"] = 13;
+ doc["kinds"]["ZBXX"] = 24;
+
+ auto add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionAllFieldsTest, DynamicFieldAndDictionaryField) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("k.*", field_types::STRING, false, true),
+ field(".*", field_types::AUTO, false, true)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);
+ ASSERT_TRUE(op.ok());
+ coll1 = op.get();
+ }
+
+ nlohmann::json doc;
+ doc["kinds"] = nlohmann::json::object();
+ doc["kinds"]["CGXX"] = 13;
+ doc["kinds"]["ZBXX"] = 24;
+
+ auto add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `kinds` must be a string.", add_op.error());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionAllFieldsTest, RegexpIntFieldWithFallbackStringType) {
Collection *coll1;