Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow dict field with wildcard definition.
Kishore Nallan4b479291e4fb
diff --git a/src/collection.cpp b/src/collection.cppindex a3c51725..56e29282 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2320,7 +2320,6 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constif(std::regex_match (kv.key(), std::regex(dynamic_field.name))) {new_field = dynamic_field;new_field.name = fname;- new_field.type = dynamic_field.type;found_dynamic_field = true;break;}@@ -2357,6 +2356,11 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constcontinue;}+ if(kv.value().is_object()) {+ kv++;+ continue;+ }+if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");} else {diff --git a/src/index.cpp b/src/index.cppindex 19885cba..cbf38bef 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -309,15 +309,6 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3return coerce_op;}}-- if(document[field_name].get<int64_t>() > INT32_MAX) {- if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT)) {- document.erase(field_name);- continue;- } else {- return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");- }- }} else if(a_field.type == field_types::INT64 && !document[field_name].is_number_integer()) {Option<uint32_t> coerce_op = coerce_int64_t(dirty_values, a_field, document, field_name, dummy_iter, false, array_ele_erased);if(!coerce_op.ok()) {@@ -537,6 +528,12 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bfor(size_t i = 0; i < iter_batch.size(); i++) {auto& index_rec = iter_batch[i];++ if(!index_rec.indexed.ok()) {+ // some records could have been invalidated upstream+ continue;+ }+if(index_rec.is_update) {index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);} else if(index_rec.indexed.ok()) {@@ -581,6 +578,10 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>if(afield.name == "id") {for(const auto& record: iter_batch) {+ if(!record.indexed.ok()) {+ // some records could have been invalidated upstream+ continue;+ }if(!record.is_update && record.indexed.ok()) {// for updates, the seq_id will already existseq_ids.append(record.seq_id);@@ -602,6 +603,11 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>int64_t max_score = INT64_MIN;for(const auto& record: iter_batch) {+ if(!record.indexed.ok()) {+ // some records could have been invalidated upstream+ continue;+ }+const auto& document = record.doc;const auto seq_id = record.seq_id;@@ -3615,6 +3621,7 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const farray_iter = document[field_name].erase(array_iter);array_ele_erased = true;}+return Option<uint32_t>(200);}@@ -3650,6 +3657,14 @@ Option<uint32_t> Index::coerce_int32_t(const DIRTY_VALUES& dirty_values, const f}}+ if(document.contains(field_name) && document[field_name].get<int64_t>() > INT32_MAX) {+ if(a_field.optional && (dirty_values == DIRTY_VALUES::DROP || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT)) {+ document.erase(field_name);+ } else {+ return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");+ }+ }+return Option<uint32_t>(200);}diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 10f8fffa..6ef6103d 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -406,6 +406,7 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {doc["title"] = "THIRD";add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `int_values` must be an array of string.", add_op.error());// singular field coerciondoc["single_int"] = 100;@@ -413,41 +414,64 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `single_int` must be a string.", add_op.error());- // uncoercable field, e.g. nested dict- doc["dict"] = nlohmann::json::object();- doc["dict"]["one"] = 1;- doc["dict"]["two"] = 2;+ // try with empty array+ doc["title"] = "FIFTH";+ doc["int_values"] = {"100"};+ doc["int_values_2"] = nlohmann::json::array();+ doc["single_int"] = "200";add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);ASSERT_FALSE(add_op.ok());- ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());+ ASSERT_EQ("Type of field `int_values_2` is invalid.", add_op.error());- // try with coerce_or_reject- add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);- ASSERT_FALSE(add_op.ok());- ASSERT_EQ("Type of field `dict` is invalid.", add_op.error());+ collectionManager.drop_collection("coll1");+}- // try with drop- doc["title"] = "FIFTH";- add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);+TEST_F(CollectionAllFieldsTest, IntegerAllValues) {+ Collection *coll1;++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, {}, "", 0, "int32").get();+ }++ nlohmann::json doc;+ doc["age"] = 100;+ doc["year"] = 2000;++ Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0");ASSERT_TRUE(add_op.ok());+ auto added_doc = add_op.get();- results = coll1->search("fifth", {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ("FIFTH", results["hits"][0]["document"]["title"].get<std::string>());- ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));++ // try with DROP+ doc["age"] = "SECOND";+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::DROP);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(2, results["hits"].size());++ // try with REJECT+ doc["age"] = "THIRD";+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `age` must be an int32.", add_op.error());++ // try with coerce_or_reject+ add_op = coll1->add(doc.dump(), CREATE, "", DIRTY_VALUES::COERCE_OR_REJECT);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `age` must be an int32.", add_op.error());// try with coerce_or_drop- doc["title"] = "SIXTH";+ doc["age"] = "FOURTH";add_op = coll1->add(doc.dump(), CREATE, "66", DIRTY_VALUES::COERCE_OR_DROP);ASSERT_TRUE(add_op.ok());- results = coll1->search("sixth", {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();- ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ("SIXTH", results["hits"][0]["document"]["title"].get<std::string>());- ASSERT_EQ(0, results["hits"][0]["document"].count("dict"));-collectionManager.drop_collection("coll1");}@@ -1060,6 +1084,54 @@ TEST_F(CollectionAllFieldsTest, BothFallbackAndDynamicFields) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionAllFieldsTest, WildcardFieldAndDictionaryField) {+ Collection *coll1;++ std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);+ ASSERT_TRUE(op.ok());+ coll1 = op.get();+ }++ nlohmann::json doc;+ doc["kinds"] = nlohmann::json::object();+ doc["kinds"]["CGXX"] = 13;+ doc["kinds"]["ZBXX"] = 24;++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionAllFieldsTest, DynamicFieldAndDictionaryField) {+ Collection *coll1;++ std::vector<field> fields = {field("k.*", field_types::STRING, false, true),+ field(".*", field_types::AUTO, false, true)};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);+ ASSERT_TRUE(op.ok());+ coll1 = op.get();+ }++ nlohmann::json doc;+ doc["kinds"] = nlohmann::json::object();+ doc["kinds"]["CGXX"] = 13;+ doc["kinds"]["ZBXX"] = 24;++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `kinds` must be a string.", add_op.error());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionAllFieldsTest, RegexpIntFieldWithFallbackStringType) {Collection *coll1;