Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor type detection logic for auto/string* fields.
Kishore Nallana365bf96776e
diff --git a/src/collection.cpp b/src/collection.cppindex 35d03a9f..35efdddf 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2254,42 +2254,47 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constcontinue;}- if(!found_dynamic_field ||- new_field.type == field_types::AUTO || field_types::is_string_or_array(new_field.type)) {+ if(!new_field.index) {+ kv++;+ continue;+ }- // detect the actual type- if(!found_dynamic_field && fallback_field_type != field_types::AUTO &&- !field_types::is_string_or_array(fallback_field_type)) {- new_field.type = fallback_field_type;- } else {- parseable = field::get_type(kv.value(), field_type);- if(!parseable) {- if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {- return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");- } else {- // DROP or COERCE_OR_DROP- kv = document.erase(kv);- continue;- }+ // Type detection scenarios:+ // a) Not a dynamic field + fallback type is explicit: use fallback type+ // b) Dynamic field + type is explicit: use explicit type+ // c) Not a dynamic field + fallback type is auto: detect and assign type+ // d) Dynamic field + type is auto: detect and assign type+ // e) Not a dynamic field + fallback type is string*: map to string/string[]+ // f) Dynamic field + type is string*: map to string/string[]++ const std::string& test_field_type = found_dynamic_field ? new_field.type : fallback_field_type;++ if(test_field_type == field_types::AUTO || field_types::is_string_or_array(test_field_type)) {+ parseable = field::get_type(kv.value(), field_type);+ if(!parseable) {+ if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {+ return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");+ } else {+ // DROP or COERCE_OR_DROP+ kv = document.erase(kv);+ continue;}+ }+ if(test_field_type == field_types::AUTO) {new_field.type = field_type;-- if (field_types::is_string_or_array(fallback_field_type)) {- // Supporting single/array field detection only for strings,- // as it does not seem to be too useful for other field types.- if (new_field.is_array()) {- new_field.type = field_types::STRING_ARRAY;- } else {- new_field.type = field_types::STRING;- }+ } else {+ if (kv.value().is_array()) {+ new_field.type = field_types::STRING_ARRAY;+ } else {+ new_field.type = field_types::STRING;}}+}- if(!new_field.index) {- kv++;- continue;+ else {+ new_field.type = test_field_type;}search_schema.emplace(new_field.name, new_field);diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex c4374760..7429ab8e 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -370,6 +370,14 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {ASSERT_TRUE(add_op.ok());auto added_doc = add_op.get();+ auto schema = coll1->get_fields();++ ASSERT_EQ("int_values", schema[0].name);+ ASSERT_EQ(field_types::STRING_ARRAY, schema[0].type);++ ASSERT_EQ("title", schema[1].name);+ ASSERT_EQ(field_types::STRING, schema[1].type);+ASSERT_EQ("1", added_doc["int_values"][0].get<std::string>());ASSERT_EQ("2", added_doc["int_values"][1].get<std::string>());@@ -464,6 +472,12 @@ TEST_F(CollectionAllFieldsTest, SearchStringifiedField) {Option<nlohmann::json> add_op = coll1->add(doc.dump(), CREATE, "0");ASSERT_TRUE(add_op.ok());+ // department field's type must be "solidified" to an actual type++ auto schema = coll1->get_fields();+ ASSERT_EQ("department", schema[4].name);+ ASSERT_EQ(field_types::STRING, schema[4].type);+auto results_op = coll1->search("stark", {"company_name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false});ASSERT_TRUE(results_op.ok());ASSERT_EQ(1, results_op.get()["hits"].size());@@ -827,7 +841,8 @@ TEST_F(CollectionAllFieldsTest, RegexpExplicitFieldTypeCoercion) {std::vector<field> fields = {field("title", field_types::STRING, true),field("i.*", field_types::INT32, false, true),field("s.*", field_types::STRING, false, true),- field("a.*", field_types::STRING_ARRAY, false, true),};+ field("a.*", field_types::STRING_ARRAY, false, true),+ field("num.*", "string*", false, true),};coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {@@ -839,6 +854,7 @@ TEST_F(CollectionAllFieldsTest, RegexpExplicitFieldTypeCoercion) {doc["i_age"] = "28";doc["s_name"] = nullptr;doc["a_name"] = {};+ doc["num_employees"] = 28;// should coerce while retaining expected type@@ -847,14 +863,18 @@ TEST_F(CollectionAllFieldsTest, RegexpExplicitFieldTypeCoercion) {auto schema = coll1->get_fields();- ASSERT_EQ("a_name", schema[4].name);- ASSERT_EQ(field_types::STRING_ARRAY, schema[4].type);+ ASSERT_EQ("a_name", schema[5].name);+ ASSERT_EQ(field_types::STRING_ARRAY, schema[5].type);- ASSERT_EQ("i_age", schema[5].name);- ASSERT_EQ(field_types::INT32, schema[5].type);+ ASSERT_EQ("i_age", schema[6].name);+ ASSERT_EQ(field_types::INT32, schema[6].type);++ // num_employees field's type must be "solidified" to an actual type+ ASSERT_EQ("num_employees", schema[7].name);+ ASSERT_EQ(field_types::STRING, schema[7].type);- ASSERT_EQ("s_name", schema[6].name);- ASSERT_EQ(field_types::STRING, schema[6].type);+ ASSERT_EQ("s_name", schema[8].name);+ ASSERT_EQ(field_types::STRING, schema[8].type);auto results = coll1->search("rand", {"title"}, "i_age: 28", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(1, results["hits"].size());@@ -957,6 +977,7 @@ TEST_F(CollectionAllFieldsTest, RegexpIntFieldWithFallbackStringType) {std::vector<field> fields = {field("title", field_types::STRING, true),field("n.*", field_types::INT32, false, true),+ field("s.*", "string*", false, true),field(".*", field_types::STRING, false, true)};coll1 = collectionManager.get_collection("coll1").get();@@ -969,6 +990,7 @@ TEST_F(CollectionAllFieldsTest, RegexpIntFieldWithFallbackStringType) {nlohmann::json doc;doc["title"] = "Amazon Inc.";doc["n_age"] = 32;+ doc["s_tags"] = {"shopping"};doc["rand_str"] = "fizzbuzz";auto add_op = coll1->add(doc.dump(), CREATE);@@ -976,11 +998,15 @@ TEST_F(CollectionAllFieldsTest, RegexpIntFieldWithFallbackStringType) {// n_age should be of type int32auto schema = coll1->get_fields();- ASSERT_EQ("n_age", schema[3].name);- ASSERT_EQ(field_types::INT32, schema[3].type);- ASSERT_EQ("rand_str", schema[4].name);- ASSERT_EQ(field_types::STRING, schema[4].type);+ ASSERT_EQ("n_age", schema[4].name);+ ASSERT_EQ(field_types::INT32, schema[4].type);++ ASSERT_EQ("rand_str", schema[5].name);+ ASSERT_EQ(field_types::STRING, schema[5].type);++ ASSERT_EQ("s_tags", schema[6].name);+ ASSERT_EQ(field_types::STRING_ARRAY, schema[6].type);collectionManager.drop_collection("coll1");}