Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle fields not matching type declared in the schema during indexing.
Kishore Nallanfab27d9f5cf7
diff --git a/TODO.md b/TODO.mdindex 56ccc9ff..c5483538 100644--- a/TODO.md+++ b/TODO.md@@ -27,7 +27,7 @@- ~~Assumption that all tokens match for scoring is no longer true~~- ~~Filters~~- ~~Facets~~-- Schema validation during insertion (missing fields + type errors)+- ~~Schema validation during insertion (missing fields + type errors)~~- Proper score field for ranking tokens- Prevent string copy during indexing- clean special chars before indexingdiff --git a/src/collection.cpp b/src/collection.cppindex 75415be8..023b8c8f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -95,21 +95,60 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinart_tree *t = search_index.at(field_name);if(field_pair.second.type == field_types::STRING) {+ if(!document[field_name].is_string()) {+ return Option<>(400, "Search field `" + field_name + "` must be a STRING.");+ }const std::string & text = document[field_name];index_string_field(text, points, t, seq_id, false);} else if(field_pair.second.type == field_types::INT32) {+ if(!document[field_name].is_number()) {+ return Option<>(400, "Search field `" + field_name + "` must be an INT32.");+ }++ if(document[field_name].get<int64_t>() > INT32_MAX) {+ return Option<>(400, "Search field `" + field_name + "` exceeds maximum value of INT32.");+ }+uint32_t value = document[field_name];index_int32_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::INT64) {+ if(!document[field_name].is_number()) {+ return Option<>(400, "Search field `" + field_name + "` must be an INT64.");+ }+uint64_t value = document[field_name];index_int64_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {+ if(!document[field_name].is_array()) {+ return Option<>(400, "Search field `" + field_name + "` must be a STRING_ARRAY.");+ }++ if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {+ return Option<>(400, "Search field `" + field_name + "` must be a STRING_ARRAY.");+ }+std::vector<std::string> strings = document[field_name];index_string_array_field(strings, points, t, seq_id, false);} else if(field_pair.second.type == field_types::INT32_ARRAY) {+ if(!document[field_name].is_array()) {+ return Option<>(400, "Search field `" + field_name + "` must be an INT32_ARRAY.");+ }++ if(document[field_name].size() > 0 && !document[field_name][0].is_number()) {+ return Option<>(400, "Search field `" + field_name + "` must be an INT32_ARRAY.");+ }+std::vector<int32_t> values = document[field_name];index_int32_array_field(values, points, t, seq_id);} else if(field_pair.second.type == field_types::INT64_ARRAY) {+ if(!document[field_name].is_array()) {+ return Option<>(400, "Search field `" + field_name + "` must be an INT64_ARRAY.");+ }++ if(document[field_name].size() > 0 && !document[field_name][0].is_number()) {+ return Option<>(400, "Search field `" + field_name + "` must be an INT64_ARRAY.");+ }+std::vector<int64_t> values = document[field_name];index_int64_array_field(values, points, t, seq_id);}@@ -125,9 +164,20 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinart_tree *t = facet_index.at(field_name);if(field_pair.second.type == field_types::STRING) {+ if(!document[field_name].is_string()) {+ return Option<>(400, "Facet field `" + field_name + "` must be a STRING.");+ }const std::string & text = document[field_name];index_string_field(text, points, t, seq_id, true);} else if(field_pair.second.type == field_types::STRING_ARRAY) {+ if(!document[field_name].is_array()) {+ return Option<>(400, "Facet field `" + field_name + "` must be a STRING_ARRAY.");+ }++ if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {+ return Option<>(400, "Facet field `" + field_name + "` must be a STRING_ARRAY.");+ }+std::vector<std::string> strings = document[field_name];index_string_array_field(strings, points, t, seq_id, true);}@@ -139,6 +189,10 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin"but is not found in the document.");}+ if(!document[rank_field].is_number()) {+ return Option<>(400, "Rank field `" + rank_field + "` must be an integer.");+ }+spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = rank_index.at(rank_field);doc_to_score->emplace(seq_id, document[rank_fields[0]].get<int64_t>());}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 4d6659da..4564ee1a 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -817,13 +817,27 @@ TEST_F(CollectionTest, IndexingWithBadData) {ASSERT_STREQ("Field `average` has been declared as a rank field in the schema, but is not found in the document.",rank_fields_missing_op1.error().c_str());- // handle type errors-- const char *doc_str2 = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22}";- const Option<std::string> & rank_fields_missing_op2 = sample_collection->add(doc_str2);- ASSERT_FALSE(rank_fields_missing_op2.ok());- ASSERT_STREQ("Field `average` has been declared as a rank field in the schema, but is not found in the document.",- rank_fields_missing_op2.error().c_str());+ // Handle type errors++ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22}";+ const Option<std::string> & bad_facet_field_op = sample_collection->add(doc_str);+ ASSERT_FALSE(bad_facet_field_op.ok());+ ASSERT_STREQ("Facet field `tags` must be a STRING_ARRAY.",+ bad_facet_field_op.error().c_str());++ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";+ const Option<std::string> & empty_facet_field_op = sample_collection->add(doc_str);+ ASSERT_TRUE(empty_facet_field_op.ok());++ doc_str = "{\"name\": \"foo\", \"age\": \"34\", \"tags\": [], \"average\": 34 }";+ const Option<std::string> & bad_search_field_op = sample_collection->add(doc_str);+ ASSERT_FALSE(bad_search_field_op.ok());+ ASSERT_STREQ("Search field `age` must be an INT32.", bad_search_field_op.error().c_str());++ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";+ const Option<std::string> & bad_rank_field_op = sample_collection->add(doc_str);+ ASSERT_FALSE(bad_rank_field_op.ok());+ ASSERT_STREQ("Rank field `average` must be an integer.", bad_rank_field_op.error().c_str());collectionManager.drop_collection("sample_collection");}\ No newline at end of file