Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle fields not matching type declared in the schema during indexing.

Kishore Nallanfab27d9f5cf7
diff --git a/TODO.md b/TODO.md
index 56ccc9ff..c5483538 100644
--- a/TODO.md
+++ b/TODO.md
@@ -27,7 +27,7 @@
- ~~Assumption that all tokens match for scoring is no longer true~~
- ~~Filters~~
- ~~Facets~~
-- Schema validation during insertion (missing fields + type errors)
+- ~~Schema validation during insertion (missing fields + type errors)~~
- Proper score field for ranking tokens
- Prevent string copy during indexing
- clean special chars before indexing
diff --git a/src/collection.cpp b/src/collection.cpp
index 75415be8..023b8c8f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -95,21 +95,60 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
art_tree *t = search_index.at(field_name);
if(field_pair.second.type == field_types::STRING) {
+ if(!document[field_name].is_string()) {
+ return Option<>(400, "Search field `" + field_name + "` must be a STRING.");
+ }
const std::string & text = document[field_name];
index_string_field(text, points, t, seq_id, false);
} else if(field_pair.second.type == field_types::INT32) {
+ if(!document[field_name].is_number()) {
+ return Option<>(400, "Search field `" + field_name + "` must be an INT32.");
+ }
+
+ if(document[field_name].get<int64_t>() > INT32_MAX) {
+ return Option<>(400, "Search field `" + field_name + "` exceeds maximum value of INT32.");
+ }
+
uint32_t value = document[field_name];
index_int32_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::INT64) {
+ if(!document[field_name].is_number()) {
+ return Option<>(400, "Search field `" + field_name + "` must be an INT64.");
+ }
+
uint64_t value = document[field_name];
index_int64_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Search field `" + field_name + "` must be a STRING_ARRAY.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
+ return Option<>(400, "Search field `" + field_name + "` must be a STRING_ARRAY.");
+ }
+
std::vector<std::string> strings = document[field_name];
index_string_array_field(strings, points, t, seq_id, false);
} else if(field_pair.second.type == field_types::INT32_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Search field `" + field_name + "` must be an INT32_ARRAY.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_number()) {
+ return Option<>(400, "Search field `" + field_name + "` must be an INT32_ARRAY.");
+ }
+
std::vector<int32_t> values = document[field_name];
index_int32_array_field(values, points, t, seq_id);
} else if(field_pair.second.type == field_types::INT64_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Search field `" + field_name + "` must be an INT64_ARRAY.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_number()) {
+ return Option<>(400, "Search field `" + field_name + "` must be an INT64_ARRAY.");
+ }
+
std::vector<int64_t> values = document[field_name];
index_int64_array_field(values, points, t, seq_id);
}
@@ -125,9 +164,20 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
art_tree *t = facet_index.at(field_name);
if(field_pair.second.type == field_types::STRING) {
+ if(!document[field_name].is_string()) {
+ return Option<>(400, "Facet field `" + field_name + "` must be a STRING.");
+ }
const std::string & text = document[field_name];
index_string_field(text, points, t, seq_id, true);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Facet field `" + field_name + "` must be a STRING_ARRAY.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
+ return Option<>(400, "Facet field `" + field_name + "` must be a STRING_ARRAY.");
+ }
+
std::vector<std::string> strings = document[field_name];
index_string_array_field(strings, points, t, seq_id, true);
}
@@ -139,6 +189,10 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
"but is not found in the document.");
}
+ if(!document[rank_field].is_number()) {
+ return Option<>(400, "Rank field `" + rank_field + "` must be an integer.");
+ }
+
spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = rank_index.at(rank_field);
doc_to_score->emplace(seq_id, document[rank_fields[0]].get<int64_t>());
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 4d6659da..4564ee1a 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -817,13 +817,27 @@ TEST_F(CollectionTest, IndexingWithBadData) {
ASSERT_STREQ("Field `average` has been declared as a rank field in the schema, but is not found in the document.",
rank_fields_missing_op1.error().c_str());
- // handle type errors
-
- const char *doc_str2 = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22}";
- const Option<std::string> & rank_fields_missing_op2 = sample_collection->add(doc_str2);
- ASSERT_FALSE(rank_fields_missing_op2.ok());
- ASSERT_STREQ("Field `average` has been declared as a rank field in the schema, but is not found in the document.",
- rank_fields_missing_op2.error().c_str());
+ // Handle type errors
+
+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22}";
+ const Option<std::string> & bad_facet_field_op = sample_collection->add(doc_str);
+ ASSERT_FALSE(bad_facet_field_op.ok());
+ ASSERT_STREQ("Facet field `tags` must be a STRING_ARRAY.",
+ bad_facet_field_op.error().c_str());
+
+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";
+ const Option<std::string> & empty_facet_field_op = sample_collection->add(doc_str);
+ ASSERT_TRUE(empty_facet_field_op.ok());
+
+ doc_str = "{\"name\": \"foo\", \"age\": \"34\", \"tags\": [], \"average\": 34 }";
+ const Option<std::string> & bad_search_field_op = sample_collection->add(doc_str);
+ ASSERT_FALSE(bad_search_field_op.ok());
+ ASSERT_STREQ("Search field `age` must be an INT32.", bad_search_field_op.error().c_str());
+
+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";
+ const Option<std::string> & bad_rank_field_op = sample_collection->add(doc_str);
+ ASSERT_FALSE(bad_rank_field_op.ok());
+ ASSERT_STREQ("Rank field `average` must be an integer.", bad_rank_field_op.error().c_str());
collectionManager.drop_collection("sample_collection");
}
\ No newline at end of file