Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Validate override configuration during creation
Jason Boscoeed5d61c1589
diff --git a/include/collection.h b/include/collection.hindex 2246a033..29984509 100644--- a/include/collection.h+++ b/include/collection.h@@ -44,27 +44,97 @@ struct override_t {override_t() {}- override_t(const nlohmann::json & override) {- id = override["id"].get<std::string>();- rule.query = override["rule"]["query"].get<std::string>();- rule.match = override["rule"]["match"].get<std::string>();+ static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {+ if(!override_json.is_object()) {+ return Option<bool>(400, "Bad JSON.");+ }++ if(override_json.count("rule") == 0 || !override_json["rule"].is_object()) {+ return Option<bool>(400, "Missing `rule` definition.");+ }++ if(override_json["rule"].count("query") == 0 || override_json["rule"].count("match") == 0) {+ return Option<bool>(400, "The `rule` definition must contain a `query` or `match`.");+ }++ if(override_json.count("includes") == 0 && override_json.count("excludes") == 0) {+ return Option<bool>(400, "Must contain either `includes` or `excludes`.");+ }++ if(override_json.count("includes") != 0) {+ if(!override_json["includes"].is_array()) {+ return Option<bool>(400, "The `includes` value must be an array.");+ }++ for(const auto & include_obj: override_json["includes"]) {+ if(!include_obj.is_object()) {+ return Option<bool>(400, "The `includes` value must be an array of objects.");+ }++ if(include_obj.count("id") == 0 || include_obj.count("position") == 0) {+ return Option<bool>(400, "Inclusion definition must define both `id` and `position` keys.");+ }++ if(!include_obj["id"].is_string()) {+ return Option<bool>(400, "Inclusion `id` must be a string.");+ }++ if(!include_obj["position"].is_number_integer()) {+ return Option<bool>(400, "Inclusion `position` must be an integer.");+ }+ }+ }++ if(override_json.count("excludes") != 0) {+ if(!override_json["excludes"].is_array()) {+ return Option<bool>(400, "The `excludes` value must be an array.");+ }++ for(const auto & exclude_obj: override_json["excludes"]) {+ if(!exclude_obj.is_object()) {+ return Option<bool>(400, "The `excludes` value must be an array of objects.");+ }++ if(exclude_obj.count("id") == 0) {+ return Option<bool>(400, "Exclusion definition must define an `id`.");+ }++ if(!exclude_obj["id"].is_string()) {+ return Option<bool>(400, "Exclusion `id` must be a string.");+ }+ }- if (override.count("includes") != 0) {- for(const auto & include: override["includes"]) {+ }++ if(!id.empty()) {+ override.id = id;+ } else if(override_json.count("id") != 0) {+ override.id = override_json["id"].get<std::string>();+ } else {+ return Option<bool>(400, "Override `id` not provided.");+ }++ override.rule.query = override_json["rule"]["query"].get<std::string>();+ override.rule.match = override_json["rule"]["match"].get<std::string>();++ if (override_json.count("includes") != 0) {+ for(const auto & include: override_json["includes"]) {add_hit_t add_hit;add_hit.doc_id = include["id"].get<std::string>();add_hit.position = include["position"].get<uint32_t>();- add_hits.push_back(add_hit);+ override.add_hits.push_back(add_hit);}}- if (override.count("excludes") != 0) {- for(const auto & exclude: override["excludes"]) {+ if (override_json.count("excludes") != 0) {+ for(const auto & exclude: override_json["excludes"]) {drop_hit_t drop_hit;drop_hit.doc_id = exclude["id"].get<std::string>();- drop_hits.push_back(drop_hit);+ override.drop_hits.push_back(drop_hit);}}++ return Option<bool>(true);}nlohmann::json to_json() const {diff --git a/src/auth_manager.cpp b/src/auth_manager.cppindex 4f8bca94..4c8d1ba9 100644--- a/src/auth_manager.cpp+++ b/src/auth_manager.cpp@@ -267,7 +267,7 @@ Option<bool> AuthManager::authenticate_parse_params(const std::string& scoped_apreturn Option<bool>(403, "Forbidden.");}- // parent key's expiry timestamp always takes precedence+ // if parent key's expiry timestamp is smaller, it takes precedenceuint64_t expiry_ts = std::min(api_key.expires_at, embedded_params["expires_at"].get<uint64_t>());if(std::time(0) > expiry_ts) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 2c31f6d8..453e046c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -136,8 +136,13 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {for(const auto & collection_override_json: collection_override_jsons) {nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);- override_t override(collection_override);- collection->add_override(override);+ override_t override;+ auto parse_op = override_t::parse(collection_override, "", override);+ if(parse_op.ok()) {+ collection->add_override(override);+ } else {+ LOG(ERROR) << "Skipping loading of override: " << parse_op.error();+ }}// initialize synonymsdiff --git a/src/core_api.cpp b/src/core_api.cppindex 9b1f5fec..0d067f18 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1145,20 +1145,14 @@ bool put_override(http_req &req, http_res &res) {res.set_400("Bad JSON.");return false;}-- // validate format of req_json- if(!req_json.is_object() ||- (req_json.count("rule") == 0) ||- (req_json["rule"].count("query") == 0 || req_json["rule"].count("match") == 0) ||- (req_json.count("includes") == 0 && req_json.count("excludes") == 0)- ) {- res.set_400("Bad JSON.");++ override_t override;+ Option<bool> parse_op = override_t::parse(req_json, override_id, override);+ if(!parse_op.ok()) {+ res.set(parse_op.code(), parse_op.error());return false;}-- req_json["id"] = override_id;-- override_t override(req_json);+Option<uint32_t> add_op = collection->add_override(override);if(!add_op.ok()) {diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 74132378..b6e1df35 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -376,8 +376,12 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {override_json_exclude["excludes"][0] = nlohmann::json::object();override_json_exclude["excludes"][0]["id"] = "2";- override_t override1(override_json_include);- override_t override2(override_json_exclude);+ override_t override1;+ override_t override2;++ override_t::parse(override_json_include, "", override1);+ override_t::parse(override_json_exclude, "", override2);+Option<uint32_t> ov1_op = coll_group->add_override(override1);Option<uint32_t> ov2_op = coll_group->add_override(override2);diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 396dbddb..bc642d97 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -164,7 +164,8 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {override_json_include["includes"][1]["id"] = "3";override_json_include["includes"][1]["position"] = 2;- override_t override_include(override_json_include);+ override_t override_include;+ override_t::parse(override_json_include, "", override_include);nlohmann::json override_json = {{"id", "exclude-rule"},@@ -182,19 +183,25 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {override_json["excludes"][1] = nlohmann::json::object();override_json["excludes"][1]["id"] = "11";- override_t override_exclude(override_json);+ override_t override_exclude;+ override_t::parse(override_json, "", override_exclude);nlohmann::json override_json_deleted = {{"id", "deleted-rule"},{"rule", {- {"query", "of"},- {"match", override_t::MATCH_EXACT}- }+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }}};- override_t override_deleted(override_json_deleted);+ override_json_deleted["excludes"] = nlohmann::json::array();+ override_json_deleted["excludes"][0] = nlohmann::json::object();+ override_json_deleted["excludes"][0]["id"] = "11";++ override_t override_deleted;+ override_t::parse(override_json_deleted, "", override_deleted);collection1->add_override(override_include);collection1->add_override(override_exclude);diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex cb31e14e..40c45c39 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -71,7 +71,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {override_json["excludes"][1] = nlohmann::json::object();override_json["excludes"][1]["id"] = "11";- override_t override(override_json);+ override_t override;+ override_t::parse(override_json, "", override);+coll_mul_fields->add_override(override);std::vector<std::string> facets = {"cast"};@@ -107,7 +109,8 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {override_json_include["includes"][1]["id"] = "3";override_json_include["includes"][1]["position"] = 2;- override_t override_include(override_json_include);+ override_t override_include;+ override_t::parse(override_json_include, "", override_include);coll_mul_fields->add_override(override_include);@@ -150,7 +153,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {override_contains_inc["includes"][1]["id"] = "1";override_contains_inc["includes"][1]["position"] = 7; // purposely setting it way out- override_t override_inc_contains(override_contains_inc);+ override_t override_inc_contains;+ override_t::parse(override_contains_inc, "", override_inc_contains);+coll_mul_fields->add_override(override_inc_contains);res_op = coll_mul_fields->search("will smith", {"title"}, "", {}, {}, 0, 10);@@ -168,6 +173,96 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {coll_mul_fields->remove_override("include-rule");}+TEST_F(CollectionOverrideTest, OverrideJSONValidation) {+ nlohmann::json exclude_json = {+ {"id", "exclude-rule"},+ {+ "rule", {+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };++ exclude_json["excludes"] = nlohmann::json::array();+ exclude_json["excludes"][0] = nlohmann::json::object();+ exclude_json["excludes"][0]["id"] = 11;++ override_t override1;+ auto parse_op = override_t::parse(exclude_json, "", override1);++ ASSERT_FALSE(parse_op.ok());+ ASSERT_STREQ("Exclusion `id` must be a string.", parse_op.error().c_str());++ nlohmann::json include_json = {+ {"id", "include-rule"},+ {+ "rule", {+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };++ include_json["includes"] = nlohmann::json::array();+ include_json["includes"][0] = nlohmann::json::object();+ include_json["includes"][0]["id"] = "11";++ override_t override2;+ parse_op = override_t::parse(include_json, "", override2);++ ASSERT_FALSE(parse_op.ok());+ ASSERT_STREQ("Inclusion definition must define both `id` and `position` keys.", parse_op.error().c_str());++ include_json["includes"][0]["position"] = "1";++ parse_op = override_t::parse(include_json, "", override2);+ ASSERT_FALSE(parse_op.ok());+ ASSERT_STREQ("Inclusion `position` must be an integer.", parse_op.error().c_str());++ include_json["includes"][0]["position"] = 1;+ parse_op = override_t::parse(include_json, "", override2);+ ASSERT_TRUE(parse_op.ok());++ nlohmann::json include_json2 = {+ {"id", "include-rule"},+ {+ "rule", {+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };++ parse_op = override_t::parse(include_json2, "", override2);+ ASSERT_FALSE(parse_op.ok());+ ASSERT_STREQ("Must contain either `includes` or `excludes`.", parse_op.error().c_str());++ include_json2["includes"] = nlohmann::json::array();+ include_json2["includes"][0] = 100;++ parse_op = override_t::parse(include_json2, "", override2);+ ASSERT_FALSE(parse_op.ok());+ ASSERT_STREQ("The `includes` value must be an array of objects.", parse_op.error().c_str());++ nlohmann::json exclude_json2 = {+ {"id", "exclude-rule"},+ {+ "rule", {+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };++ exclude_json2["excludes"] = nlohmann::json::array();+ exclude_json2["excludes"][0] = "100";++ parse_op = override_t::parse(exclude_json2, "", override2);+ ASSERT_FALSE(parse_op.ok());+ ASSERT_STREQ("The `excludes` value must be an array of objects.", parse_op.error().c_str());+}+TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {// Check facet field highlight for overridden resultsnlohmann::json override_json_include = {@@ -189,7 +284,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {override_json_include["includes"][1]["id"] = "2";override_json_include["includes"][1]["position"] = 2;- override_t override_include(override_json_include);+ override_t override_include;+ override_t::parse(override_json_include, "", override_include);+coll_mul_fields->add_override(override_include);auto results = coll_mul_fields->search("not-found", {"title"}, "", {"starring"}, {}, 0, 10, 1, FREQUENCY,@@ -217,7 +314,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {override_json_exclude["excludes"][0] = nlohmann::json::object();override_json_exclude["excludes"][0]["id"] = "10";- override_t override(override_json_exclude);+ override_t override;+ override_t::parse(override_json_exclude, "", override);+coll_mul_fields->add_override(override);results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, 0, 10, 1, FREQUENCY,@@ -335,7 +434,9 @@ TEST_F(CollectionOverrideTest, IncludeExcludeHitsQuery) {override_json_include["includes"][1]["id"] = "8";override_json_include["includes"][1]["position"] = 1;- override_t override_include(override_json_include);+ override_t override_include;+ override_t::parse(override_json_include, "", override_include);+coll_mul_fields->add_override(override_include);results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, 0, 50, 1, FREQUENCY,