Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Validate override configuration during creation

Jason Boscoeed5d61c1589
diff --git a/include/collection.h b/include/collection.h
index 2246a033..29984509 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -44,27 +44,97 @@ struct override_t {
override_t() {}
- override_t(const nlohmann::json & override) {
- id = override["id"].get<std::string>();
- rule.query = override["rule"]["query"].get<std::string>();
- rule.match = override["rule"]["match"].get<std::string>();
+ static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {
+ if(!override_json.is_object()) {
+ return Option<bool>(400, "Bad JSON.");
+ }
+
+ if(override_json.count("rule") == 0 || !override_json["rule"].is_object()) {
+ return Option<bool>(400, "Missing `rule` definition.");
+ }
+
+ if(override_json["rule"].count("query") == 0 || override_json["rule"].count("match") == 0) {
+ return Option<bool>(400, "The `rule` definition must contain a `query` or `match`.");
+ }
+
+ if(override_json.count("includes") == 0 && override_json.count("excludes") == 0) {
+ return Option<bool>(400, "Must contain either `includes` or `excludes`.");
+ }
+
+ if(override_json.count("includes") != 0) {
+ if(!override_json["includes"].is_array()) {
+ return Option<bool>(400, "The `includes` value must be an array.");
+ }
+
+ for(const auto & include_obj: override_json["includes"]) {
+ if(!include_obj.is_object()) {
+ return Option<bool>(400, "The `includes` value must be an array of objects.");
+ }
+
+ if(include_obj.count("id") == 0 || include_obj.count("position") == 0) {
+ return Option<bool>(400, "Inclusion definition must define both `id` and `position` keys.");
+ }
+
+ if(!include_obj["id"].is_string()) {
+ return Option<bool>(400, "Inclusion `id` must be a string.");
+ }
+
+ if(!include_obj["position"].is_number_integer()) {
+ return Option<bool>(400, "Inclusion `position` must be an integer.");
+ }
+ }
+ }
+
+ if(override_json.count("excludes") != 0) {
+ if(!override_json["excludes"].is_array()) {
+ return Option<bool>(400, "The `excludes` value must be an array.");
+ }
+
+ for(const auto & exclude_obj: override_json["excludes"]) {
+ if(!exclude_obj.is_object()) {
+ return Option<bool>(400, "The `excludes` value must be an array of objects.");
+ }
+
+ if(exclude_obj.count("id") == 0) {
+ return Option<bool>(400, "Exclusion definition must define an `id`.");
+ }
+
+ if(!exclude_obj["id"].is_string()) {
+ return Option<bool>(400, "Exclusion `id` must be a string.");
+ }
+ }
- if (override.count("includes") != 0) {
- for(const auto & include: override["includes"]) {
+ }
+
+ if(!id.empty()) {
+ override.id = id;
+ } else if(override_json.count("id") != 0) {
+ override.id = override_json["id"].get<std::string>();
+ } else {
+ return Option<bool>(400, "Override `id` not provided.");
+ }
+
+ override.rule.query = override_json["rule"]["query"].get<std::string>();
+ override.rule.match = override_json["rule"]["match"].get<std::string>();
+
+ if (override_json.count("includes") != 0) {
+ for(const auto & include: override_json["includes"]) {
add_hit_t add_hit;
add_hit.doc_id = include["id"].get<std::string>();
add_hit.position = include["position"].get<uint32_t>();
- add_hits.push_back(add_hit);
+ override.add_hits.push_back(add_hit);
}
}
- if (override.count("excludes") != 0) {
- for(const auto & exclude: override["excludes"]) {
+ if (override_json.count("excludes") != 0) {
+ for(const auto & exclude: override_json["excludes"]) {
drop_hit_t drop_hit;
drop_hit.doc_id = exclude["id"].get<std::string>();
- drop_hits.push_back(drop_hit);
+ override.drop_hits.push_back(drop_hit);
}
}
+
+ return Option<bool>(true);
}
nlohmann::json to_json() const {
diff --git a/src/auth_manager.cpp b/src/auth_manager.cpp
index 4f8bca94..4c8d1ba9 100644
--- a/src/auth_manager.cpp
+++ b/src/auth_manager.cpp
@@ -267,7 +267,7 @@ Option<bool> AuthManager::authenticate_parse_params(const std::string& scoped_ap
return Option<bool>(403, "Forbidden.");
}
- // parent key's expiry timestamp always takes precedence
+ // if parent key's expiry timestamp is smaller, it takes precedence
uint64_t expiry_ts = std::min(api_key.expires_at, embedded_params["expires_at"].get<uint64_t>());
if(std::time(0) > expiry_ts) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 2c31f6d8..453e046c 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -136,8 +136,13 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
for(const auto & collection_override_json: collection_override_jsons) {
nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);
- override_t override(collection_override);
- collection->add_override(override);
+ override_t override;
+ auto parse_op = override_t::parse(collection_override, "", override);
+ if(parse_op.ok()) {
+ collection->add_override(override);
+ } else {
+ LOG(ERROR) << "Skipping loading of override: " << parse_op.error();
+ }
}
// initialize synonyms
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 9b1f5fec..0d067f18 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1145,20 +1145,14 @@ bool put_override(http_req &req, http_res &res) {
res.set_400("Bad JSON.");
return false;
}
-
- // validate format of req_json
- if(!req_json.is_object() ||
- (req_json.count("rule") == 0) ||
- (req_json["rule"].count("query") == 0 || req_json["rule"].count("match") == 0) ||
- (req_json.count("includes") == 0 && req_json.count("excludes") == 0)
- ) {
- res.set_400("Bad JSON.");
+
+ override_t override;
+ Option<bool> parse_op = override_t::parse(req_json, override_id, override);
+ if(!parse_op.ok()) {
+ res.set(parse_op.code(), parse_op.error());
return false;
}
-
- req_json["id"] = override_id;
-
- override_t override(req_json);
+
Option<uint32_t> add_op = collection->add_override(override);
if(!add_op.ok()) {
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 74132378..b6e1df35 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -376,8 +376,12 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
override_json_exclude["excludes"][0] = nlohmann::json::object();
override_json_exclude["excludes"][0]["id"] = "2";
- override_t override1(override_json_include);
- override_t override2(override_json_exclude);
+ override_t override1;
+ override_t override2;
+
+ override_t::parse(override_json_include, "", override1);
+ override_t::parse(override_json_exclude, "", override2);
+
Option<uint32_t> ov1_op = coll_group->add_override(override1);
Option<uint32_t> ov2_op = coll_group->add_override(override2);
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 396dbddb..bc642d97 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -164,7 +164,8 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
override_json_include["includes"][1]["id"] = "3";
override_json_include["includes"][1]["position"] = 2;
- override_t override_include(override_json_include);
+ override_t override_include;
+ override_t::parse(override_json_include, "", override_include);
nlohmann::json override_json = {
{"id", "exclude-rule"},
@@ -182,19 +183,25 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
override_json["excludes"][1] = nlohmann::json::object();
override_json["excludes"][1]["id"] = "11";
- override_t override_exclude(override_json);
+ override_t override_exclude;
+ override_t::parse(override_json, "", override_exclude);
nlohmann::json override_json_deleted = {
{"id", "deleted-rule"},
{
"rule", {
- {"query", "of"},
- {"match", override_t::MATCH_EXACT}
- }
+ {"query", "of"},
+ {"match", override_t::MATCH_EXACT}
+ }
}
};
- override_t override_deleted(override_json_deleted);
+ override_json_deleted["excludes"] = nlohmann::json::array();
+ override_json_deleted["excludes"][0] = nlohmann::json::object();
+ override_json_deleted["excludes"][0]["id"] = "11";
+
+ override_t override_deleted;
+ override_t::parse(override_json_deleted, "", override_deleted);
collection1->add_override(override_include);
collection1->add_override(override_exclude);
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index cb31e14e..40c45c39 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -71,7 +71,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {
override_json["excludes"][1] = nlohmann::json::object();
override_json["excludes"][1]["id"] = "11";
- override_t override(override_json);
+ override_t override;
+ override_t::parse(override_json, "", override);
+
coll_mul_fields->add_override(override);
std::vector<std::string> facets = {"cast"};
@@ -107,7 +109,8 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {
override_json_include["includes"][1]["id"] = "3";
override_json_include["includes"][1]["position"] = 2;
- override_t override_include(override_json_include);
+ override_t override_include;
+ override_t::parse(override_json_include, "", override_include);
coll_mul_fields->add_override(override_include);
@@ -150,7 +153,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {
override_contains_inc["includes"][1]["id"] = "1";
override_contains_inc["includes"][1]["position"] = 7; // purposely setting it way out
- override_t override_inc_contains(override_contains_inc);
+ override_t override_inc_contains;
+ override_t::parse(override_contains_inc, "", override_inc_contains);
+
coll_mul_fields->add_override(override_inc_contains);
res_op = coll_mul_fields->search("will smith", {"title"}, "", {}, {}, 0, 10);
@@ -168,6 +173,96 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeExactQueryMatch) {
coll_mul_fields->remove_override("include-rule");
}
+TEST_F(CollectionOverrideTest, OverrideJSONValidation) {
+ nlohmann::json exclude_json = {
+ {"id", "exclude-rule"},
+ {
+ "rule", {
+ {"query", "of"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ }
+ };
+
+ exclude_json["excludes"] = nlohmann::json::array();
+ exclude_json["excludes"][0] = nlohmann::json::object();
+ exclude_json["excludes"][0]["id"] = 11;
+
+ override_t override1;
+ auto parse_op = override_t::parse(exclude_json, "", override1);
+
+ ASSERT_FALSE(parse_op.ok());
+ ASSERT_STREQ("Exclusion `id` must be a string.", parse_op.error().c_str());
+
+ nlohmann::json include_json = {
+ {"id", "include-rule"},
+ {
+ "rule", {
+ {"query", "of"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ }
+ };
+
+ include_json["includes"] = nlohmann::json::array();
+ include_json["includes"][0] = nlohmann::json::object();
+ include_json["includes"][0]["id"] = "11";
+
+ override_t override2;
+ parse_op = override_t::parse(include_json, "", override2);
+
+ ASSERT_FALSE(parse_op.ok());
+ ASSERT_STREQ("Inclusion definition must define both `id` and `position` keys.", parse_op.error().c_str());
+
+ include_json["includes"][0]["position"] = "1";
+
+ parse_op = override_t::parse(include_json, "", override2);
+ ASSERT_FALSE(parse_op.ok());
+ ASSERT_STREQ("Inclusion `position` must be an integer.", parse_op.error().c_str());
+
+ include_json["includes"][0]["position"] = 1;
+ parse_op = override_t::parse(include_json, "", override2);
+ ASSERT_TRUE(parse_op.ok());
+
+ nlohmann::json include_json2 = {
+ {"id", "include-rule"},
+ {
+ "rule", {
+ {"query", "of"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ }
+ };
+
+ parse_op = override_t::parse(include_json2, "", override2);
+ ASSERT_FALSE(parse_op.ok());
+ ASSERT_STREQ("Must contain either `includes` or `excludes`.", parse_op.error().c_str());
+
+ include_json2["includes"] = nlohmann::json::array();
+ include_json2["includes"][0] = 100;
+
+ parse_op = override_t::parse(include_json2, "", override2);
+ ASSERT_FALSE(parse_op.ok());
+ ASSERT_STREQ("The `includes` value must be an array of objects.", parse_op.error().c_str());
+
+ nlohmann::json exclude_json2 = {
+ {"id", "exclude-rule"},
+ {
+ "rule", {
+ {"query", "of"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ }
+ };
+
+ exclude_json2["excludes"] = nlohmann::json::array();
+ exclude_json2["excludes"][0] = "100";
+
+ parse_op = override_t::parse(exclude_json2, "", override2);
+ ASSERT_FALSE(parse_op.ok());
+ ASSERT_STREQ("The `excludes` value must be an array of objects.", parse_op.error().c_str());
+}
+
TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {
// Check facet field highlight for overridden results
nlohmann::json override_json_include = {
@@ -189,7 +284,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {
override_json_include["includes"][1]["id"] = "2";
override_json_include["includes"][1]["position"] = 2;
- override_t override_include(override_json_include);
+ override_t override_include;
+ override_t::parse(override_json_include, "", override_include);
+
coll_mul_fields->add_override(override_include);
auto results = coll_mul_fields->search("not-found", {"title"}, "", {"starring"}, {}, 0, 10, 1, FREQUENCY,
@@ -217,7 +314,9 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {
override_json_exclude["excludes"][0] = nlohmann::json::object();
override_json_exclude["excludes"][0]["id"] = "10";
- override_t override(override_json_exclude);
+ override_t override;
+ override_t::parse(override_json_exclude, "", override);
+
coll_mul_fields->add_override(override);
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, 0, 10, 1, FREQUENCY,
@@ -335,7 +434,9 @@ TEST_F(CollectionOverrideTest, IncludeExcludeHitsQuery) {
override_json_include["includes"][1]["id"] = "8";
override_json_include["includes"][1]["position"] = 1;
- override_t override_include(override_json_include);
+ override_t override_include;
+ override_t::parse(override_json_include, "", override_include);
+
coll_mul_fields->add_override(override_include);
results = coll_mul_fields->search("the", {"title"}, "", {"starring"}, {}, 0, 50, 1, FREQUENCY,