Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Persist override configuration.
Kishore Nallan237196204c04
diff --git a/include/collection.h b/include/collection.hindex f2164a11..853070ab 100644--- a/include/collection.h+++ b/include/collection.h@@ -65,6 +65,31 @@ struct override_t {}}}++ std::string to_json_str() const {+ nlohmann::json override;+ override["id"] = id;+ override["rule"]["query"] = rule.query;+ override["rule"]["match"] = rule.match;++ override["includes"] = nlohmann::json::array();++ for(const auto & add_hit: add_hits) {+ nlohmann::json include;+ include["id"] = add_hit.doc_id;+ include["position"] = add_hit.position;+ override["includes"].push_back(include);+ }++ override["excludes"] = nlohmann::json::array();+ for(const auto & drop_hit: drop_hits) {+ nlohmann::json exclude;+ exclude["id"] = drop_hit.doc_id;+ override["excludes"].push_back(exclude);+ }++ return override.dump();+ }};class Collection {@@ -142,6 +167,8 @@ public:static std::string get_meta_key(const std::string & collection_name);+ static std::string get_override_key(const std::string & collection_name, const std::string & override_id);+std::string get_seq_id_collection_prefix();std::string get_name();@@ -190,24 +217,13 @@ public:Option<std::string> remove(const std::string & id, const bool remove_from_store = true);- // FIXME: add persistence- bool add_override(override_t & override) {- if(overrides.count("id") != 0) {- return false;- }+ Option<uint32_t> add_override(const override_t & override);- overrides[override.id] = override;- return true;- }+ Option<uint32_t> remove_override(const std::string & id);- bool remove_override(const std::string & id) {- if(overrides.count("id") != 0) {- overrides.erase(id);- return true;- }-- return false;- }+ std::map<std::string, override_t> get_overrides() {+ return overrides;+ };size_t get_num_indices();@@ -232,6 +248,7 @@ public:// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV storestatic constexpr const char* COLLECTION_META_PREFIX = "$CM";static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";+ static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";static constexpr const char* SEQ_ID_PREFIX = "$SI";static constexpr const char* DOC_ID_PREFIX = "$DI";};diff --git a/src/collection.cpp b/src/collection.cppindex 38d7365b..a0240816 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -927,6 +927,33 @@ Option<std::string> Collection::remove(const std::string & id, const bool removereturn Option<std::string>(id);}+Option<uint32_t> Collection::add_override(const override_t & override) {+ if(overrides.count("id") != 0) {+ return Option<uint32_t>(409, "There is already another entry with that `id`.");+ }++ bool inserted = store->insert(Collection::get_override_key(name, override.id), override.to_json_str());+ if(!inserted) {+ return Option<uint32_t>(500, "Error while storing the override on disk.");+ }++ overrides[override.id] = override;+ return Option<uint32_t>(200);+}++Option<uint32_t> Collection::remove_override(const std::string & id) {+ if(overrides.count(id) != 0) {+ bool removed = store->remove(Collection::get_override_key(name, id));+ if(!removed) {+ return Option<uint32_t>(500, "Error while deleting the override from disk.");+ }+ overrides.erase(id);+ return Option<uint32_t>(200);+ }++ return Option<uint32_t>(404, "Could not find that `id`.");+}+size_t Collection::get_num_indices() {return num_indices;}@@ -1013,6 +1040,10 @@ std::string Collection::get_meta_key(const std::string & collection_name) {return std::string(COLLECTION_META_PREFIX) + "_" + collection_name;}+std::string Collection::get_override_key(const std::string & collection_name, const std::string & override_id) {+ return std::string(COLLECTION_OVERRIDE_PREFIX) + "_" + collection_name + "_" + override_id;+}+std::string Collection::get_seq_id_collection_prefix() {return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 36ced335..ddfe6c10 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -97,6 +97,16 @@ Option<bool> CollectionManager::init(Store *store,LOG(INFO) << "Loading collection " << collection->get_name() << std::endl;+ // initialize overrides+ std::vector<std::string> collection_override_jsons;+ store->scan_fill(Collection::get_override_key(this_collection_name, ""), collection_override_jsons);++ for(const auto & collection_override_json: collection_override_jsons) {+ nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);+ override_t override(collection_override);+ collection->add_override(override);+ }+// Fetch records from the store and re-create memory indexstd::vector<std::string> documents;const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();diff --git a/src/index.cpp b/src/index.cppindex ff623363..5418323a 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -838,11 +838,6 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f}}- if(override_query.size() == 0) {- // happens when the curated hit's field has no overlap with query string- //return ;- }-spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;for (art_leaf *token_leaf : override_query) {diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex a9e2170d..6408c3e7 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -143,6 +143,63 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {infile.close();+ // add some overrides+ nlohmann::json override_json_include = {+ {"id", "include-rule"},+ {+ "rule", {+ {"query", "in"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };+ override_json_include["includes"] = nlohmann::json::array();+ override_json_include["includes"][0] = nlohmann::json::object();+ override_json_include["includes"][0]["id"] = "0";+ override_json_include["includes"][0]["position"] = 1;++ override_json_include["includes"][1] = nlohmann::json::object();+ override_json_include["includes"][1]["id"] = "3";+ override_json_include["includes"][1]["position"] = 2;++ override_t override_include(override_json_include);++ nlohmann::json override_json = {+ {"id", "exclude-rule"},+ {+ "rule", {+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };+ override_json["excludes"] = nlohmann::json::array();+ override_json["excludes"][0] = nlohmann::json::object();+ override_json["excludes"][0]["id"] = "4";++ override_json["excludes"][1] = nlohmann::json::object();+ override_json["excludes"][1]["id"] = "11";++ override_t override_exclude(override_json);++ nlohmann::json override_json_deleted = {+ {"id", "deleted-rule"},+ {+ "rule", {+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };++ override_t override_deleted(override_json_deleted);++ collection1->add_override(override_include);+ collection1->add_override(override_exclude);+ collection1->add_override(override_deleted);++ collection1->remove_override("deleted-rule");+std::vector<std::string> search_fields = {"starring", "title"};std::vector<std::string> facets;@@ -168,6 +225,10 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_EQ(schema.size(), collection1->get_schema().size());ASSERT_EQ("points", collection1->get_default_sorting_field());+ ASSERT_EQ(2, collection1->get_overrides().size());+ ASSERT_STREQ("exclude-rule", collection1->get_overrides()["exclude-rule"].id.c_str());+ ASSERT_STREQ("include-rule", collection1->get_overrides()["include-rule"].id.c_str());+results = collection1->search("thomas", search_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(4, results["hits"].size());}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex d9470efa..3df4847c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -838,6 +838,10 @@ TEST_F(CollectionTest, ExcludeIncludeExactQueryMatch) {ASSERT_EQ(3, results["found"].get<uint32_t>());ASSERT_EQ(6, results["facet_counts"][0]["counts"].size());+ ASSERT_STREQ("12", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("5", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("17", results["hits"][2]["document"]["id"].get<std::string>().c_str());+// includenlohmann::json override_json_include = {{"id", "include-rule"},@@ -868,6 +872,10 @@ TEST_F(CollectionTest, ExcludeIncludeExactQueryMatch) {ASSERT_EQ(3, results["hits"].size());ASSERT_EQ(3, results["found"].get<uint32_t>());+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("13", results["hits"][2]["document"]["id"].get<std::string>().c_str());+coll_mul_fields->remove_override("exclude-rule");coll_mul_fields->remove_override("include-rule");