Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use `num_memory_shards` as parameter name.
kishorenc2f21819db073
diff --git a/include/collection.h b/include/collection.hindex 665f8aaf..11c8fb19 100644--- a/include/collection.h+++ b/include/collection.h@@ -140,7 +140,7 @@ private:std::string default_sorting_field;- size_t num_indices;+ size_t num_memory_shards;const float max_memory_ratio;@@ -177,9 +177,9 @@ private:public:Collection() = delete;- Collection(const std::string name, const uint32_t collection_id, const uint64_t created_at,- const uint32_t next_seq_id, Store *store, const std::vector<field> & fields,- const std::string & default_sorting_field, const size_t num_indices, const float max_memory_ratio);+ Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,+ const uint32_t next_seq_id, Store *store, const std::vector<field>& fields,+ const std::string& default_sorting_field, const size_t num_memory_shards, const float max_memory_ratio);~Collection();@@ -257,7 +257,7 @@ public:return overrides;};- size_t get_num_indices();+ size_t get_num_memory_shards();static uint32_t get_seq_id_from_key(const std::string & key);diff --git a/include/collection_manager.h b/include/collection_manager.hindex 01207f8a..dc20f594 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -29,7 +29,7 @@ private:static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "fields";static constexpr const char* COLLECTION_DEFAULT_SORTING_FIELD_KEY = "default_sorting_field";static constexpr const char* COLLECTION_CREATED = "created_at";- static constexpr const char* COLLECTION_NUM_INDICES = "num_indices";+ static constexpr const char* COLLECTION_NUM_MEMORY_SHARDS = "num_memory_shards";std::string bootstrap_auth_key;std::string bootstrap_search_only_auth_key;@@ -73,7 +73,7 @@ public:bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,const std::string& collection, std::map<std::string, std::string>& params);- Option<Collection*> create_collection(const std::string name, const size_t num_indices,+ Option<Collection*> create_collection(const std::string& name, const size_t num_memory_shards,const std::vector<field> & fields,const std::string & default_sorting_field,const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)));@@ -105,7 +105,7 @@ public:Option<bool> delete_symlink(const std::string & symlink_name);- static const size_t DEFAULT_NUM_INDICES = 4;+ static const size_t DEFAULT_NUM_MEMORY_SHARDS = 4;static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";static constexpr const char* SYMLINK_PREFIX = "$SL";diff --git a/src/collection.cpp b/src/collection.cppindex 06a66725..39cf62bd 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -35,14 +35,14 @@ struct match_index_t {}};-Collection::Collection(const std::string name, const uint32_t collection_id, const uint64_t created_at,+Collection::Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,- const std::string & default_sorting_field, const size_t num_indices,+ const std::string& default_sorting_field, const size_t num_memory_shards,const float max_memory_ratio):- name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),- fields(fields), default_sorting_field(default_sorting_field),- num_indices(num_indices),- max_memory_ratio(max_memory_ratio) {+ name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),+ fields(fields), default_sorting_field(default_sorting_field),+ num_memory_shards(num_memory_shards),+ max_memory_ratio(max_memory_ratio) {for(const field& field: fields) {search_schema.emplace(field.name, field);@@ -56,7 +56,7 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con}}- for(size_t i = 0; i < num_indices; i++) {+ for(size_t i = 0; i < num_memory_shards; i++) {Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);indices.push_back(index);std::thread* thread = new std::thread(&Index::run_search, index);@@ -132,7 +132,7 @@ nlohmann::json Collection::get_summary_json() {nlohmann::json json_response;json_response["name"] = name;- json_response["num_indices"] = num_indices;+ json_response["num_memory_shards"] = num_memory_shards;json_response["num_documents"] = num_documents;json_response["created_at"] = created_at;@@ -193,7 +193,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {std::vector<std::vector<index_record>> iter_batch;- for(size_t i = 0; i < num_indices; i++) {+ for(size_t i = 0; i < num_memory_shards; i++) {iter_batch.emplace_back(std::vector<index_record>());}@@ -233,11 +233,11 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {const uint32_t seq_id = doc_seq_id_op.get();index_record record(i, seq_id, document);- iter_batch[seq_id % this->get_num_indices()].emplace_back(record);+ iter_batch[seq_id % this->get_num_memory_shards()].emplace_back(record);if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {batch_index(iter_batch, json_lines, num_indexed);- for(size_t i_index = 0; i_index < get_num_indices(); i_index++) {+ for(size_t i_index = 0; i_index < get_num_memory_shards(); i_index++) {iter_batch[i_index].clear();}}@@ -298,7 +298,7 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinreturn validation_op;}- Index* index = indices[seq_id % num_indices];+ Index* index = indices[seq_id % num_memory_shards];index->index_in_memory(document, seq_id, default_sorting_field);num_documents += 1;@@ -310,7 +310,7 @@ size_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> &std::vector<std::future<size_t>> futures;size_t num_indexed = 0;- for(size_t i=0; i < num_indices; i++) {+ for(size_t i=0; i < num_memory_shards; i++) {futures.push_back(std::async(&Index::batch_memory_index, indices[i], std::ref(iter_batch[i]), default_sorting_field,search_schema, facet_schema)@@ -470,14 +470,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::for(size_t inner_pos = 0; inner_pos < std::min(ids_per_pos, pos_ids.second.size()); inner_pos++) {auto seq_id = pos_ids.second[inner_pos];- auto index_id = (seq_id % num_indices);+ auto index_id = (seq_id % num_memory_shards);index_to_included_ids[index_id][outer_pos][inner_pos] = seq_id;//LOG(INFO) << "Adding seq_id " << seq_id << " to index_id " << index_id;}}for(auto seq_id: excluded_ids) {- auto index_id = (seq_id % num_indices);+ auto index_id = (seq_id % num_memory_shards);index_to_excluded_ids[index_id].push_back(seq_id);}@@ -1202,7 +1202,7 @@ void Collection::highlight_result(const field &search_field,for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {// Must search for the token string fresh on that field for the given document since `token_leaf`// is from the best matched field and need not be present in other fields of a document.- Index* index = indices[field_order_kv->key % num_indices];+ Index* index = indices[field_order_kv->key % num_memory_shards];art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);if(actual_leaf != nullptr) {query_suggestion.push_back(actual_leaf);@@ -1380,7 +1380,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) {void Collection::remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {const std::string& id = document["id"];- Index* index = indices[seq_id % num_indices];+ Index* index = indices[seq_id % num_memory_shards];index->remove(seq_id, document);num_documents -= 1;@@ -1454,8 +1454,8 @@ Option<uint32_t> Collection::remove_override(const std::string & id) {return Option<uint32_t>(404, "Could not find that `id`.");}-size_t Collection::get_num_indices() {- return num_indices;+size_t Collection::get_num_memory_shards() {+ return num_memory_shards;}uint32_t Collection::get_seq_id_from_key(const std::string & key) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 6d0b667d..416d770d 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -4,8 +4,8 @@#include "collection_manager.h"#include "logger.h"-constexpr const char* CollectionManager::COLLECTION_NUM_INDICES;-constexpr const size_t CollectionManager::DEFAULT_NUM_INDICES;+constexpr const char* CollectionManager::COLLECTION_NUM_MEMORY_SHARDS;+constexpr const size_t CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;CollectionManager::CollectionManager() {@@ -34,11 +34,11 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionuint64_t created_at = collection_meta.find((const char*)COLLECTION_CREATED) != collection_meta.end() ?collection_meta[COLLECTION_CREATED].get<uint64_t>() : 0;- size_t num_indices = collection_meta.count(COLLECTION_NUM_INDICES) != 0 ?- collection_meta[COLLECTION_NUM_INDICES].get<size_t>() :- DEFAULT_NUM_INDICES;+ size_t num_memory_shards = collection_meta.count(COLLECTION_NUM_MEMORY_SHARDS) != 0 ?+ collection_meta[COLLECTION_NUM_MEMORY_SHARDS].get<size_t>() :+ DEFAULT_NUM_MEMORY_SHARDS;- LOG(INFO) << "Loading collection with " << num_indices << " indices.";+ LOG(INFO) << "Loading collection with " << num_memory_shards << " memory shards.";Collection* collection = new Collection(this_collection_name,collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),@@ -47,7 +47,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionstore,fields,default_sorting_field,- num_indices,+ num_memory_shards,max_memory_ratio);return collection;@@ -138,7 +138,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {std::vector<std::vector<index_record>> iter_batch;- for(size_t i = 0; i < collection->get_num_indices(); i++) {+ for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {iter_batch.push_back(std::vector<index_record>());}@@ -158,7 +158,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {num_docs_read++;- iter_batch[seq_id % collection->get_num_indices()].emplace_back(index_record(0, seq_id, document));+ iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document));// Peek and check for last record right here so that we handle batched indexing correctly// Without doing this, the "last batch" would have to be indexed outside the loop.@@ -171,7 +171,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {collection->par_index_in_memory(iter_batch, indexed_counts);- for(size_t i = 0; i < collection->get_num_indices(); i++) {+ for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {size_t num_records = iter_batch[i].size();size_t num_indexed = indexed_counts[i];@@ -231,8 +231,8 @@ bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,return auth_manager.authenticate(auth_key_sent, action, collection, params);}-Option<Collection*> CollectionManager::create_collection(const std::string name,- const size_t num_indices,+Option<Collection*> CollectionManager::create_collection(const std::string& name,+ const size_t num_memory_shards,const std::vector<field> & fields,const std::string & default_sorting_field,const uint64_t created_at) {@@ -279,10 +279,10 @@ Option<Collection*> CollectionManager::create_collection(const std::string name,collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;collection_meta[COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;collection_meta[COLLECTION_CREATED] = created_at;- collection_meta[COLLECTION_NUM_INDICES] = num_indices;+ collection_meta[COLLECTION_NUM_MEMORY_SHARDS] = num_memory_shards;Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,- default_sorting_field, num_indices,+ default_sorting_field, num_memory_shards,this->max_memory_ratio);next_collection_id++;diff --git a/src/core_api.cpp b/src/core_api.cppindex fc347711..9b6567bb 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -42,7 +42,7 @@ bool get_collections(http_req & req, http_res & res) {}bool post_create_collection(http_req & req, http_res & res) {- const char* NUM_INDICES = "num_indices";+ const char* NUM_MEMORY_SHARDS = "num_memory_shards";nlohmann::json req_json;try {@@ -62,8 +62,8 @@ bool post_create_collection(http_req & req, http_res & res) {return false;}- if(req_json.count(NUM_INDICES) == 0) {- req_json[NUM_INDICES] = CollectionManager::DEFAULT_NUM_INDICES;+ if(req_json.count(NUM_MEMORY_SHARDS) == 0) {+ req_json[NUM_MEMORY_SHARDS] = CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;}if(req_json.count("fields") == 0) {@@ -84,13 +84,13 @@ bool post_create_collection(http_req & req, http_res & res) {return false;}- if(!req_json[NUM_INDICES].is_number_unsigned()) {- res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");+ if(!req_json[NUM_MEMORY_SHARDS].is_number_unsigned()) {+ res.set_400(std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");return false;}- if(req_json[NUM_INDICES].get<size_t>() == 0) {- res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");+ if(req_json[NUM_MEMORY_SHARDS].get<size_t>() == 0) {+ res.set_400(std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");return false;}@@ -140,7 +140,7 @@ bool post_create_collection(http_req & req, http_res & res) {const std::string & default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get<std::string>();const Option<Collection*> & collection_op =- collectionManager.create_collection(req_json["name"], req_json[NUM_INDICES].get<size_t>(),+ collectionManager.create_collection(req_json["name"], req_json[NUM_MEMORY_SHARDS].get<size_t>(),fields, default_sorting_field);if(collection_op.ok()) {diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 902d61cd..63fbea4e 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -86,7 +86,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {"\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"},""{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},""{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"- "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\",\"num_indices\":4}",+ "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\",\"num_memory_shards\":4}",collection_meta_json);ASSERT_EQ("1", next_collection_id);}