Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Use `num_memory_shards` as parameter name.

kishorenc2f21819db073
diff --git a/include/collection.h b/include/collection.h
index 665f8aaf..11c8fb19 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -140,7 +140,7 @@ private:
std::string default_sorting_field;
- size_t num_indices;
+ size_t num_memory_shards;
const float max_memory_ratio;
@@ -177,9 +177,9 @@ private:
public:
Collection() = delete;
- Collection(const std::string name, const uint32_t collection_id, const uint64_t created_at,
- const uint32_t next_seq_id, Store *store, const std::vector<field> & fields,
- const std::string & default_sorting_field, const size_t num_indices, const float max_memory_ratio);
+ Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,
+ const uint32_t next_seq_id, Store *store, const std::vector<field>& fields,
+ const std::string& default_sorting_field, const size_t num_memory_shards, const float max_memory_ratio);
~Collection();
@@ -257,7 +257,7 @@ public:
return overrides;
};
- size_t get_num_indices();
+ size_t get_num_memory_shards();
static uint32_t get_seq_id_from_key(const std::string & key);
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 01207f8a..dc20f594 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -29,7 +29,7 @@ private:
static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "fields";
static constexpr const char* COLLECTION_DEFAULT_SORTING_FIELD_KEY = "default_sorting_field";
static constexpr const char* COLLECTION_CREATED = "created_at";
- static constexpr const char* COLLECTION_NUM_INDICES = "num_indices";
+ static constexpr const char* COLLECTION_NUM_MEMORY_SHARDS = "num_memory_shards";
std::string bootstrap_auth_key;
std::string bootstrap_search_only_auth_key;
@@ -73,7 +73,7 @@ public:
bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,
const std::string& collection, std::map<std::string, std::string>& params);
- Option<Collection*> create_collection(const std::string name, const size_t num_indices,
+ Option<Collection*> create_collection(const std::string& name, const size_t num_memory_shards,
const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)));
@@ -105,7 +105,7 @@ public:
Option<bool> delete_symlink(const std::string & symlink_name);
- static const size_t DEFAULT_NUM_INDICES = 4;
+ static const size_t DEFAULT_NUM_MEMORY_SHARDS = 4;
static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
static constexpr const char* SYMLINK_PREFIX = "$SL";
diff --git a/src/collection.cpp b/src/collection.cpp
index 06a66725..39cf62bd 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -35,14 +35,14 @@ struct match_index_t {
}
};
-Collection::Collection(const std::string name, const uint32_t collection_id, const uint64_t created_at,
+Collection::Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,
const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,
- const std::string & default_sorting_field, const size_t num_indices,
+ const std::string& default_sorting_field, const size_t num_memory_shards,
const float max_memory_ratio):
- name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
- fields(fields), default_sorting_field(default_sorting_field),
- num_indices(num_indices),
- max_memory_ratio(max_memory_ratio) {
+ name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
+ fields(fields), default_sorting_field(default_sorting_field),
+ num_memory_shards(num_memory_shards),
+ max_memory_ratio(max_memory_ratio) {
for(const field& field: fields) {
search_schema.emplace(field.name, field);
@@ -56,7 +56,7 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con
}
}
- for(size_t i = 0; i < num_indices; i++) {
+ for(size_t i = 0; i < num_memory_shards; i++) {
Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);
indices.push_back(index);
std::thread* thread = new std::thread(&Index::run_search, index);
@@ -132,7 +132,7 @@ nlohmann::json Collection::get_summary_json() {
nlohmann::json json_response;
json_response["name"] = name;
- json_response["num_indices"] = num_indices;
+ json_response["num_memory_shards"] = num_memory_shards;
json_response["num_documents"] = num_documents;
json_response["created_at"] = created_at;
@@ -193,7 +193,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {
std::vector<std::vector<index_record>> iter_batch;
- for(size_t i = 0; i < num_indices; i++) {
+ for(size_t i = 0; i < num_memory_shards; i++) {
iter_batch.emplace_back(std::vector<index_record>());
}
@@ -233,11 +233,11 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {
const uint32_t seq_id = doc_seq_id_op.get();
index_record record(i, seq_id, document);
- iter_batch[seq_id % this->get_num_indices()].emplace_back(record);
+ iter_batch[seq_id % this->get_num_memory_shards()].emplace_back(record);
if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {
batch_index(iter_batch, json_lines, num_indexed);
- for(size_t i_index = 0; i_index < get_num_indices(); i_index++) {
+ for(size_t i_index = 0; i_index < get_num_memory_shards(); i_index++) {
iter_batch[i_index].clear();
}
}
@@ -298,7 +298,7 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
return validation_op;
}
- Index* index = indices[seq_id % num_indices];
+ Index* index = indices[seq_id % num_memory_shards];
index->index_in_memory(document, seq_id, default_sorting_field);
num_documents += 1;
@@ -310,7 +310,7 @@ size_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> &
std::vector<std::future<size_t>> futures;
size_t num_indexed = 0;
- for(size_t i=0; i < num_indices; i++) {
+ for(size_t i=0; i < num_memory_shards; i++) {
futures.push_back(
std::async(&Index::batch_memory_index, indices[i], std::ref(iter_batch[i]), default_sorting_field,
search_schema, facet_schema)
@@ -470,14 +470,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
for(size_t inner_pos = 0; inner_pos < std::min(ids_per_pos, pos_ids.second.size()); inner_pos++) {
auto seq_id = pos_ids.second[inner_pos];
- auto index_id = (seq_id % num_indices);
+ auto index_id = (seq_id % num_memory_shards);
index_to_included_ids[index_id][outer_pos][inner_pos] = seq_id;
//LOG(INFO) << "Adding seq_id " << seq_id << " to index_id " << index_id;
}
}
for(auto seq_id: excluded_ids) {
- auto index_id = (seq_id % num_indices);
+ auto index_id = (seq_id % num_memory_shards);
index_to_excluded_ids[index_id].push_back(seq_id);
}
@@ -1202,7 +1202,7 @@ void Collection::highlight_result(const field &search_field,
for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {
// Must search for the token string fresh on that field for the given document since `token_leaf`
// is from the best matched field and need not be present in other fields of a document.
- Index* index = indices[field_order_kv->key % num_indices];
+ Index* index = indices[field_order_kv->key % num_memory_shards];
art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);
if(actual_leaf != nullptr) {
query_suggestion.push_back(actual_leaf);
@@ -1380,7 +1380,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) {
void Collection::remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {
const std::string& id = document["id"];
- Index* index = indices[seq_id % num_indices];
+ Index* index = indices[seq_id % num_memory_shards];
index->remove(seq_id, document);
num_documents -= 1;
@@ -1454,8 +1454,8 @@ Option<uint32_t> Collection::remove_override(const std::string & id) {
return Option<uint32_t>(404, "Could not find that `id`.");
}
-size_t Collection::get_num_indices() {
- return num_indices;
+size_t Collection::get_num_memory_shards() {
+ return num_memory_shards;
}
uint32_t Collection::get_seq_id_from_key(const std::string & key) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 6d0b667d..416d770d 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -4,8 +4,8 @@
#include "collection_manager.h"
#include "logger.h"
-constexpr const char* CollectionManager::COLLECTION_NUM_INDICES;
-constexpr const size_t CollectionManager::DEFAULT_NUM_INDICES;
+constexpr const char* CollectionManager::COLLECTION_NUM_MEMORY_SHARDS;
+constexpr const size_t CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;
CollectionManager::CollectionManager() {
@@ -34,11 +34,11 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
uint64_t created_at = collection_meta.find((const char*)COLLECTION_CREATED) != collection_meta.end() ?
collection_meta[COLLECTION_CREATED].get<uint64_t>() : 0;
- size_t num_indices = collection_meta.count(COLLECTION_NUM_INDICES) != 0 ?
- collection_meta[COLLECTION_NUM_INDICES].get<size_t>() :
- DEFAULT_NUM_INDICES;
+ size_t num_memory_shards = collection_meta.count(COLLECTION_NUM_MEMORY_SHARDS) != 0 ?
+ collection_meta[COLLECTION_NUM_MEMORY_SHARDS].get<size_t>() :
+ DEFAULT_NUM_MEMORY_SHARDS;
- LOG(INFO) << "Loading collection with " << num_indices << " indices.";
+ LOG(INFO) << "Loading collection with " << num_memory_shards << " memory shards.";
Collection* collection = new Collection(this_collection_name,
collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
@@ -47,7 +47,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
store,
fields,
default_sorting_field,
- num_indices,
+ num_memory_shards,
max_memory_ratio);
return collection;
@@ -138,7 +138,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
std::vector<std::vector<index_record>> iter_batch;
- for(size_t i = 0; i < collection->get_num_indices(); i++) {
+ for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {
iter_batch.push_back(std::vector<index_record>());
}
@@ -158,7 +158,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
num_docs_read++;
- iter_batch[seq_id % collection->get_num_indices()].emplace_back(index_record(0, seq_id, document));
+ iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document));
// Peek and check for last record right here so that we handle batched indexing correctly
// Without doing this, the "last batch" would have to be indexed outside the loop.
@@ -171,7 +171,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
collection->par_index_in_memory(iter_batch, indexed_counts);
- for(size_t i = 0; i < collection->get_num_indices(); i++) {
+ for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {
size_t num_records = iter_batch[i].size();
size_t num_indexed = indexed_counts[i];
@@ -231,8 +231,8 @@ bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,
return auth_manager.authenticate(auth_key_sent, action, collection, params);
}
-Option<Collection*> CollectionManager::create_collection(const std::string name,
- const size_t num_indices,
+Option<Collection*> CollectionManager::create_collection(const std::string& name,
+ const size_t num_memory_shards,
const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at) {
@@ -279,10 +279,10 @@ Option<Collection*> CollectionManager::create_collection(const std::string name,
collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
collection_meta[COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;
collection_meta[COLLECTION_CREATED] = created_at;
- collection_meta[COLLECTION_NUM_INDICES] = num_indices;
+ collection_meta[COLLECTION_NUM_MEMORY_SHARDS] = num_memory_shards;
Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,
- default_sorting_field, num_indices,
+ default_sorting_field, num_memory_shards,
this->max_memory_ratio);
next_collection_id++;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index fc347711..9b6567bb 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -42,7 +42,7 @@ bool get_collections(http_req & req, http_res & res) {
}
bool post_create_collection(http_req & req, http_res & res) {
- const char* NUM_INDICES = "num_indices";
+ const char* NUM_MEMORY_SHARDS = "num_memory_shards";
nlohmann::json req_json;
try {
@@ -62,8 +62,8 @@ bool post_create_collection(http_req & req, http_res & res) {
return false;
}
- if(req_json.count(NUM_INDICES) == 0) {
- req_json[NUM_INDICES] = CollectionManager::DEFAULT_NUM_INDICES;
+ if(req_json.count(NUM_MEMORY_SHARDS) == 0) {
+ req_json[NUM_MEMORY_SHARDS] = CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;
}
if(req_json.count("fields") == 0) {
@@ -84,13 +84,13 @@ bool post_create_collection(http_req & req, http_res & res) {
return false;
}
- if(!req_json[NUM_INDICES].is_number_unsigned()) {
- res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");
+ if(!req_json[NUM_MEMORY_SHARDS].is_number_unsigned()) {
+ res.set_400(std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");
return false;
}
- if(req_json[NUM_INDICES].get<size_t>() == 0) {
- res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");
+ if(req_json[NUM_MEMORY_SHARDS].get<size_t>() == 0) {
+ res.set_400(std::string("`") + NUM_MEMORY_SHARDS + "` should be a positive integer.");
return false;
}
@@ -140,7 +140,7 @@ bool post_create_collection(http_req & req, http_res & res) {
const std::string & default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get<std::string>();
const Option<Collection*> & collection_op =
- collectionManager.create_collection(req_json["name"], req_json[NUM_INDICES].get<size_t>(),
+ collectionManager.create_collection(req_json["name"], req_json[NUM_MEMORY_SHARDS].get<size_t>(),
fields, default_sorting_field);
if(collection_op.ok()) {
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 902d61cd..63fbea4e 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -86,7 +86,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
"\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"
"{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"
"{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"
- "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\",\"num_indices\":4}",
+ "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\",\"num_memory_shards\":4}",
collection_meta_json);
ASSERT_EQ("1", next_collection_id);
}