Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Parameterize number of indices per collection.

kishorenc09541e531180
diff --git a/include/collection_manager.h b/include/collection_manager.h
index dc8c475a..01207f8a 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -29,12 +29,11 @@ private:
static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "fields";
static constexpr const char* COLLECTION_DEFAULT_SORTING_FIELD_KEY = "default_sorting_field";
static constexpr const char* COLLECTION_CREATED = "created_at";
+ static constexpr const char* COLLECTION_NUM_INDICES = "num_indices";
std::string bootstrap_auth_key;
std::string bootstrap_search_only_auth_key;
- size_t default_num_indices;
-
float max_memory_ratio;
CollectionManager();
@@ -60,8 +59,7 @@ public:
CollectionManager(CollectionManager const&) = delete;
void operator=(CollectionManager const&) = delete;
- void init(Store *store, const size_t default_num_indices, const float max_memory_ratio,
- const std::string & auth_key);
+ void init(Store *store, const float max_memory_ratio, const std::string & auth_key);
Option<bool> load(const size_t init_batch_size=1000);
@@ -75,7 +73,8 @@ public:
bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,
const std::string& collection, std::map<std::string, std::string>& params);
- Option<Collection*> create_collection(const std::string name, const std::vector<field> & fields,
+ Option<Collection*> create_collection(const std::string name, const size_t num_indices,
+ const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)));
@@ -106,6 +105,8 @@ public:
Option<bool> delete_symlink(const std::string & symlink_name);
+ static const size_t DEFAULT_NUM_INDICES = 4;
+
static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
static constexpr const char* SYMLINK_PREFIX = "$SL";
};
\ No newline at end of file
diff --git a/include/config.h b/include/config.h
index 9df51978..b58e2748 100644
--- a/include/config.h
+++ b/include/config.h
@@ -29,8 +29,6 @@ private:
bool enable_cors;
- size_t indices_per_collection;
-
float max_memory_ratio;
std::string config_file;
@@ -43,7 +41,6 @@ public:
this->api_port = 8108;
this->peering_port = 8107;
this->enable_cors = false;
- this->indices_per_collection = 4;
this->max_memory_ratio = 1.0f;
}
@@ -90,10 +87,6 @@ public:
this->enable_cors = enable_cors;
}
- void set_indices_per_collection(size_t indices_per_collection) {
- this->indices_per_collection = indices_per_collection;
- }
-
// getters
std::string get_data_dir() const {
@@ -141,10 +134,6 @@ public:
return this->enable_cors;
}
- size_t get_indices_per_collection() const {
- return indices_per_collection;
- }
-
std::string get_peering_address() const {
return this->peering_address;
}
diff --git a/include/core_api.h b/include/core_api.h
index e8e01c3a..6f29771b 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -69,6 +69,6 @@ bool get_metrics_json(http_req& req, http_res& res);
// Misc helpers
-bool async_write_request(void *data);
+bool raft_write_send_response(void *data);
static constexpr const char* SEND_RESPONSE_MSG = "send_response";
diff --git a/include/http_data.h b/include/http_data.h
index 101d0e26..2aa7f1b6 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -15,7 +15,7 @@ extern "C" {
}
struct h2o_custom_timer_t {
- h2o_timer_t timer{};
+ h2o_timer_t timer;
void *data;
h2o_custom_timer_t(): data(nullptr) {}
@@ -182,6 +182,14 @@ struct http_req {
return content.dump();
}
+
+ bool is_ending() {
+ return stream_state == "END" || stream_state == "NON_STREAMING";
+ }
+
+ bool is_starting() {
+ return stream_state == "START" || stream_state == "NON_STREAMING";
+ }
};
struct request_response {
diff --git a/include/http_server.h b/include/http_server.h
index 574297df..e3ea3273 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -127,6 +127,8 @@ public:
void send_response(http_req* request, http_res* response);
+ static void destroy_request_response(http_req* request, http_res* response);
+
static void stream_response(http_req& request, http_res& response);
uint64_t find_route(const std::vector<std::string> & path_parts, const std::string & http_method,
diff --git a/src/collection.cpp b/src/collection.cpp
index a1219c93..3a12f0c2 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -132,6 +132,7 @@ nlohmann::json Collection::get_summary_json() {
nlohmann::json json_response;
json_response["name"] = name;
+ json_response["num_indices"] = num_indices;
json_response["num_documents"] = num_documents;
json_response["created_at"] = created_at;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 8f27f93e..6e628fd8 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -4,6 +4,9 @@
#include "collection_manager.h"
#include "logger.h"
+constexpr const char* CollectionManager::COLLECTION_NUM_INDICES;
+constexpr const size_t CollectionManager::DEFAULT_NUM_INDICES;
+
CollectionManager::CollectionManager() {
}
@@ -31,6 +34,12 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
uint64_t created_at = collection_meta.find((const char*)COLLECTION_CREATED) != collection_meta.end() ?
collection_meta[COLLECTION_CREATED].get<uint64_t>() : 0;
+ size_t num_indices = collection_meta.count(COLLECTION_NUM_INDICES) != 0 ?
+ collection_meta[COLLECTION_NUM_INDICES].get<size_t>() :
+ DEFAULT_NUM_INDICES;
+
+ LOG(INFO) << "Loading collection with " << num_indices << " indices.";
+
Collection* collection = new Collection(this_collection_name,
collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
created_at,
@@ -38,7 +47,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
store,
fields,
default_sorting_field,
- default_num_indices,
+ num_indices,
max_memory_ratio);
return collection;
@@ -50,12 +59,10 @@ void CollectionManager::add_to_collections(Collection* collection) {
}
void CollectionManager::init(Store *store,
- const size_t default_num_indices,
const float max_memory_ratio,
const std::string & auth_key) {
this->store = store;
this->bootstrap_auth_key = auth_key;
- this->default_num_indices = default_num_indices;
this->max_memory_ratio = max_memory_ratio;
auth_manager.init(store);
@@ -227,7 +234,9 @@ bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,
return auth_manager.authenticate(auth_key_sent, action, collection, params);
}
-Option<Collection*> CollectionManager::create_collection(const std::string name, const std::vector<field> & fields,
+Option<Collection*> CollectionManager::create_collection(const std::string name,
+ const size_t num_indices,
+ const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at) {
if(store->contains(Collection::get_meta_key(name))) {
@@ -273,9 +282,10 @@ Option<Collection*> CollectionManager::create_collection(const std::string name,
collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
collection_meta[COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;
collection_meta[COLLECTION_CREATED] = created_at;
+ collection_meta[COLLECTION_NUM_INDICES] = num_indices;
Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,
- default_sorting_field, this->default_num_indices,
+ default_sorting_field, num_indices,
this->max_memory_ratio);
next_collection_id++;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index aead9d48..4ffa9059 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -42,6 +42,7 @@ bool get_collections(http_req & req, http_res & res) {
}
bool post_create_collection(http_req & req, http_res & res) {
+ const char* NUM_INDICES = "num_indices";
nlohmann::json req_json;
try {
@@ -61,6 +62,10 @@ bool post_create_collection(http_req & req, http_res & res) {
return false;
}
+ if(req_json.count(NUM_INDICES) == 0) {
+ req_json[NUM_INDICES] = CollectionManager::DEFAULT_NUM_INDICES;
+ }
+
if(req_json.count("fields") == 0) {
res.set_400("Parameter `fields` is required.");
return false;
@@ -79,6 +84,16 @@ bool post_create_collection(http_req & req, http_res & res) {
return false;
}
+ if(!req_json[NUM_INDICES].is_number_unsigned()) {
+ res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");
+ return false;
+ }
+
+ if(req_json[NUM_INDICES].get<size_t>() == 0) {
+ res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");
+ return false;
+ }
+
if(collectionManager.get_collection(req_json["name"]) != nullptr) {
res.set_409("Collection with name `" + req_json["name"].get<std::string>() + "` already exists.");
return false;
@@ -125,7 +140,8 @@ bool post_create_collection(http_req & req, http_res & res) {
const std::string & default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get<std::string>();
const Option<Collection*> & collection_op =
- collectionManager.create_collection(req_json["name"], fields, default_sorting_field);
+ collectionManager.create_collection(req_json["name"], req_json[NUM_INDICES].get<size_t>(),
+ fields, default_sorting_field);
if(collection_op.ok()) {
nlohmann::json json_response = collection_op.get()->get_summary_json();
@@ -571,14 +587,21 @@ bool post_import_documents(http_req& req, http_res& res) {
if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {
req.stream_state = "NON_STREAMING";
- res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be an unsigned integer.");
+ res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
HttpServer::stream_response(req, res);
return false;
}
const size_t IMPORT_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]);
- LOG(INFO) << "Import batch size: " << IMPORT_BATCH_SIZE;
+ if(IMPORT_BATCH_SIZE == 0) {
+ res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
+ return false;
+ }
+
+ if(req.stream_state == "START" && req.body_index == 0) {
+ LOG(INFO) << "Import batch size: " << IMPORT_BATCH_SIZE;
+ }
CollectionManager & collectionManager = CollectionManager::get_instance();
Collection* collection = collectionManager.get_collection(req.params["collection"]);
@@ -1036,8 +1059,8 @@ bool del_key(http_req &req, http_res &res) {
return true;
}
-bool async_write_request(void *data) {
- //LOG(INFO) << "async_write_request called";
+bool raft_write_send_response(void *data) {
+ //LOG(INFO) << "raft_write_send_response called";
AsyncIndexArg* index_arg = static_cast<AsyncIndexArg*>(data);
std::unique_ptr<AsyncIndexArg> index_arg_guard(index_arg);
@@ -1047,10 +1070,10 @@ bool async_write_request(void *data) {
// route not found
index_arg->res->set_400("Not found.");
} else if(index_arg->req->route_hash != static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED)) {
- // call the underlying http handler
route_path* found_rpath = nullptr;
bool route_found = server->get_route(index_arg->req->route_hash, &found_rpath);
if(route_found) {
+ // call request handler
found_rpath->handler(*index_arg->req, *index_arg->res);
async_res = found_rpath->async_res;
} else {
@@ -1058,17 +1081,11 @@ bool async_write_request(void *data) {
}
}
- if(!async_res && index_arg->req->_req != nullptr) {
- // we have to return a response to the client
+ if(!async_res) {
+ // only handle synchronous responses as async ones are handled by their handlers
server->send_response(index_arg->req, index_arg->res);
}
- if(index_arg->req->_req == nullptr) {
- // indicates raft serialized request and response -- lifecycle must be managed here
- delete index_arg->req;
- delete index_arg->res;
- }
-
if(index_arg->promise != nullptr) {
index_arg->promise->set_value(true); // returns control back to caller
}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index ef55e4f7..31bf29e3 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -71,7 +71,6 @@ void HttpServer::on_ssl_refresh_timeout(h2o_timer_t *entry) {
int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
// Set up a timer to refresh SSL config from disk. Also, initializing upfront so that destructor works
ssl_refresh_timer = h2o_custom_timer_t(this);
- ssl_refresh_timer.timer.expire_at = UINT64_MAX-1;
h2o_timer_init(&ssl_refresh_timer.timer, on_ssl_refresh_timeout);
h2o_timer_link(ctx.loop, SSL_REFRESH_INTERVAL_MS, &ssl_refresh_timer.timer); // every 8 hours
@@ -278,14 +277,7 @@ void HttpServer::on_res_generator_dispose(void *self) {
res_generator->rpath->handler(*res_generator->request, *res_generator->response);
}
- if(res_generator->request->defer_timer.data != nullptr) {
- deferred_req_res_t* deferred_req_res = static_cast<deferred_req_res_t*>(res_generator->request->defer_timer.data);
- delete deferred_req_res;
- }
-
- // res_generator itself is reference counted, so we only delete the members
- delete res_generator->request;
- delete res_generator->response;
+ destroy_request_response(res_generator->request, res_generator->response);
}
int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
@@ -509,24 +501,15 @@ void HttpServer::defer_processing(http_req& req, http_res& res, size_t timeout_m
if(req.defer_timer.data == nullptr) {
auto deferred_req_res = new deferred_req_res_t{&req, &res, this};
req.defer_timer.data = deferred_req_res;
- req.defer_timer.timer.expire_at = UINT64_MAX-1;
h2o_timer_init(&req.defer_timer.timer, on_deferred_process_request);
}
+ h2o_timer_unlink(&req.defer_timer.timer);
h2o_timer_link(ctx.loop, timeout_ms, &req.defer_timer.timer);
}
-void HttpServer::send_response(http_req* request, http_res* response) {
- h2o_req_t* req = request->_req;
- h2o_generator_t& generator = *response->generator;
-
- h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);
- req->res.status = response->status_code;
- req->res.reason = http_res::get_status_reason(response->status_code);
- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE,
- nullptr, response->content_type_header.c_str(), response->content_type_header.size());
- h2o_start_response(req, &generator);
- h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);
+void HttpServer::send_message(const std::string & type, void* data) {
+ message_dispatcher->send_message(type, data);
}
int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string & message) {
@@ -540,8 +523,22 @@ int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string
return 0;
}
-void HttpServer::send_message(const std::string & type, void* data) {
- message_dispatcher->send_message(type, data);
+void HttpServer::send_response(http_req* request, http_res* response) {
+ if(request->_req == nullptr) {
+ // indicates serialized request and response -- lifecycle must be managed here
+ return destroy_request_response(request, response);
+ }
+
+ h2o_req_t* req = request->_req;
+ h2o_generator_t& generator = *response->generator;
+
+ h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);
+ req->res.status = response->status_code;
+ req->res.reason = http_res::get_status_reason(response->status_code);
+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE,
+ nullptr, response->content_type_header.c_str(), response->content_type_header.size());
+ h2o_start_response(req, &generator);
+ h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);
}
void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
@@ -564,9 +561,10 @@ void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
}
void HttpServer::stream_response(http_req& request, http_res& response) {
- if(!request._req) {
- // underlying request is no longer available
- return ;
+ if(request._req == nullptr) {
+ // serialized request or underlying request is no longer available
+ destroy_request_response(&request, &response);
+ return;
}
h2o_req_t* req = request._req;
@@ -590,6 +588,16 @@ void HttpServer::stream_response(http_req& request, http_res& response) {
h2o_send(req, &body, 1, state);
}
+void HttpServer::destroy_request_response(http_req* request, http_res* response) {
+ if(request->defer_timer.data != nullptr) {
+ deferred_req_res_t* deferred_req_res = static_cast<deferred_req_res_t*>(request->defer_timer.data);
+ delete deferred_req_res;
+ }
+
+ delete request;
+ delete response;
+}
+
void HttpServer::set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const route_path& rpath,
const std::string& auth_key)) {
auth_handler = handler;
@@ -675,3 +683,4 @@ bool HttpServer::get_route(uint64_t hash, route_path** found_rpath) {
uint64_t HttpServer::node_state() const {
return replication_state->node_state();
}
+
diff --git a/src/main/main.cpp b/src/main/main.cpp
index fc5ad311..9551038c 100644
--- a/src/main/main.cpp
+++ b/src/main/main.cpp
@@ -35,7 +35,7 @@ int main(int argc, char* argv[]) {
Collection *collection = collectionManager.get_collection("github_top1k");
if(collection == nullptr) {
- collection = collectionManager.create_collection("github_top1k", fields_to_index, "stars").get();
+ collection = collectionManager.create_collection("github_top1k", 4, fields_to_index, "stars").get();
}
int j = 0;
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 03f54dc9..e3057afc 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -92,6 +92,7 @@ std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_end
}
void ReplicationState::write(http_req* request, http_res* response) {
+ // NOTE: this is executed on a different thread and runs concurrent to http thread
if (!is_leader()) {
if(node->leader_id().is_empty()) {
// Handle no leader scenario
@@ -187,11 +188,10 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
} else {
// Parse request from the log
response = new http_res;
- http_req* remote_request = new http_req;
- remote_request->deserialize(iter.data().to_string());
- remote_request->_req = nullptr; // indicates remote request
- request = remote_request;
+ request = new http_req;
+ request->deserialize(iter.data().to_string());
+ request->_req = nullptr; // indicates remote request
}
if(request->_req == nullptr && request->body == "INIT_SNAPSHOT") {
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index abff98cb..076ad979 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -329,8 +329,7 @@ int run_server(const Config & config, const std::string & version, void (*master
Store store(db_dir);
CollectionManager & collectionManager = CollectionManager::get_instance();
- collectionManager.init(&store, config.get_indices_per_collection(),
- config.get_max_memory_ratio(), config.get_api_key());
+ collectionManager.init(&store, config.get_max_memory_ratio(), config.get_api_key());
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
@@ -348,7 +347,7 @@ int run_server(const Config & config, const std::string & version, void (*master
server->set_auth_handler(handle_authentication);
server->on(SEND_RESPONSE_MSG, on_send_response);
- server->on(ReplicationState::REPLICATION_MSG, async_write_request);
+ server->on(ReplicationState::REPLICATION_MSG, raft_write_send_response);
// first we start the peering service
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 84ae0f67..7d430a7c 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
}
@@ -50,7 +50,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@@ -336,7 +336,7 @@ TEST_F(CollectionFacetingTest, FacetCountsBool) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@@ -390,7 +390,7 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@@ -507,7 +507,7 @@ TEST_F(CollectionFacetingTest, FacetStatOnFloatFields) {
coll_float_fields = collectionManager.get_collection("coll_float_fields");
if(coll_float_fields == nullptr) {
- coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "average").get();
+ coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "average").get();
}
std::string json_line;
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index b680cc8b..07cfa639 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -18,7 +18,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
std::vector<field> fields = {
@@ -31,7 +31,7 @@ protected:
coll_group = collectionManager.get_collection("coll_group");
if(coll_group == nullptr) {
- coll_group = collectionManager.create_collection("coll_group", fields, "rating").get();
+ coll_group = collectionManager.create_collection("coll_group", 4, fields, "rating").get();
}
std::ifstream infile(std::string(ROOT_DIR)+"test/group_documents.jsonl");
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index eed11192..902d61cd 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
search_fields = {
@@ -31,7 +31,7 @@ protected:
};
sort_fields = { sort_by("points", "DESC") };
- collection1 = collectionManager.create_collection("collection1", search_fields, "points", 12345).get();
+ collection1 = collectionManager.create_collection("collection1", 4, search_fields, "points", 12345).get();
}
virtual void SetUp() {
@@ -86,7 +86,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
"\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"
"{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"
"{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"
- "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\"}",
+ "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\",\"num_indices\":4}",
collection_meta_json);
ASSERT_EQ("1", next_collection_id);
}
@@ -124,7 +124,7 @@ TEST_F(CollectionManagerTest, GetAllCollections) {
ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());
// try creating one more collection
- collectionManager.create_collection("collection2", search_fields, "points");
+ collectionManager.create_collection("collection2", 4, search_fields, "points");
collection_vec = collectionManager.get_collections();
ASSERT_EQ(2, collection_vec.size());
@@ -212,7 +212,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager & collectionManager2 = CollectionManager::get_instance();
- collectionManager2.init(store, 4, 1.0, "auth_key");
+ collectionManager2.init(store, 1.0, "auth_key");
collectionManager2.load();
collection1 = collectionManager2.get_collection("collection1");
@@ -280,7 +280,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
Store *store = new Store(state_dir_path);
- cmanager.init(store, 4, 1.0, "auth_key");
+ cmanager.init(store, 1.0, "auth_key");
cmanager.load();
// try resolving on a blank slate
@@ -346,7 +346,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
// should be able to restore state on init
CollectionManager & cmanager2 = CollectionManager::get_instance();
- cmanager2.init(store, 4, 1.0, "auth_key");
+ cmanager2.init(store, 1.0, "auth_key");
cmanager2.load();
collection_option = cmanager2.resolve_symlink("company");
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index ccc9997e..86a1452b 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -18,7 +18,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
@@ -31,7 +31,7 @@ protected:
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 8cdb8104..8e101cd3 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
}
@@ -45,7 +45,7 @@ TEST_F(CollectionSortingTest, SortingOrder) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
@@ -127,7 +127,7 @@ TEST_F(CollectionSortingTest, DefaultSortingFieldValidations) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
- Option<Collection*> collection_op = collectionManager.create_collection("sample_collection", fields, "name");
+ Option<Collection*> collection_op = collectionManager.create_collection("sample_collection", 4, fields, "name");
EXPECT_FALSE(collection_op.ok());
EXPECT_EQ("Default sorting field `name` must be a single valued numerical field.", collection_op.error());
collectionManager.drop_collection("sample_collection");
@@ -135,7 +135,7 @@ TEST_F(CollectionSortingTest, DefaultSortingFieldValidations) {
// Default sorting field must exist as a field in schema
sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
- collection_op = collectionManager.create_collection("sample_collection", fields, "NOT-DEFINED");
+ collection_op = collectionManager.create_collection("sample_collection", 4, fields, "NOT-DEFINED");
EXPECT_FALSE(collection_op.ok());
EXPECT_EQ("Default sorting field is defined as `NOT-DEFINED` but is not found in the schema.", collection_op.error());
collectionManager.drop_collection("sample_collection");
@@ -152,7 +152,7 @@ TEST_F(CollectionSortingTest, Int64AsDefaultSortingField) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
auto doc_str1 = "{\"title\": \"foo\", \"starring\": \"bar\", \"points\": 343234324234233234, \"cast\": [\"baz\"] }";
@@ -211,7 +211,7 @@ TEST_F(CollectionSortingTest, SortOnFloatFields) {
coll_float_fields = collectionManager.get_collection("coll_float_fields");
if(coll_float_fields == nullptr) {
- coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "score").get();
+ coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "score").get();
}
std::string json_line;
@@ -279,7 +279,7 @@ TEST_F(CollectionSortingTest, ThreeSortFieldsLimit) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc1;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 571561c3..461da75c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
@@ -34,7 +34,7 @@ protected:
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
- collection = collectionManager.create_collection("collection", search_fields, "points").get();
+ collection = collectionManager.create_collection("collection", 4, search_fields, "points").get();
}
std::string json_line;
@@ -581,7 +581,7 @@ TEST_F(CollectionTest, MultiOccurrenceString) {
coll_multi_string = collectionManager.get_collection("coll_multi_string");
if (coll_multi_string == nullptr) {
- coll_multi_string = collectionManager.create_collection("coll_multi_string", fields, "points").get();
+ coll_multi_string = collectionManager.create_collection("coll_multi_string", 4, fields, "points").get();
}
nlohmann::json document;
@@ -609,7 +609,7 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
coll_array_text = collectionManager.get_collection("coll_array_text");
if (coll_array_text == nullptr) {
- coll_array_text = collectionManager.create_collection("coll_array_text", fields, "points").get();
+ coll_array_text = collectionManager.create_collection("coll_array_text", 4, fields, "points").get();
}
std::string json_line;
@@ -774,7 +774,7 @@ TEST_F(CollectionTest, MultipleFields) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
@@ -892,7 +892,7 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
@@ -984,12 +984,12 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
// ensure that default_sorting_field is a non-array numerical field
- auto coll_op = collectionManager.create_collection("coll_array_fields", fields, "years");
+ auto coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "years");
ASSERT_EQ(false, coll_op.ok());
ASSERT_STREQ("Default sorting field `years` must be a single valued numerical field.", coll_op.error().c_str());
// let's try again properly
- coll_op = collectionManager.create_collection("coll_array_fields", fields, "age");
+ coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "age");
coll_array_fields = coll_op.get();
}
@@ -1129,7 +1129,7 @@ TEST_F(CollectionTest, FilterOnFloatFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@@ -1284,7 +1284,7 @@ TEST_F(CollectionTest, ImportDocuments) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
// try importing records
@@ -1393,7 +1393,7 @@ TEST_F(CollectionTest, QueryBoolFields) {
coll_bool = collectionManager.get_collection("coll_bool");
if(coll_bool == nullptr) {
- coll_bool = collectionManager.create_collection("coll_bool", fields, "rating").get();
+ coll_bool = collectionManager.create_collection("coll_bool", 4, fields, "rating").get();
}
std::string json_line;
@@ -1498,7 +1498,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@@ -1582,7 +1582,7 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@@ -1638,7 +1638,7 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@@ -1693,7 +1693,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
sample_collection = collectionManager.get_collection("sample_collection");
if(sample_collection == nullptr) {
- sample_collection = collectionManager.create_collection("sample_collection", fields, "age").get();
+ sample_collection = collectionManager.create_collection("sample_collection", 4, fields, "age").get();
}
const Option<nlohmann::json> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29, \"average\": 78}");
@@ -1773,7 +1773,7 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
empty_coll = collectionManager.get_collection("empty_coll");
if(empty_coll == nullptr) {
- empty_coll = collectionManager.create_collection("empty_coll", fields, "age").get();
+ empty_coll = collectionManager.create_collection("empty_coll", 4, fields, "age").get();
}
nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
@@ -1793,7 +1793,7 @@ TEST_F(CollectionTest, IdFieldShouldBeAString) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "age").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "age").get();
}
nlohmann::json doc;
@@ -1821,7 +1821,7 @@ TEST_F(CollectionTest, AnIntegerCanBePassedToAFloatField) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "average").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "average").get();
}
nlohmann::json doc;
@@ -1849,7 +1849,7 @@ TEST_F(CollectionTest, DeletionOfADocument) {
Collection *collection_for_del;
collection_for_del = collectionManager.get_collection("collection_for_del");
if(collection_for_del == nullptr) {
- collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, "points").get();
+ collection_for_del = collectionManager.create_collection("collection_for_del", 4, search_fields, "points").get();
}
std::string json_line;
@@ -1930,7 +1930,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@@ -2001,7 +2001,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {
coll_large_text = collectionManager.get_collection("coll_large_text");
if(coll_large_text == nullptr) {
- coll_large_text = collectionManager.create_collection("coll_large_text", fields, "age").get();
+ coll_large_text = collectionManager.create_collection("coll_large_text", 4, fields, "age").get();
}
std::string json_line;
@@ -2083,7 +2083,7 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@@ -2108,7 +2108,7 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@@ -2151,7 +2151,7 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "points").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@@ -2226,7 +2226,7 @@ TEST_F(CollectionTest, OptionalFields) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, "max").get();
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "max").get();
}
std::ifstream infile(std::string(ROOT_DIR)+"test/optional_fields.jsonl");
@@ -2290,7 +2290,7 @@ TEST_F(CollectionTest, OptionalFields) {
field("score", field_types::INT32, false, true),
};
- auto create_op = collectionManager.create_collection("coll2", fields, "score");
+ auto create_op = collectionManager.create_collection("coll2", 4, fields, "score");
ASSERT_FALSE(create_op.ok());
ASSERT_STREQ("Default sorting field `score` cannot be an optional field.", create_op.error().c_str());