Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.26-facets' into v27

Kishore Nallan8ef6bcb93942
diff --cc include/collection.h
index 949f923c,4a4ff158..dad4df0c
--- a/include/collection.h
+++ b/include/collection.h
@@@ -630,7 -626,7 +629,7 @@@ public
// synonym operations
- spp::sparse_hash_map<uint32_t, synonym_t> get_synonyms();
- Option<spp::sparse_hash_map<std::string, synonym_t*>> get_synonyms(uint32_t limit=0, uint32_t offset=0);
++ Option<std::map<uint32_t, synonym_t*>> get_synonyms(uint32_t limit=0, uint32_t offset=0);
bool get_synonym(const std::string& id, synonym_t& synonym);
diff --cc include/collection_manager.h
index cc5be3f0,4400376f..5cca9c09
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@@ -103,10 -101,7 +101,9 @@@ public
static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
static constexpr const char* SYMLINK_PREFIX = "$SL";
static constexpr const char* PRESET_PREFIX = "$PS";
- static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";
+ uint16_t filter_by_max_ops;
+
static CollectionManager & get_instance() {
static CollectionManager instance;
return instance;
@@@ -147,12 -144,10 +146,12 @@@
// PUBLICLY EXPOSED API
void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,
- const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer,
- const std::string & auth_key, std::atomic<bool>& quit);
++ const std::string & auth_key, std::atomic<bool>& quit,
+ const uint16_t& filter_by_max_operations = Config::FILTER_BY_DEFAULT_OPERATIONS);
// only for tests!
- void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);
+ void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit,
+ const uint16_t& filter_by_max_operations = Config::FILTER_BY_DEFAULT_OPERATIONS);
Option<bool> load(const size_t collection_batch_size, const size_t document_batch_size);
diff --cc include/synonym_index.h
index 8c067088,76facae0..88c5a74e
--- a/include/synonym_index.h
+++ b/include/synonym_index.h
@@@ -50,10 -49,8 +50,10 @@@ private
mutable std::shared_mutex mutex;
Store* store;
- spp::sparse_hash_map<std::string, synonym_t> synonym_definitions;
- spp::sparse_hash_map<uint64_t, std::vector<std::string>> synonym_index;
+ spp::sparse_hash_map<std::string, uint32_t> synonym_ids_index_map;
+ art_tree* synonym_index_tree;
+ uint32_t synonym_index = 0;
- spp::sparse_hash_map<uint32_t, synonym_t> synonym_definitions;
++ std::map<uint32_t, synonym_t> synonym_definitions;
void synonym_reduction_internal(const std::vector<std::string>& tokens,
size_t start_window_size,
@@@ -79,10 -68,9 +79,10 @@@ public
static std::string get_synonym_key(const std::string & collection_name, const std::string & synonym_id);
void synonym_reduction(const std::vector<std::string>& tokens,
- std::vector<std::vector<std::string>>& results) const;
+ std::vector<std::vector<std::string>>& results,
+ bool synonym_prefix, uint32_t synonym_num_typos) const;
- spp::sparse_hash_map<uint32_t, synonym_t> get_synonyms();
- Option<spp::sparse_hash_map<std::string, synonym_t*>> get_synonyms(uint32_t limit=0, uint32_t offset=0);
++ Option<std::map<uint32_t, synonym_t*>> get_synonyms(uint32_t limit=0, uint32_t offset=0);
bool get_synonym(const std::string& id, synonym_t& synonym);
diff --cc src/collection.cpp
index 6c6f5ef9,3cff1c81..c8f8e1ff
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -274,10 -291,13 +292,14 @@@ Option<bool> Collection::add_reference_
auto const& item_value = item.value();
if (item_value.is_string() && ref_field_type == field_types::STRING) {
filter_query += item_value.get<std::string>();
+ filter_values_added = true;
} else if (item_value.is_number_integer() && (ref_field_type == field_types::INT64 ||
- (ref_field_type == field_types::INT32 && StringUtils::is_int32_t(std::to_string(item_value.get<int64_t>()))))) {
+ (ref_field_type == field_types::INT32 &&
+ StringUtils::is_int32_t(std::to_string(item_value.get<int64_t>()))))) {
filter_query += std::to_string(item_value.get<int64_t>());
+ filter_values_added = true;
+ } else if (optional && item_value.is_null()) {
+ continue;
} else {
return Option<bool>(400, "Field `" + field_name + "` must only have `" + ref_field_type + "` values.");
}
@@@ -4714,15 -4734,57 +4746,58 @@@ Option<bool> Collection::remove_synonym
}
void Collection::synonym_reduction(const std::vector<std::string>& tokens,
- std::vector<std::vector<std::string>>& results) const {
+ std::vector<std::vector<std::string>>& results,
+ bool synonym_prefix, uint32_t synonym_num_typos) const {
std::shared_lock lock(mutex);
- return synonym_index->synonym_reduction(tokens, results);
+ return synonym_index->synonym_reduction(tokens, results, synonym_prefix, synonym_num_typos);
}
- spp::sparse_hash_map<uint32_t, synonym_t> Collection::get_synonyms() {
+ Option<override_t> Collection::get_override(const std::string& override_id) {
+ std::shared_lock lock(mutex);
+
+ if(overrides.count(override_id) == 0) {
+ return Option<override_t>(404, "override " + override_id + " not found.");
+ }
+
+ return Option<override_t>(overrides.at(override_id));
+ }
+
+ Option<std::map<std::string, override_t*>> Collection::get_overrides(uint32_t limit, uint32_t offset) {
+ std::shared_lock lock(mutex);
+ std::map<std::string, override_t*> overrides_map;
+
+ auto overrides_it = overrides.begin();
+
+ if(offset > 0) {
+ if(offset >= overrides.size()) {
+ return Option<std::map<std::string, override_t*>>(400, "Invalid offset param.");
+ }
+
+ std::advance(overrides_it, offset);
+ }
+
+ auto overrides_end = overrides.end();
+
+ if(limit > 0 && (offset + limit < overrides.size())) {
+ overrides_end = overrides_it;
+ std::advance(overrides_end, limit);
+ }
+
+ for (overrides_it; overrides_it != overrides_end; ++overrides_it) {
+ overrides_map[overrides_it->first] = &overrides_it->second;
+ }
+
+ return Option<std::map<std::string, override_t*>>(overrides_map);
+ }
+
-Option<spp::sparse_hash_map<std::string, synonym_t*>> Collection::get_synonyms(uint32_t limit, uint32_t offset) {
++Option<std::map<uint32_t, synonym_t*>> Collection::get_synonyms(uint32_t limit, uint32_t offset) {
std::shared_lock lock(mutex);
- return synonym_index->get_synonyms();
- auto synonyms_op =synonym_index->get_synonyms(limit, offset);
++ auto synonyms_op = synonym_index->get_synonyms(limit, offset);
+ if(!synonyms_op.ok()) {
- return Option<spp::sparse_hash_map<std::string, synonym_t*>>(synonyms_op.code(), synonyms_op.error());
++ return Option<std::map<uint32_t, synonym_t*>>(synonyms_op.code(), synonyms_op.error());
+ }
+
+ return synonyms_op;
}
SynonymIndex* Collection::get_synonym_index() {
diff --cc src/collection_manager.cpp
index 5b841d88,148185ee..c6dafa6f
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -209,9 -210,7 +210,8 @@@ void CollectionManager::add_to_collecti
void CollectionManager::init(Store *store, ThreadPool* thread_pool,
const float max_memory_ratio,
const std::string & auth_key,
- std::atomic<bool>& quit) {
+ std::atomic<bool>& quit,
- BatchedIndexer* batch_indexer,
+ const uint16_t& filter_by_max_operations) {
std::unique_lock lock(mutex);
this->store = store;
@@@ -219,16 -218,44 +219,46 @@@
this->bootstrap_auth_key = auth_key;
this->max_memory_ratio = max_memory_ratio;
this->quit = &quit;
- this->batch_indexer = batch_indexer;
+ this->filter_by_max_ops = filter_by_max_operations;
}
// used only in tests!
void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,
- std::atomic<bool>& quit) {
+ std::atomic<bool>& quit,
+ const uint16_t& filter_by_max_operations) {
ThreadPool* thread_pool = new ThreadPool(8);
- init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr, filter_by_max_operations);
- init(store, thread_pool, max_memory_ratio, auth_key, quit);
++ init(store, thread_pool, max_memory_ratio, auth_key, quit, filter_by_max_operations);
+ }
+
+ void CollectionManager::_populate_referenced_ins(const std::string& collection_meta_json,
+ std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins) {
+ auto const& obj = nlohmann::json::parse(collection_meta_json, nullptr, false);
+
+ if (!obj.is_discarded() && obj.is_object() && obj.contains("name") && obj["name"].is_string() &&
+ obj.contains("fields")) {
+ auto const& collection_name = obj["name"];
+
+ for (const auto &field: obj["fields"]) {
+ if (!field.contains("name") || !field.contains("reference")) {
+ continue;
+ }
+ auto field_name = std::string(field["name"]) + fields::REFERENCE_HELPER_FIELD_SUFFIX;
+ std::vector<std::string> split_result;
+ StringUtils::split(field["reference"], split_result, ".");
+ auto ref_coll_name = split_result.front();
+
+ // Resolves alias if used in schema.
+ auto actual_ref_coll_it = CollectionManager::get_instance().collection_symlinks.find(ref_coll_name);
+ if (actual_ref_coll_it != CollectionManager::get_instance().collection_symlinks.end()) {
+ ref_coll_name = actual_ref_coll_it->second;
+ }
+ if (referenced_ins.count(ref_coll_name) == 0) {
+ referenced_ins.emplace(ref_coll_name, spp::sparse_hash_map<std::string, std::string>());
+ }
+
+ referenced_ins[ref_coll_name].emplace(collection_name, field_name);
+ }
+ }
}
Option<bool> CollectionManager::load(const size_t collection_batch_size, const size_t document_batch_size) {
diff --cc src/synonym_index.cpp
index 6e0fa204,05ff068c..0e8c70b9
--- a/src/synonym_index.cpp
+++ b/src/synonym_index.cpp
@@@ -186,9 -179,32 +186,32 @@@ Option<bool> SynonymIndex::remove_synon
return Option<bool>(404, "Could not find that `id`.");
}
- spp::sparse_hash_map<uint32_t, synonym_t> SynonymIndex::get_synonyms() {
-Option<spp::sparse_hash_map<std::string, synonym_t*>> SynonymIndex::get_synonyms(uint32_t limit, uint32_t offset) {
++Option<std::map<uint32_t, synonym_t*>> SynonymIndex::get_synonyms(uint32_t limit, uint32_t offset) {
std::shared_lock lock(mutex);
- return synonym_definitions;
- spp::sparse_hash_map<std::string, synonym_t*> synonyms_map;
++ std::map<uint32_t, synonym_t*> synonyms_map;
+
+ auto synonym_it = synonym_definitions.begin();
+
+ if(offset > 0) {
+ if(offset >= synonym_definitions.size()) {
- return Option<spp::sparse_hash_map<std::string, synonym_t*>>(400, "Invalid offset param.");
++ return Option<std::map<uint32_t, synonym_t*>>(400, "Invalid offset param.");
+ }
+
+ std::advance(synonym_it, offset);
+ }
+
+ auto synonym_end = synonym_definitions.end();
+
+ if(limit > 0 && (offset + limit < synonym_definitions.size())) {
+ synonym_end = synonym_it;
+ std::advance(synonym_end, limit);
+ }
+
+ for (synonym_it; synonym_it != synonym_end; ++synonym_it) {
+ synonyms_map[synonym_it->first] = &synonym_it->second;
+ }
+
- return Option<spp::sparse_hash_map<std::string, synonym_t*>>(synonyms_map);
++ return Option<std::map<uint32_t, synonym_t*>>(synonyms_map);
}
std::string SynonymIndex::get_synonym_key(const std::string & collection_name, const std::string & synonym_id) {
diff --cc src/tsconfig.cpp
index 08185546,51113b3c..99b56ff1
--- a/src/tsconfig.cpp
+++ b/src/tsconfig.cpp
@@@ -124,4 -124,495 +124,507 @@@ Option<std::string> Config::fetch_nodes
}
return Option<std::string>(nodes_config);
- }
+ }
+
+ void Config::load_config_env() {
+ this->data_dir = get_env("TYPESENSE_DATA_DIR");
+ this->log_dir = get_env("TYPESENSE_LOG_DIR");
+ this->analytics_dir = get_env("TYPESENSE_ANALYTICS_DIR");
+ this->api_key = get_env("TYPESENSE_API_KEY");
+
+ // @deprecated
+ this->search_only_api_key = get_env("TYPESENSE_SEARCH_ONLY_API_KEY");
+
+ this->health_rusage_api_key = get_env("TYPESENSE_HEALTH_RUSAGE_API_KEY");
+
+ if(!get_env("TYPESENSE_LISTEN_ADDRESS").empty()) {
+ this->api_address = get_env("TYPESENSE_LISTEN_ADDRESS");
+ }
+
+ if(!get_env("TYPESENSE_LISTEN_PORT").empty()) {
+ this->api_port = std::stoi(get_env("TYPESENSE_LISTEN_PORT"));
+ }
+
+ if(!get_env("TYPESENSE_API_ADDRESS").empty()) {
+ this->api_address = get_env("TYPESENSE_API_ADDRESS");
+ }
+
+ if(!get_env("TYPESENSE_API_PORT").empty()) {
+ this->api_port = std::stoi(get_env("TYPESENSE_API_PORT"));
+ }
+
+ if(!get_env("TYPESENSE_PEERING_ADDRESS").empty()) {
+ this->peering_address = get_env("TYPESENSE_PEERING_ADDRESS");
+ }
+
+ if(!get_env("TYPESENSE_PEERING_PORT").empty()) {
+ this->peering_port = std::stoi(get_env("TYPESENSE_PEERING_PORT"));
+ }
+
+ if(!get_env("TYPESENSE_PEERING_SUBNET").empty()) {
+ this->peering_subnet = get_env("TYPESENSE_PEERING_SUBNET");
+ }
+
+ this->nodes = get_env("TYPESENSE_NODES");
+
+ this->master = get_env("TYPESENSE_MASTER");
+ this->ssl_certificate = get_env("TYPESENSE_SSL_CERTIFICATE");
+ this->ssl_certificate_key = get_env("TYPESENSE_SSL_CERTIFICATE_KEY");
+
+ std::string enable_cors_str = get_env("TYPESENSE_ENABLE_CORS");
+ StringUtils::toupper(enable_cors_str);
+ this->enable_cors = ("TRUE" == enable_cors_str || enable_cors_str.empty()) ? true : false;
+
+ std::string cors_domains_value = get_env("TYPESENSE_CORS_DOMAINS");
+ set_cors_domains(cors_domains_value);
+
+ if(!get_env("TYPESENSE_MAX_MEMORY_RATIO").empty()) {
+ this->max_memory_ratio = std::stof(get_env("TYPESENSE_MAX_MEMORY_RATIO"));
+ }
+
+ if(!get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS").empty()) {
+ this->snapshot_interval_seconds = std::stoi(get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS"));
+ }
+
+ if(!get_env("TYPESENSE_HEALTHY_READ_LAG").empty()) {
+ this->healthy_read_lag = std::stoi(get_env("TYPESENSE_HEALTHY_READ_LAG"));
+ }
+
+ if(!get_env("TYPESENSE_HEALTHY_WRITE_LAG").empty()) {
+ this->healthy_write_lag = std::stoi(get_env("TYPESENSE_HEALTHY_WRITE_LAG"));
+ }
+
+ if(!get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS").empty()) {
+ this->log_slow_requests_time_ms = std::stoi(get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS"));
+ }
+
+ if(!get_env("TYPESENSE_LOG_SLOW_SEARCHES_TIME_MS").empty()) {
+ this->log_slow_searches_time_ms = std::stoi(get_env("TYPESENSE_LOG_SLOW_SEARCHES_TIME_MS"));
+ }
+
+ if(!get_env("TYPESENSE_NUM_COLLECTIONS_PARALLEL_LOAD").empty()) {
+ this->num_collections_parallel_load = std::stoi(get_env("TYPESENSE_NUM_COLLECTIONS_PARALLEL_LOAD"));
+ }
+
+ if(!get_env("TYPESENSE_NUM_DOCUMENTS_PARALLEL_LOAD").empty()) {
+ this->num_documents_parallel_load = std::stoi(get_env("TYPESENSE_NUM_DOCUMENTS_PARALLEL_LOAD"));
+ }
+
+ if(!get_env("TYPESENSE_CACHE_NUM_ENTRIES").empty()) {
+ this->cache_num_entries = std::stoi(get_env("TYPESENSE_CACHE_NUM_ENTRIES"));
+ }
+
+ if(!get_env("TYPESENSE_ANALYTICS_FLUSH_INTERVAL").empty()) {
+ this->analytics_flush_interval = std::stoi(get_env("TYPESENSE_ANALYTICS_FLUSH_INTERVAL"));
+ }
+
+ if(!get_env("TYPESENSE_HOUSEKEEPING_INTERVAL").empty()) {
+ this->housekeeping_interval = std::stoi(get_env("TYPESENSE_HOUSEKEEPING_INTERVAL"));
+ }
+
+ if(!get_env("TYPESENSE_DB_COMPACTION_INTERVAL").empty()) {
+ this->db_compaction_interval = std::stoi(get_env("TYPESENSE_DB_COMPACTION_INTERVAL"));
+ }
+
+ if(!get_env("TYPESENSE_THREAD_POOL_SIZE").empty()) {
+ this->thread_pool_size = std::stoi(get_env("TYPESENSE_THREAD_POOL_SIZE"));
+ }
+
+ if(!get_env("TYPESENSE_SSL_REFRESH_INTERVAL_SECONDS").empty()) {
+ this->ssl_refresh_interval_seconds = std::stoi(get_env("TYPESENSE_SSL_REFRESH_INTERVAL_SECONDS"));
+ }
+
+ if(!get_env("TYPESENSE_SNAPSHOT_MAX_BYTE_COUNT_PER_RPC").empty()) {
+ this->snapshot_max_byte_count_per_rpc = std::stoi(get_env("TYPESENSE_SNAPSHOT_MAX_BYTE_COUNT_PER_RPC"));
+ }
+
+ this->enable_access_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_ACCESS_LOGGING"));
+ this->enable_search_analytics = ("TRUE" == get_env("TYPESENSE_ENABLE_SEARCH_ANALYTICS"));
+ this->enable_search_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_SEARCH_LOGGING"));
+
+ if(!get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE").empty()) {
+ this->disk_used_max_percentage = std::stoi(get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE"));
+ }
+
+ if(!get_env("TYPESENSE_MEMORY_USED_MAX_PERCENTAGE").empty()) {
+ this->memory_used_max_percentage = std::stoi(get_env("TYPESENSE_MEMORY_USED_MAX_PERCENTAGE"));
+ }
+
++ if(!get_env("TYPESENSE_FILTER_BY_MAX_OPS").empty()) {
++ this->filter_by_max_ops = std::stoi(get_env("TYPESENSE_FILTER_BY_MAX_OPS"));
++ }
++
+ this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));
+ this->reset_peers_on_error = ("TRUE" == get_env("TYPESENSE_RESET_PEERS_ON_ERROR"));
+ }
+
+ void Config::load_config_file(cmdline::parser& options) {
+ this->config_file = options.exist("config") ? options.get<std::string>("config") : "";
+
+ if(!options.exist("config")) {
+ config_file_validity = 0;
+ return;
+ }
+
+ this->config_file = options.get<std::string>("config");
+
+ INIReader reader(this->config_file);
+
+ if (reader.ParseError() != 0) {
+ LOG(ERROR) << "Error while parsing config file, code = " << reader.ParseError();
+ config_file_validity = -1;
+ return ;
+ }
+
+ config_file_validity = 1;
+
+ if(reader.Exists("server", "data-dir")) {
+ this->data_dir = reader.Get("server", "data-dir", "");
+ }
+
+ if(reader.Exists("server", "log-dir")) {
+ this->log_dir = reader.Get("server", "log-dir", "");
+ }
+
+ if(reader.Exists("server", "analytics-dir")) {
+ this->analytics_dir = reader.Get("server", "analytics-dir", "");
+ }
+
+ if(reader.Exists("server", "api-key")) {
+ this->api_key = reader.Get("server", "api-key", "");
+ }
+
+ // @deprecated
+ if(reader.Exists("server", "search-only-api-key")) {
+ this->search_only_api_key = reader.Get("server", "search-only-api-key", "");
+ }
+
+ if(reader.Exists("server", "health-rusage-api-key")) {
+ this->health_rusage_api_key = reader.Get("server", "health-rusage-api-key", "");
+ }
+
+ if(reader.Exists("server", "listen-address")) {
+ this->api_address = reader.Get("server", "listen-address", "");
+ }
+
+ if(reader.Exists("server", "api-address")) {
+ this->api_address = reader.Get("server", "api-address", "");
+ }
+
+ if(reader.Exists("server", "master")) {
+ this->master = reader.Get("server", "master", "");
+ }
+
+ if(reader.Exists("server", "ssl-certificate")) {
+ this->ssl_certificate = reader.Get("server", "ssl-certificate", "");
+ }
+
+ if(reader.Exists("server", "ssl-certificate-key")) {
+ this->ssl_certificate_key = reader.Get("server", "ssl-certificate-key", "");
+ }
+
+ if(reader.Exists("server", "listen-port")) {
+ this->api_port = reader.GetInteger("server", "listen-port", 8108);
+ }
+
+ if(reader.Exists("server", "api-port")) {
+ this->api_port = reader.GetInteger("server", "api-port", 8108);
+ }
+
+ if(reader.Exists("server", "enable-cors")) {
+ auto enable_cors_value = reader.Get("server", "enable-cors", "true");
+ StringUtils::tolowercase(enable_cors_value);
+ this->enable_cors = enable_cors_value == "true";
+ }
+
+ if(reader.Exists("server", "cors-domains")) {
+ std::string cors_value = reader.Get("server", "cors-domains", "");
+ set_cors_domains(cors_value);
+ }
+
+ if(reader.Exists("server", "peering-address")) {
+ this->peering_address = reader.Get("server", "peering-address", "");
+ }
+
+ if(reader.Exists("server", "peering-port")) {
+ this->peering_port = reader.GetInteger("server", "peering-port", 8107);
+ }
+
+ if(reader.Exists("server", "peering-subnet")) {
+ this->peering_subnet = reader.Get("server", "peering-subnet", "");
+ }
+
+ if(reader.Exists("server", "nodes")) {
+ this->nodes = reader.Get("server", "nodes", "");
+ }
+
+ if(reader.Exists("server", "max-memory-ratio")) {
+ this->max_memory_ratio = (float) reader.GetReal("server", "max-memory-ratio", 1.0f);
+ }
+
+ if(reader.Exists("server", "snapshot-interval-seconds")) {
+ this->snapshot_interval_seconds = (int) reader.GetInteger("server", "snapshot-interval-seconds", 3600);
+ }
+
+ if(reader.Exists("server", "snapshot-max-byte-count-per-rpc")) {
+ this->snapshot_max_byte_count_per_rpc = (int) reader.GetInteger("server", "snapshot-max-byte-count-per-rpc", 4194304);
+ }
+
+ if(reader.Exists("server", "healthy-read-lag")) {
+ this->healthy_read_lag = (size_t) reader.GetInteger("server", "healthy-read-lag", 1000);
+ }
+
+ if(reader.Exists("server", "healthy-write-lag")) {
+ this->healthy_write_lag = (size_t) reader.GetInteger("server", "healthy-write-lag", 100);
+ }
+
+ if(reader.Exists("server", "log-slow-requests-time-ms")) {
+ this->log_slow_requests_time_ms = (int) reader.GetInteger("server", "log-slow-requests-time-ms", -1);
+ }
+
+ if(reader.Exists("server", "log-slow-searches-time-ms")) {
+ this->log_slow_searches_time_ms = (int) reader.GetInteger("server", "log-slow-searches-time-ms", 30*1000);
+ }
+
+ if(reader.Exists("server", "num-collections-parallel-load")) {
+ this->num_collections_parallel_load = (int) reader.GetInteger("server", "num-collections-parallel-load", 0);
+ }
+
+ if(reader.Exists("server", "num-documents-parallel-load")) {
+ this->num_documents_parallel_load = (int) reader.GetInteger("server", "num-documents-parallel-load", 1000);
+ }
+
+ if(reader.Exists("server", "cache-num-entries")) {
+ this->cache_num_entries = (int) reader.GetInteger("server", "cache-num-entries", 1000);
+ }
+
+ if(reader.Exists("server", "analytics-flush-interval")) {
+ this->analytics_flush_interval = (int) reader.GetInteger("server", "analytics-flush-interval", 3600);
+ }
+
+ if(reader.Exists("server", "housekeeping-interval")) {
+ this->housekeeping_interval = (int) reader.GetInteger("server", "housekeeping-interval", 1800);
+ }
+
+ if(reader.Exists("server", "db-compaction-interval")) {
+ this->db_compaction_interval = (int) reader.GetInteger("server", "db-compaction-interval", 0);
+ }
+
+ if(reader.Exists("server", "thread-pool-size")) {
+ this->thread_pool_size = (int) reader.GetInteger("server", "thread-pool-size", 0);
+ }
+
+ if(reader.Exists("server", "ssl-refresh-interval-seconds")) {
+ this->ssl_refresh_interval_seconds = (int) reader.GetInteger("server", "ssl-refresh-interval-seconds", 8 * 60 * 60);
+ }
+
+ if(reader.Exists("server", "enable-access-logging")) {
+ auto enable_access_logging_str = reader.Get("server", "enable-access-logging", "false");
+ this->enable_access_logging = (enable_access_logging_str == "true");
+ }
+
+ if(reader.Exists("server", "enable-search-analytics")) {
+ auto enable_search_analytics_str = reader.Get("server", "enable-search-analytics", "false");
+ this->enable_search_analytics = (enable_search_analytics_str == "true");
+ }
+
+ if(reader.Exists("server", "enable-search-logging")) {
+ auto enable_search_logging_str = reader.Get("server", "enable-search-logging", "false");
+ this->enable_search_logging = (enable_search_logging_str == "true");
+ }
+
+ if(reader.Exists("server", "disk-used-max-percentage")) {
+ this->disk_used_max_percentage = (int) reader.GetInteger("server", "disk-used-max-percentage", 100);
+ }
+
+ if(reader.Exists("server", "memory-used-max-percentage")) {
+ this->memory_used_max_percentage = (int) reader.GetInteger("server", "memory-used-max-percentage", 100);
+ }
+
+ if(reader.Exists("server", "skip-writes")) {
+ auto skip_writes_str = reader.Get("server", "skip-writes", "false");
+ this->skip_writes = (skip_writes_str == "true");
+ }
+
+ if(reader.Exists("server", "reset-peers-on-error")) {
+ auto reset_peers_on_error_str = reader.Get("server", "reset-peers-on-error", "false");
+ this->reset_peers_on_error = (reset_peers_on_error_str == "true");
+ }
++
++ if(reader.Exists("server", "filter-by-max-ops")) {
++ this->filter_by_max_ops = (uint16_t) reader.GetInteger("server", "filter-by-max-ops", FILTER_BY_DEFAULT_OPERATIONS);
++ }
+ }
+
+ void Config::load_config_cmd_args(cmdline::parser& options) {
+ if(options.exist("data-dir")) {
+ this->data_dir = options.get<std::string>("data-dir");
+ }
+
+ if(options.exist("log-dir")) {
+ this->log_dir = options.get<std::string>("log-dir");
+ }
+
+ if(options.exist("analytics-dir")) {
+ this->analytics_dir = options.get<std::string>("analytics-dir");
+ }
+
+ if(options.exist("api-key")) {
+ this->api_key = options.get<std::string>("api-key");
+ }
+
+ // @deprecated
+ if(options.exist("search-only-api-key")) {
+ this->search_only_api_key = options.get<std::string>("search-only-api-key");
+ }
+
+ if(options.exist("health-rusage-api-key")) {
+ this->health_rusage_api_key = options.get<std::string>("health-rusage-api-key");
+ }
+
+ if(options.exist("listen-address")) {
+ this->api_address = options.get<std::string>("listen-address");
+ }
+
+ if(options.exist("api-address")) {
+ this->api_address = options.get<std::string>("api-address");
+ }
+
+ if(options.exist("master")) {
+ this->master = options.get<std::string>("master");
+ }
+
+ if(options.exist("ssl-certificate")) {
+ this->ssl_certificate = options.get<std::string>("ssl-certificate");
+ }
+
+ if(options.exist("ssl-certificate-key")) {
+ this->ssl_certificate_key = options.get<std::string>("ssl-certificate-key");
+ }
+
+ if(options.exist("listen-port")) {
+ this->api_port = options.get<uint32_t>("listen-port");
+ }
+
+ if(options.exist("api-port")) {
+ this->api_port = options.get<uint32_t>("api-port");
+ }
+
+ if(options.exist("enable-cors")) {
+ this->enable_cors = options.get<bool>("enable-cors");
+ }
+
+ if(options.exist("cors-domains")) {
+ std::string cors_domains_value = options.get<std::string>("cors-domains");
+ set_cors_domains(cors_domains_value);
+ }
+
+ if(options.exist("peering-address")) {
+ this->peering_address = options.get<std::string>("peering-address");
+ }
+
+ if(options.exist("peering-port")) {
+ this->peering_port = options.get<uint32_t>("peering-port");
+ }
+
+ if(options.exist("peering-subnet")) {
+ this->peering_subnet = options.get<std::string>("peering-subnet");
+ }
+
+ if(options.exist("nodes")) {
+ this->nodes = options.get<std::string>("nodes");
+ }
+
+ if(options.exist("max-memory-ratio")) {
+ this->max_memory_ratio = options.get<float>("max-memory-ratio");
+ }
+
+ if(options.exist("snapshot-interval-seconds")) {
+ this->snapshot_interval_seconds = options.get<int>("snapshot-interval-seconds");
+ }
+
+ if(options.exist("snapshot-max-byte-count-per-rpc")) {
+ this->snapshot_max_byte_count_per_rpc = options.get<int>("snapshot-max-byte-count-per-rpc");
+ }
+
+ if(options.exist("healthy-read-lag")) {
+ this->healthy_read_lag = options.get<size_t>("healthy-read-lag");
+ }
+
+ if(options.exist("healthy-write-lag")) {
+ this->healthy_write_lag = options.get<size_t>("healthy-write-lag");
+ }
+
+ if(options.exist("log-slow-requests-time-ms")) {
+ this->log_slow_requests_time_ms = options.get<int>("log-slow-requests-time-ms");
+ }
+
+ if(options.exist("log-slow-searches-time-ms")) {
+ this->log_slow_searches_time_ms = options.get<int>("log-slow-searches-time-ms");
+ }
+
+ if(options.exist("num-collections-parallel-load")) {
+ this->num_collections_parallel_load = options.get<uint32_t>("num-collections-parallel-load");
+ }
+
+ if(options.exist("num-documents-parallel-load")) {
+ this->num_documents_parallel_load = options.get<uint32_t>("num-documents-parallel-load");
+ }
+
+ if(options.exist("cache-num-entries")) {
+ this->cache_num_entries = options.get<uint32_t>("cache-num-entries");
+ }
+
+ if(options.exist("analytics-flush-interval")) {
+ this->analytics_flush_interval = options.get<uint32_t>("analytics-flush-interval");
+ }
+
+ if(options.exist("housekeeping-interval")) {
+ this->housekeeping_interval = options.get<uint32_t>("housekeeping-interval");
+ }
+
+ if(options.exist("db-compaction-interval")) {
+ this->db_compaction_interval = options.get<uint32_t>("db-compaction-interval");
+ }
+
+ if(options.exist("thread-pool-size")) {
+ this->thread_pool_size = options.get<uint32_t>("thread-pool-size");
+ }
+
+ if(options.exist("ssl-refresh-interval-seconds")) {
+ this->ssl_refresh_interval_seconds = options.get<uint32_t>("ssl-refresh-interval-seconds");
+ }
+
+ if(options.exist("enable-access-logging")) {
+ this->enable_access_logging = options.get<bool>("enable-access-logging");
+ }
+
+ if(options.exist("disk-used-max-percentage")) {
+ this->disk_used_max_percentage = options.get<int>("disk-used-max-percentage");
+ }
+
+ if(options.exist("memory-used-max-percentage")) {
+ this->memory_used_max_percentage = options.get<int>("memory-used-max-percentage");
+ }
+
+ if(options.exist("skip-writes")) {
+ this->skip_writes = options.get<bool>("skip-writes");
+ }
+
+ if(options.exist("reset-peers-on-error")) {
+ this->reset_peers_on_error = options.get<bool>("reset-peers-on-error");
+ }
+
+ if(options.exist("enable-search-analytics")) {
+ this->enable_search_analytics = options.get<bool>("enable-search-analytics");
+ }
+
+ if(options.exist("enable-search-logging")) {
+ this->enable_search_logging = options.get<bool>("enable-search-logging");
+ }
++
++ if(options.exist("filter-by-max-ops")) {
++ this->filter_by_max_ops = options.get<uint16_t>("filter-by-max-ops");
++ }
+ }
+
diff --cc src/typesense_server_utils.cpp
index 2cf22675,37459cb5..99f1c20c
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@@ -437,7 -436,7 +437,7 @@@ int run_server(const Config & config, c
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),
- config.get_api_key(), quit_raft_service, batch_indexer, config.get_filter_by_max_ops());
- config.get_api_key(), quit_raft_service);
++ config.get_api_key(), quit_raft_service, config.get_filter_by_max_ops());
StopwordsManager& stopwordsManager = StopwordsManager::get_instance();
stopwordsManager.init(&store);
diff --cc test/collection_manager_test.cpp
index 4cd77d39,5c1f5c51..0fff3940
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@@ -520,20 -521,20 +521,20 @@@ TEST_F(CollectionManagerTest, RestoreRe
ASSERT_TRUE(collection1->get_enable_nested_fields());
- ASSERT_EQ(2, collection1->get_overrides().size());
- ASSERT_STREQ("exclude-rule", collection1->get_overrides()["exclude-rule"].id.c_str());
- ASSERT_STREQ("include-rule", collection1->get_overrides()["include-rule"].id.c_str());
+ ASSERT_EQ(2, collection1->get_overrides().get().size());
+ ASSERT_STREQ("exclude-rule", collection1->get_overrides().get()["exclude-rule"]->id.c_str());
+ ASSERT_STREQ("include-rule", collection1->get_overrides().get()["include-rule"]->id.c_str());
- const auto& synonyms = collection1->get_synonyms();
+ const auto& synonyms = collection1->get_synonyms().get();
ASSERT_EQ(2, synonyms.size());
- ASSERT_STREQ("id1", synonyms.at(0).id.c_str());
- ASSERT_EQ(2, synonyms.at(0).root.size());
- ASSERT_EQ(1, synonyms.at(0).synonyms.size());
- ASSERT_STREQ("id1", synonyms.at("id1")->id.c_str());
- ASSERT_EQ(2, synonyms.at("id1")->root.size());
- ASSERT_EQ(1, synonyms.at("id1")->synonyms.size());
++ ASSERT_STREQ("id1", synonyms.at(0)->id.c_str());
++ ASSERT_EQ(2, synonyms.at(0)->root.size());
++ ASSERT_EQ(1, synonyms.at(0)->synonyms.size());
- ASSERT_STREQ("id3", synonyms.at(1).id.c_str());
- ASSERT_EQ(0, synonyms.at(1).root.size());
- ASSERT_EQ(2, synonyms.at(1).synonyms.size());
- ASSERT_STREQ("id3", synonyms.at("id3")->id.c_str());
- ASSERT_EQ(0, synonyms.at("id3")->root.size());
- ASSERT_EQ(2, synonyms.at("id3")->synonyms.size());
++ ASSERT_STREQ("id3", synonyms.at(1)->id.c_str());
++ ASSERT_EQ(0, synonyms.at(1)->root.size());
++ ASSERT_EQ(2, synonyms.at(1)->synonyms.size());
std::vector<char> expected_symbols = {'+'};
std::vector<char> expected_separators = {'-'};
diff --cc test/collection_synonyms_test.cpp
index 130d77d5,449245de..9d9eebac
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@@ -700,12 -700,11 +700,12 @@@ TEST_F(CollectionSynonymsTest, DeleteAn
coll_mul_fields->add_synonym(R"({"id": "ipod-synonyms", "root": "ipod", "synonyms": ["i pod", "ipod"]})"_json);
coll_mul_fields->add_synonym(R"({"id": "samsung-synonyms", "root": "s3", "synonyms": ["s3 phone", "samsung"]})"_json);
- ASSERT_EQ(2, coll_mul_fields->get_synonyms().size());
+ ASSERT_EQ(2, coll_mul_fields->get_synonyms().get().size());
coll_mul_fields->remove_synonym("ipod-synonyms");
- auto synonyms = coll_mul_fields->get_synonyms();
- ASSERT_EQ(1, coll_mul_fields->get_synonyms().get().size());
- ASSERT_STREQ("samsung-synonyms", coll_mul_fields->get_synonyms().get()["samsung-synonyms"]->id.c_str());
++ auto synonyms = coll_mul_fields->get_synonyms().get();
+ ASSERT_EQ(1, synonyms.size());
- ASSERT_STREQ("samsung-synonyms", synonyms.begin()->second.id.c_str());
++ ASSERT_EQ("samsung-synonyms", synonyms.begin()->second->id);
// try to upsert synonym with same ID
@@@ -1069,255 -1068,95 +1069,348 @@@ TEST_F(CollectionSynonymsTest, Multiple
ASSERT_EQ(1, res["hits"].size());
}
+TEST_F(CollectionSynonymsTest, EnableSynonymFlag) {
+ nlohmann::json schema = R"({
+ "name": "coll2",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "gender", "type": "string"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll2 = op.get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"Beautiful Blazer", "Male"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["gender"] = records[i][1];
+
+ auto add_op = coll2->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ nlohmann::json synonym1 = R"({
+ "id": "foobar",
+ "synonyms": ["blazer", "suit"]
+ })"_json;
+
+ nlohmann::json synonym2 = R"({
+ "id": "foobar2",
+ "synonyms": ["male", "man"]
+ })"_json;
+
+
+ ASSERT_TRUE(coll2->add_synonym(synonym1).ok());
+ ASSERT_TRUE(coll2->add_synonym(synonym2).ok());
+ bool enable_synonyms = true;
+
+ auto res = coll2->search("suit man", {"title", "gender"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, enable_synonyms).get();
+
+ ASSERT_EQ(1, res["hits"].size());
+
+ enable_synonyms = false;
+ res = coll2->search("suit man", {"title", "gender"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, enable_synonyms).get();
+
+ ASSERT_EQ(0, res["hits"].size());
+}
+
+TEST_F(CollectionSynonymsTest, SynonymTypos) {
+ nlohmann::json schema = R"({
+ "name": "coll3",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll3 = op.get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "Cool Trousers";
+
+ auto add_op = coll3->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json synonym1 = R"({
+ "id": "foobar",
+ "synonyms": ["trousers", "pants"]
+ })"_json;
+
+ ASSERT_TRUE(coll3->add_synonym(synonym1).ok());
+
+ auto res = coll3->search("trousers", {"title"}, "", {},
+ {}, {0}, 10, 1, FREQUENCY, {true},0).get();
+ ASSERT_EQ(1, res["hits"].size());
+
+ res = coll3->search("pants", {"title"}, "", {},
+ {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(1, res["hits"].size());
+
+ //try with typos
+ uint32_t synonym_num_typos = 0;
+ res = coll3->search("patns", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, false, synonym_num_typos).get();
+ ASSERT_EQ(0, res["hits"].size());
+
+ synonym_num_typos = 2;
+
+ res = coll3->search("patns", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, false, synonym_num_typos).get();
+ ASSERT_EQ(1, res["hits"].size());
+
+ //max 2 typos supported
+ synonym_num_typos = 3;
+ auto search_op = coll3->search("trosuers", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, false, synonym_num_typos);
+
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Value of `synonym_num_typos` must not be greater than 2.",search_op.error());
+}
+
+TEST_F(CollectionSynonymsTest, SynonymPrefix) {
+ nlohmann::json schema = R"({
+ "name": "coll3",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll3 = op.get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "Cool Trousers";
+ auto add_op = coll3->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ doc["id"] = "1";
+ doc["title"] = "Cool Pants";
+ add_op = coll3->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json synonym1 = R"({
+ "id": "foobar",
+ "synonyms": ["trousers", "pants"]
+ })"_json;
+
+ ASSERT_TRUE(coll3->add_synonym(synonym1).ok());
+
+ bool synonym_prefix = false;
+
+ auto res = coll3->search("pan", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {false},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, synonym_prefix).get();
+
+ ASSERT_EQ(0, res["hits"].size());
+
+ synonym_prefix = true;
+
+ res = coll3->search("pan", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {false},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, synonym_prefix).get();
+
+ ASSERT_EQ(2, res["hits"].size());
- }
++}
++
+ TEST_F(CollectionSynonymsTest, SynonymsPagination) {
+ Collection *coll3;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false)};
+
+ coll3 = collectionManager.get_collection("coll3").get();
+ if (coll3 == nullptr) {
+ coll3 = collectionManager.create_collection("coll3", 1, fields, "points").get();
+ }
+
+ for (int i = 0; i < 5; ++i) {
+ nlohmann::json synonym_json = R"(
+ {
+ "id": "foobar",
+ "synonyms": ["blazer", "suit"]
+ })"_json;
+
+ synonym_json["id"] = synonym_json["id"].get<std::string>() + std::to_string(i + 1);
+
+ coll3->add_synonym(synonym_json);
+ }
+
+ uint32_t limit = 0, offset = 0;
+
+ //limit collections by 2
+ limit = 2;
+ auto synonym_op = coll3->get_synonyms(limit);
+ auto synonym_map = synonym_op.get();
+ auto it = synonym_map.begin();
+ ASSERT_EQ(2, synonym_map.size());
- ASSERT_EQ("foobar4", it->second->id); it++;
- ASSERT_EQ("foobar5", it->second->id);
++ ASSERT_EQ("foobar1", it->second->id); it++;
++ ASSERT_EQ("foobar2", it->second->id);
+
+ //get 2 collection from offset 3
+ offset = 3;
+ synonym_op = coll3->get_synonyms(limit, offset);
+ synonym_map = synonym_op.get();
+ it = synonym_map.begin();
+ ASSERT_EQ(2, synonym_map.size());
- ASSERT_EQ("foobar3", it->second->id); it++;
- ASSERT_EQ("foobar2", it->second->id);
++ ASSERT_EQ("foobar4", it->second->id); it++;
++ ASSERT_EQ("foobar5", it->second->id);
+
+ //get all collection except first
+ offset = 1;
+ limit = 0;
+ synonym_op = coll3->get_synonyms(limit, offset);
+ synonym_map = synonym_op.get();
+ it = synonym_map.begin();
+ ASSERT_EQ(4, synonym_map.size());
- ASSERT_EQ("foobar5", it->second->id); it++;
- ASSERT_EQ("foobar1", it->second->id); it++;
- ASSERT_EQ("foobar3", it->second->id); it++;
+ ASSERT_EQ("foobar2", it->second->id); it++;
++ ASSERT_EQ("foobar3", it->second->id); it++;
++ ASSERT_EQ("foobar4", it->second->id); it++;
++ ASSERT_EQ("foobar5", it->second->id); it++;
+
+ //get last collection
+ offset = 4, limit = 1;
+ synonym_op = coll3->get_synonyms(limit, offset);
+ synonym_map = synonym_op.get();
+ it = synonym_map.begin();
+ ASSERT_EQ(1, synonym_map.size());
- ASSERT_EQ("foobar2", it->second->id);
++ ASSERT_EQ("foobar5", it->second->id);
+
+ //if limit is greater than number of collection then return all from offset
+ offset = 0;
+ limit = 8;
+ synonym_op = coll3->get_synonyms(limit, offset);
+ synonym_map = synonym_op.get();
+ it = synonym_map.begin();
+ ASSERT_EQ(5, synonym_map.size());
- ASSERT_EQ("foobar4", it->second->id); it++;
- ASSERT_EQ("foobar5", it->second->id); it++;
+ ASSERT_EQ("foobar1", it->second->id); it++;
- ASSERT_EQ("foobar3", it->second->id); it++;
+ ASSERT_EQ("foobar2", it->second->id); it++;
++ ASSERT_EQ("foobar3", it->second->id); it++;
++ ASSERT_EQ("foobar4", it->second->id); it++;
++ ASSERT_EQ("foobar5", it->second->id); it++;
+
+ offset = 3;
+ limit = 4;
+ synonym_op = coll3->get_synonyms(limit, offset);
+ synonym_map = synonym_op.get();
+ it = synonym_map.begin();
+ ASSERT_EQ(2, synonym_map.size());
- ASSERT_EQ("foobar3", it->second->id); it++;
- ASSERT_EQ("foobar2", it->second->id);
++ ASSERT_EQ("foobar4", it->second->id); it++;
++ ASSERT_EQ("foobar5", it->second->id);
+
+ //invalid offset
+ offset = 6;
+ limit = 0;
+ synonym_op = coll3->get_synonyms(limit, offset);
+ ASSERT_FALSE(synonym_op.ok());
+ ASSERT_EQ("Invalid offset param.", synonym_op.error());
-}
++}
diff --cc test/core_api_utils_test.cpp
index 446f16b0,87f884e2..3a38bd16
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@@ -1555,4 -1555,207 +1555,207 @@@ TEST_F(CoreAPIUtilsTest, DeleteNonExist
req->params["ignore_not_found"] = "true";
del_remove_document(req, res);
ASSERT_EQ(200, res->status_code);
+ }
+
+ TEST_F(CoreAPIUtilsTest, CollectionsPagination) {
+ //remove all collections first
+ auto collections = collectionManager.get_collections().get();
+ for(auto collection : collections) {
+ collectionManager.drop_collection(collection->get_name());
+ }
+
+ //create few collections
+ for(size_t i = 0; i < 5; i++) {
+ nlohmann::json coll_json = R"({
+ "name": "cp",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+ coll_json["name"] = coll_json["name"].get<std::string>() + std::to_string(i + 1);
+ auto coll_op = collectionManager.create_collection(coll_json);
+ ASSERT_TRUE(coll_op.ok());
+ }
+
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
+
+ req->params["offset"] = "0";
+ req->params["limit"] = "1";
+
+ nlohmann::json expected_meta_json = R"(
+ {
+ "created_at":1663234047,
+ "default_sorting_field":"",
+ "enable_nested_fields":false,
+ "fields":[
+ {
+ "facet":false,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"title",
+ "optional":false,
+ "sort":false,
+ "stem":false,
+ "type":"string"
+ }
+ ],
+ "name":"cp2",
+ "num_documents":0,
+ "symbols_to_index":[],
+ "token_separators":[]
+ }
+ )"_json;
+
+ get_collections(req, resp);
+
+ auto actual_json = nlohmann::json::parse(resp->body);
+ expected_meta_json["created_at"] = actual_json[0]["created_at"];
+
+ ASSERT_EQ(expected_meta_json.dump(), actual_json[0].dump());
+
+ //invalid offset string
+ req->params["offset"] = "0a";
+ get_collections(req, resp);
+
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Offset param should be unsigned integer.\"}", resp->body);
+
+ //invalid limit string
+ req->params["offset"] = "0";
+ req->params["limit"] = "-1";
+ get_collections(req, resp);
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body);
+ }
+
+ TEST_F(CoreAPIUtilsTest, OverridesPagination) {
+ Collection *coll2;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false)};
+
+ coll2 = collectionManager.get_collection("coll2").get();
+ if(coll2 == nullptr) {
+ coll2 = collectionManager.create_collection("coll2", 1, fields, "points").get();
+ }
+
+ for(int i = 0; i < 5; ++i) {
+ nlohmann::json override_json = {
+ {"id", "override"},
+ {
+ "rule", {
+ {"query", "not-found"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"metadata", { {"foo", "bar"}}},
+ };
+
+ override_json["id"] = override_json["id"].get<std::string>() + std::to_string(i + 1);
+ override_t override;
+ override_t::parse(override_json, "", override);
+
+ coll2->add_override(override);
+ }
+
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
+
+ req->params["collection"] = "coll2";
+ req->params["offset"] = "0";
+ req->params["limit"] = "1";
+
+ get_overrides(req, resp);
+ nlohmann::json expected_json = R"({
+ "overrides":[
+ {
+ "excludes":[],
+ "filter_curated_hits":false,
+ "id":"override1",
+ "includes":[],
+ "metadata":{"foo":"bar"},
+ "remove_matched_tokens":false,
+ "rule":{
+ "match":"exact",
+ "query":"not-found"
+ },
+ "stop_processing":true
+ }]
+ })"_json;
+
+ ASSERT_EQ(expected_json.dump(), resp->body);
+
+ //invalid offset string
+ req->params["offset"] = "0a";
+ get_collections(req, resp);
+
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Offset param should be unsigned integer.\"}", resp->body);
+
+ //invalid limit string
+ req->params["offset"] = "0";
+ req->params["limit"] = "-1";
+ get_collections(req, resp);
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body);
+ }
+
+ TEST_F(CoreAPIUtilsTest, SynonymsPagination) {
+ Collection *coll3;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false)};
+
+ coll3 = collectionManager.get_collection("coll3").get();
+ if (coll3 == nullptr) {
+ coll3 = collectionManager.create_collection("coll3", 1, fields, "points").get();
+ }
+
+ for (int i = 0; i < 5; ++i) {
+ nlohmann::json synonym_json = R"(
+ {
+ "id": "foobar",
+ "synonyms": ["blazer", "suit"]
+ })"_json;
+
+ synonym_json["id"] = synonym_json["id"].get<std::string>() + std::to_string(i + 1);
+
+ coll3->add_synonym(synonym_json);
+ }
+
+ auto req = std::make_shared<http_req>();
+ auto resp = std::make_shared<http_res>(nullptr);
+
+ req->params["collection"] = "coll3";
+ req->params["offset"] = "0";
+ req->params["limit"] = "1";
+
+ get_synonyms(req, resp);
+
+ nlohmann::json expected_json = R"({
+ "synonyms":[
+ {
- "id":"foobar4",
++ "id":"foobar1",
+ "root":"",
+ "synonyms":["blazer","suit"]
+ }]
+ })"_json;
+
+ ASSERT_EQ(expected_json.dump(), resp->body);
+
+ //invalid offset string
+ req->params["offset"] = "0a";
+ get_collections(req, resp);
+
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Offset param should be unsigned integer.\"}", resp->body);
+
+ //invalid limit string
+ req->params["offset"] = "0";
+ req->params["limit"] = "-1";
+ get_collections(req, resp);
+ ASSERT_EQ(400, resp->status_code);
+ ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body);
}