Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor concurrency model.
kishorenc17fbbd083887
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 4e147b41..3c0ea9fa 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -3,8 +3,8 @@ project(typesense)cmake_policy(SET CMP0074 NEW)-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++11 -O0 -g")+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -O2 -g -DNDEBUG")+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -std=c++17 -O0 -g")set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")diff --git a/docker-build.sh b/docker-build.shindex 5bab1fe5..59b81e07 100755--- a/docker-build.sh+++ b/docker-build.sh@@ -22,10 +22,10 @@ if [[ "$@" == *"--depclean"* ]]; thenfiecho "Building Typesense $TYPESENSE_VERSION..."-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \-DCMAKE_BUILD_TYPE=Release -H/typesense -B/typesense/$BUILD_DIR-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIR+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIRif [[ "$@" == *"--build-deploy-image"* ]]; thenecho "Creating deployment image for Typesense $TYPESENSE_VERSION server ..."@@ -45,7 +45,7 @@ fi##if [[ "$@" == *"--create-deb-upload"* ]]; then-# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \+# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \# -DCMAKE_BUILD_TYPE=Debug -H/typesense -B/typesense/$BUILD_DIR#fidiff --git a/docker/development.Dockerfile b/docker/development.Dockerfileindex bb480975..8840674f 100644--- a/docker/development.Dockerfile+++ b/docker/development.Dockerfile@@ -2,10 +2,10 @@## $ docker push typesense/typesense-development:latest-FROM typesense/ubuntu-10-04-gcc:6.4.0+FROM typesense/ubuntu-12-04-gcc:v10.1.0-ENV PATH /usr/local/gcc-6.4.0/bin/:$PATH-ENV LD_LIBRARY_PATH /usr/local/gcc-6.4.0/lib64+ENV PATH /usr/local/gcc-10.1.0/bin/:$PATH+ENV LD_LIBRARY_PATH /usr/local/gcc-10.1.0/lib64RUN apt-get install -y python-software-properties \&& add-apt-repository ppa:git-core/ppa \@@ -16,9 +16,9 @@ RUN apt-get install -y python-software-properties \libidn11 \git-ADD https://gcc.gnu.org/pub/binutils/snapshots/binutils-2.30.0.tar.xz /opt/binutils-2.30.0.tar.xz-RUN tar -C /opt -xf /opt/binutils-2.30.0.tar.xz-RUN cd /opt/binutils-2.30.0 && ./configure --prefix=/usr && make tooldir=/usr && make check && \+ADD https://ftp.gnu.org/gnu/binutils/binutils-2.36.tar.xz /opt/binutils-2.36.tar.xz+RUN tar -C /opt -xf /opt/binutils-2.36.tar.xz+RUN cd /opt/binutils-2.36 && ./configure --prefix=/usr && make tooldir=/usr && make check && \make -j8 tooldir=/usr install && cp include/libiberty.h /usr/includeADD https://github.com/Kitware/CMake/releases/download/v3.15.2/cmake-3.15.2-Linux-x86_64.tar.gz /opt/cmake-3.15.2-Linux-x86_64.tar.gz@@ -29,8 +29,8 @@ ADD https://launchpad.net/ubuntu/+archive/primary/+files/snappy_1.1.3.orig.tar.gRUN tar -C /opt -xf /opt/snappy_1.1.3.orig.tar.gzRUN mkdir -p /opt/snappy-1.1.3/build && cd /opt/snappy-1.1.3/build && ../configure && make -j8 && make install-ADD https://github.com/unicode-org/icu/releases/download/release-61-1/icu4c-61_1-src.tgz /opt/icu4c-61_1-src.tgz-RUN tar -C /opt -xf /opt/icu4c-61_1-src.tgz+ADD https://github.com/unicode-org/icu/releases/download/release-68-2/icu4c-68_2-src.tgz /opt/icu4c-68_2-src.tgz+RUN tar -C /opt -xf /opt/icu4c-68_2-src.tgzRUN cd /opt/icu/source && echo "#define U_DISABLE_RENAMING 1" >> common/unicode/uconfig.h && \echo "#define U_STATIC_IMPLEMENTATION 1" >> common/unicode/uconfig.h && \echo "#define U_USING_ICU_NAMESPACE 0" >> common/unicode/uconfig.h@@ -96,5 +96,5 @@ RUN tar -C /opt -xf /opt/elfutils-0.182.tar.bz2RUN cd /opt/elfutils-0.182 && ./configure --disable-libdebuginfod --disable-debuginfod && \make -j8 && make install && rm -rf /usr/local/lib/*.so*-ENV CC /usr/local/gcc-6.4.0/bin/gcc-ENV CXX /usr/local/gcc-6.4.0/bin/g+++ENV CC /usr/local/gcc-10.1.0/bin/gcc+ENV CXX /usr/local/gcc-10.1.0/bin/g++diff --git a/docker/ubuntu-12-04-gcc.Dockerfile b/docker/ubuntu-12-04-gcc.Dockerfilenew file mode 100644index 00000000..73f78191--- /dev/null+++ b/docker/ubuntu-12-04-gcc.Dockerfile@@ -0,0 +1,37 @@+# $ docker build --file $PROJECT_DIR/docker/ubuntu-10-04-gcc.Dockerfile --tag typesense/ubuntu-12.04-gcc:latest --tag typesense/ubuntu-12.04-gcc:6.4 $PROJECT_DIR/docker+#+# NOTE: Default build image is bloated. Before publishing, export from a container to squash the image:+# $ docker run -it typesense/ubuntu-12.04-gcc:latest bash -c "exit"+# $ docker ps -a | grep typesense+# $ docker export <container_id> | docker import - typesense/ubuntu-12.04-gcc:latest+# $ docker push typesense/ubuntu-12.04-gcc++FROM ubuntu:12.04++RUN sed -i -re 's/([a-z]{2}\.)?archive.ubuntu.com|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list+RUN apt-get update && apt-get install -y \+ build-essential \+ tar \+ libmpfr-dev \+ libgmp3-dev \+ libmpc-dev \+ flex \+ bison \+ zlib1g-dev++ADD https://ftp.gnu.org/gnu/gcc/gcc-10.1.0/gcc-10.1.0.tar.gz /opt/+RUN tar -C /opt -xf /opt/gcc-10.1.0.tar.gz++RUN mkdir /opt/gcc-10.1.0/build && cd /opt/gcc-10.1.0/build && ../configure --disable-checking --enable-languages=c,c++ \+ --enable-multiarch --disable-multilib --enable-shared --enable-threads=posix \+ --with-gmp=/usr/lib --with-mpc=/usr/lib --with-mpfr=/usr/lib \+ --build=x86_64-linux-gnu --host=x86_64-linux-gnu --target=x86_64-linux-gnu \+ --without-included-gettext --with-tune=generic --prefix=/usr/local/gcc-10.1.0+RUN cd /opt/gcc-10.1.0/build && make -j8+RUN cd /opt/gcc-10.1.0/build && make install++RUN apt-get remove -y gcc+RUN rm -rf /opt/gcc-10.1.0 /opt/gcc-10.1.0.tar.gz++ENV PATH /usr/local/gcc-10.1.0/bin/:$PATH+ENV LD_LIBRARY_PATH /usr/local/gcc-10.1.0/lib64diff --git a/include/auth_manager.h b/include/auth_manager.hindex 2cb10262..d1e5c571 100644--- a/include/auth_manager.h+++ b/include/auth_manager.h@@ -3,6 +3,8 @@#include <string>#include <vector>#include <map>+#include <mutex>+#include <shared_mutex>#include "json.hpp"#include "option.h"#include "store.h"@@ -78,11 +80,13 @@ class AuthManager {private:+ mutable std::shared_mutex mutex;+std::map<std::string, api_key_t> api_keys; // stores key_value => key mappingStore *store;// Auto incrementing API KEY ID- uint32_t next_api_key_id;+ std::atomic<uint32_t> next_api_key_id;// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV storestatic constexpr const char* API_KEY_NEXT_ID_KEY = "$KN";@@ -94,39 +98,31 @@ private:static std::string fmt_error(std::string&& error, const std::string& key);+ Option<bool> authenticate_parse_params(const std::string& scoped_api_key, const std::string& action,+ const std::string& collection, nlohmann::json& embedded_params) const;++ bool auth_against_key(const std::string &collection, const std::string& action,+ const api_key_t &api_key, const bool search_only) const;+public:+ static const size_t KEY_LEN = 32;+ static const size_t HMAC_BASE64_LEN = 44;+AuthManager() = default;Option<bool> init(Store *store);- Option<std::vector<api_key_t>> list_keys();+ Option<std::vector<api_key_t>> list_keys() const;- Option<api_key_t> get_key(uint32_t id, bool truncate_value = true);+ Option<api_key_t> get_key(uint32_t id, bool truncate_value = true) const;Option<api_key_t> create_key(api_key_t& api_key);Option<api_key_t> remove_key(uint32_t id);- bool authenticate(- const std::string& req_api_key,- const std::string& action,- const std::string& collection,- std::map<std::string, std::string>& params- );-- Option<bool> authenticate_parse_params(- const std::string& scoped_api_key,- const std::string& action,- const std::string& collection,- nlohmann::json& embedded_params- );-- static const size_t KEY_LEN = 32;- static const size_t HMAC_BASE64_LEN = 44;-- bool auth_against_key(const std::string &collection, const std::string& action,- const api_key_t &api_key, const bool search_only) const;+ bool authenticate(const std::string& req_api_key, const std::string& action, const std::string& collection,+ std::map<std::string, std::string>& params) const;static bool populate_req_params(std::map<std::string, std::string> &req_params,nlohmann::detail::iteration_proxy_value<nlohmann::json::iterator>& item);diff --git a/include/collection.h b/include/collection.hindex 29984509..2f60b51d 100644--- a/include/collection.h+++ b/include/collection.h@@ -7,6 +7,7 @@#include <thread>#include <mutex>#include <condition_variable>+#include <shared_mutex>#include <art.h>#include <index.h>#include <number.h>@@ -267,6 +268,8 @@ struct synonym_t {class Collection {private:+ mutable std::shared_mutex mutex;+const uint8_t CURATED_RECORD_IDENTIFIER = 100;struct highlight_t {@@ -286,26 +289,20 @@ private:}};- std::map<std::string, override_t> overrides;-- std::string name;-- uint32_t collection_id;+ const std::string name;- uint64_t created_at;+ const uint32_t collection_id;- size_t num_documents;+ const uint64_t created_at;- std::vector<Index*> indices;-- std::vector<std::thread*> index_threads;+ std::atomic<size_t> num_documents;// Auto incrementing record ID used internally for indexing - not exposed to the client- uint32_t next_seq_id;+ std::atomic<uint32_t> next_seq_id;Store* store;- std::vector<field> fields;+ const std::vector<field> fields;std::unordered_map<std::string, field> search_schema;@@ -313,18 +310,24 @@ private:std::unordered_map<std::string, field> sort_schema;+ std::map<std::string, override_t> overrides;+spp::sparse_hash_map<std::string, synonym_t> synonym_definitions;spp::sparse_hash_map<uint64_t, std::vector<std::string>> synonym_index;- std::string default_sorting_field;+ const std::string default_sorting_field;- size_t num_memory_shards;+ std::atomic<size_t> num_memory_shards;const float max_memory_ratio;- std::string get_doc_id_key(const std::string & doc_id);+ const std::vector<Index*> indices;- std::string get_seq_id_key(uint32_t seq_id);+ // methods++ std::string get_doc_id_key(const std::string & doc_id) const;++ std::string get_seq_id_key(uint32_t seq_id) const;void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,const KV* field_order_kv, const nlohmann::json &document,@@ -334,7 +337,7 @@ private:bool highlighted_fully,const std::string& highlight_start_tag,const std::string& highlight_end_tag,- highlight_t &highlight);+ highlight_t &highlight) const;void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);@@ -342,7 +345,7 @@ private:const std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,std::map<size_t, std::vector<uint32_t>>& include_ids,- std::vector<uint32_t> & excluded_ids);+ std::vector<uint32_t> & excluded_ids) const;static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,const std::pair<uint64_t, facet_count_t>& b) {@@ -356,18 +359,37 @@ private:void free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const;- Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters);+ Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters) const;- Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,+ static Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,std::map<size_t, std::vector<std::string>>& pinned_hits);void synonym_reduction_internal(const std::vector<std::string>& tokens,size_t start_window_size,size_t start_index_pos,std::set<uint64_t>& processed_syn_hashes,- std::vector<std::vector<std::string>>& results);+ std::vector<std::vector<std::string>>& results) const;++ std::vector<Index *> init_indices();public:++ enum {MAX_ARRAY_MATCHES = 5};++ const size_t PER_PAGE_MAX = 250;++ const size_t GROUP_LIMIT_MAX = 99;++ // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store+ static constexpr const char* COLLECTION_META_PREFIX = "$CM";+ static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";+ static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";+ static constexpr const char* COLLECTION_SYNONYM_PREFIX = "$CY";+ static constexpr const char* SEQ_ID_PREFIX = "$SI";+ static constexpr const char* DOC_ID_PREFIX = "$DI";++ // methods+Collection() = delete;Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,@@ -384,23 +406,17 @@ public:static std::string get_synonym_key(const std::string & collection_name, const std::string & synonym_id);- std::string get_seq_id_collection_prefix();-- std::string get_name();+ std::string get_seq_id_collection_prefix() const;- uint64_t get_created_at();+ std::string get_name() const;- size_t get_num_documents();+ uint64_t get_created_at() const;- uint32_t get_collection_id();+ uint32_t get_collection_id() const;uint32_t get_next_seq_id();- void set_next_seq_id(uint32_t seq_id);-- void increment_next_seq_id_field();-- Option<uint32_t> doc_id_to_seq_id(const std::string & doc_id);+ Option<uint32_t> doc_id_to_seq_id(const std::string & doc_id) const;std::vector<std::string> get_facet_fields();@@ -415,110 +431,106 @@ public:Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,const index_operation_t& operation, const std::string& id="");- nlohmann::json get_summary_json();-- Option<nlohmann::json> add(const std::string & json_str,- const index_operation_t& operation=CREATE, const std::string& id="");-- nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,- const index_operation_t& operation=CREATE, const std::string& id="");+ static uint32_t get_seq_id_from_key(const std::string & key);- Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,- const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,- const std::vector<sort_by> & sort_fields, int num_typos,- size_t per_page = 10, size_t page = 1,- token_ordering token_order = FREQUENCY, bool prefix = false,- size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,- const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),- const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),- size_t max_facet_values=10,- const std::string & simple_facet_query = "",- const size_t snippet_threshold = 30,- const size_t highlight_affix_num_tokens = 4,- const std::string & highlight_full_fields = "",- size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,- const std::string& pinned_hits_str="",- const std::string& hidden_hits="",- const std::vector<std::string>& group_by_fields={},- const size_t group_limit = 0,- const std::string& highlight_start_tag="<mark>",- const std::string& highlight_end_tag="</mark>",- std::vector<size_t> query_by_weights={},- size_t limit_hits=UINT32_MAX);+ Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document) const;- Option<bool> get_filter_ids(const std::string & simple_filter_query,- std::vector<std::pair<size_t, uint32_t*>>& index_ids);+ Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, bool is_update);- Option<nlohmann::json> get(const std::string & id);+ static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,+ const spp::sparse_hash_set<std::string> & exclude_fields);- bool doc_exists(const std::string & id);+ const std::vector<Index *> &_get_indexes() const;- Option<std::string> remove(const std::string & id, bool remove_from_store = true);+ bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,+ std::string &value) const;- Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);+ static void aggregate_topster(size_t query_index, Topster &topster, Topster *index_topster);- Option<uint32_t> add_override(const override_t & override);+ static void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs);- Option<uint32_t> remove_override(const std::string & id);+ void batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,+ size_t &num_indexed);- std::map<std::string, override_t>& get_overrides() {- return overrides;- };+ bool is_exceeding_memory_threshold() const;- size_t get_num_memory_shards();+ static void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,+ nlohmann::json &new_doc,+ nlohmann::json &del_doc);- static uint32_t get_seq_id_from_key(const std::string & key);+ void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,+ std::vector<std::string>& q_exclude_tokens) const;- Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document);+ // PUBLIC OPERATIONS- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, bool is_update);+ nlohmann::json get_summary_json() const;size_t par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch, std::vector<size_t>& indexed_counts);- static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,- const spp::sparse_hash_set<std::string> & exclude_fields);+ Option<nlohmann::json> add(const std::string & json_str,+ const index_operation_t& operation=CREATE, const std::string& id="");- const std::vector<Index *> &_get_indexes() const;+ nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,+ const index_operation_t& operation=CREATE, const std::string& id="");- enum {MAX_ARRAY_MATCHES = 5};+ Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,+ const std::vector<sort_by> & sort_fields, int num_typos,+ size_t per_page = 10, size_t page = 1,+ token_ordering token_order = FREQUENCY, bool prefix = false,+ size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,+ const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),+ const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),+ size_t max_facet_values=10,+ const std::string & simple_facet_query = "",+ const size_t snippet_threshold = 30,+ const size_t highlight_affix_num_tokens = 4,+ const std::string & highlight_full_fields = "",+ size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,+ const std::string& pinned_hits_str="",+ const std::string& hidden_hits="",+ const std::vector<std::string>& group_by_fields={},+ const size_t group_limit = 0,+ const std::string& highlight_start_tag="<mark>",+ const std::string& highlight_end_tag="</mark>",+ std::vector<size_t> query_by_weights={},+ size_t limit_hits=UINT32_MAX) const;- const size_t PER_PAGE_MAX = 250;+ Option<bool> get_filter_ids(const std::string & simple_filter_query,+ std::vector<std::pair<size_t, uint32_t*>>& index_ids);- const size_t GROUP_LIMIT_MAX = 99;+ Option<nlohmann::json> get(const std::string & id) const;- // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store- static constexpr const char* COLLECTION_META_PREFIX = "$CM";- static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";- static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";- static constexpr const char* COLLECTION_SYNONYM_PREFIX = "$CY";- static constexpr const char* SEQ_ID_PREFIX = "$SI";- static constexpr const char* DOC_ID_PREFIX = "$DI";+ Option<std::string> remove(const std::string & id, bool remove_from_store = true);- bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,- std::string &value);+ Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);- void aggregate_topster(size_t query_index, Topster &topster, Topster *index_topster) const;+ size_t get_num_memory_shards();- void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) const;+ size_t get_num_documents() const;- void batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,- size_t &num_indexed);+ // Override operations- bool is_exceeding_memory_threshold() const;+ Option<uint32_t> add_override(const override_t & override);- void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,- nlohmann::json &new_doc,- nlohmann::json &del_doc);+ Option<uint32_t> remove_override(const std::string & id);- void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,- std::vector<std::string>& q_exclude_tokens) const;+ std::map<std::string, override_t> get_overrides() {+ std::shared_lock lock(mutex);+ return overrides;+ };// synonym operations- void synonym_reduction(const std::vector<std::string>& tokens, std::vector<std::vector<std::string>>& results);- spp::sparse_hash_map<std::string, synonym_t>& get_synonyms();+ void synonym_reduction(const std::vector<std::string>& tokens,+ std::vector<std::vector<std::string>>& results) const;++ spp::sparse_hash_map<std::string, synonym_t> get_synonyms();+bool get_synonym(const std::string& id, synonym_t& synonym);+Option<bool> add_synonym(const synonym_t& synonym);+Option<bool> remove_synonym(const std::string & id);};diff --git a/include/collection_manager.h b/include/collection_manager.hindex 75a30172..147601d9 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -7,11 +7,54 @@#include "field.h"#include "collection.h"#include "auth_manager.h"+#include "threadpool.h"++template<typename ResourceType>+struct locked_resource_view_t {+ locked_resource_view_t(std::shared_mutex &mutex, ResourceType &resource) : _lock(mutex), _resource(&resource) {}++ locked_resource_view_t(std::shared_mutex &mutex, ResourceType *resource) : _lock(mutex), _resource(resource) {}++ locked_resource_view_t(ResourceType &&) = delete;++ locked_resource_view_t(const ResourceType &) = delete;++ locked_resource_view_t &operator=(const ResourceType &) = delete;++ ResourceType* operator->() {+ return _resource;+ }++ bool operator==(const ResourceType* other) {+ return other == _resource;+ }++ bool operator!=(const ResourceType* other) {+ return other != _resource;+ }++ void unlock() {+ _lock.unlock();+ }++ ResourceType* get() {+ return _resource;+ }++private:+ std::shared_lock<std::shared_mutex> _lock;+ ResourceType* _resource;+};+// Singleton, for managing meta information of all collections and house keepingclass CollectionManager {private:+ mutable std::shared_mutex mutex;+Store *store;+ ThreadPool* thread_pool;+AuthManager auth_manager;spp::sparse_hash_map<std::string, Collection*> collections;@@ -22,7 +65,7 @@ private:// Auto incrementing ID assigned to each collection// Using a ID instead of a collection's name makes renaming possible- uint32_t next_collection_id;+ std::atomic<uint32_t> next_collection_id;static constexpr const char* COLLECTION_NAME_KEY = "name";static constexpr const char* COLLECTION_ID_KEY = "id";@@ -42,7 +85,7 @@ private:~CollectionManager() = default;- Option<std::string> get_first_index_error(const std::vector<index_record>& index_records) {+ static Option<std::string> get_first_index_error(const std::vector<index_record>& index_records) {for(const auto & index_record: index_records) {if(!index_record.indexed.ok()) {return Option<std::string>(index_record.indexed.error());@@ -53,6 +96,11 @@ private:}public:+ static const size_t DEFAULT_NUM_MEMORY_SHARDS = 4;++ static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";+ static constexpr const char* SYMLINK_PREFIX = "$SL";+static CollectionManager & get_instance() {static CollectionManager instance;return instance;@@ -61,6 +109,19 @@ public:CollectionManager(CollectionManager const&) = delete;void operator=(CollectionManager const&) = delete;+ Collection* init_collection(const nlohmann::json & collection_meta, const uint32_t collection_next_seq_id);++ void add_to_collections(Collection* collection);++ std::vector<Collection*> get_collections() const;++ Collection* get_collection_unafe(const std::string & collection_name) const;++ // PUBLICLY EXPOSED API++ void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio, const std::string & auth_key);++ // only for tests!void init(Store *store, const float max_memory_ratio, const std::string & auth_key);Option<bool> load(const size_t init_batch_size=1000);@@ -68,49 +129,40 @@ public:// frees in-memory data structures when server is shutdown - helps us run a memory leak detector properlyvoid dispose();- Collection* init_collection(const nlohmann::json & collection_meta, const uint32_t collection_next_seq_id);-- void add_to_collections(Collection* collection);-bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,- const std::string& collection, std::map<std::string, std::string>& params);+ const std::string& collection, std::map<std::string, std::string>& params) const;Option<Collection*> create_collection(const std::string& name, const size_t num_memory_shards,const std::vector<field> & fields,const std::string & default_sorting_field,const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)));- Collection* get_collection(const std::string & collection_name);+ locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;- Collection* get_collection_with_id(uint32_t collection_id);+ locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;- std::vector<Collection*> get_collections();+ nlohmann::json get_collection_summaries() const;- Option<bool> drop_collection(const std::string& collection_name, const bool remove_from_store = true);+ Option<nlohmann::json> drop_collection(const std::string& collection_name, const bool remove_from_store = true);- uint32_t get_next_collection_id();+ uint32_t get_next_collection_id() const;static std::string get_symlink_key(const std::string & symlink_name);- void set_next_collection_id(uint32_t next_id);-Store* get_store();+ ThreadPool* get_thread_pool() const;+AuthManager& getAuthManager();static Option<bool> do_search(std::map<std::string, std::string>& req_params, std::string& results_json_str);// symlinks- Option<std::string> resolve_symlink(const std::string & symlink_name);+ Option<std::string> resolve_symlink(const std::string & symlink_name) const;- spp::sparse_hash_map<std::string, std::string> & get_symlinks();+ spp::sparse_hash_map<std::string, std::string> get_symlinks() const;Option<bool> upsert_symlink(const std::string & symlink_name, const std::string & collection_name);Option<bool> delete_symlink(const std::string & symlink_name);-- static const size_t DEFAULT_NUM_MEMORY_SHARDS = 4;-- static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";- static constexpr const char* SYMLINK_PREFIX = "$SL";};\ No newline at end of filediff --git a/include/http_server.h b/include/http_server.hindex 914d58a2..01ce190e 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -14,6 +14,7 @@ extern "C" {#include <cstdio>#include "http_data.h"#include "option.h"+#include "threadpool.h"class ReplicationState;class HttpServer;@@ -75,6 +76,8 @@ private:bool cors_enabled;+ ThreadPool* thread_pool;+bool (*auth_handler)(std::map<std::string, std::string>& params, const route_path& rpath,const std::string& auth_key);@@ -108,7 +111,9 @@ private:public:HttpServer(const std::string & version,const std::string & listen_address, uint32_t listen_port,- const std::string & ssl_cert_path, const std::string & ssl_cert_key_path, bool cors_enabled);+ const std::string & ssl_cert_path,+ const std::string & ssl_cert_key_path,+ bool cors_enabled, ThreadPool* thread_pool);~HttpServer();@@ -164,6 +169,8 @@ public:std::string get_version();+ ThreadPool* get_thread_pool() const;+static constexpr const char* STOP_SERVER_MESSAGE = "STOP_SERVER";static constexpr const char* STREAM_RESPONSE_MESSAGE = "STREAM_RESPONSE";static constexpr const char* REQUEST_PROCEED_MESSAGE = "REQUEST_PROCEED";diff --git a/include/index.h b/include/index.hindex d7ab0a9e..abcc17c7 100644--- a/include/index.h+++ b/include/index.h@@ -4,6 +4,7 @@#include <unordered_map>#include <vector>#include <mutex>+#include <shared_mutex>#include <condition_variable>#include <art.h>#include <number.h>@@ -61,9 +62,8 @@ struct search_args {Topster* curated_topster;std::vector<std::vector<KV*>> raw_result_kvs;std::vector<std::vector<KV*>> override_result_kvs;- Option<uint32_t> outcome;- search_args(): outcome(0) {+ search_args() {}@@ -83,7 +83,7 @@ struct search_args {page(page), token_order(token_order), prefix(prefix),drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),group_by_fields(group_by_fields), group_limit(group_limit),- all_result_ids_len(0), outcome(0) {+ all_result_ids_len(0) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -134,6 +134,8 @@ struct index_record {class Index {private:+ mutable std::shared_mutex mutex;+const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();std::string name;@@ -158,6 +160,8 @@ private:StringUtils string_utils;+ // Internal utility functions+static inline uint32_t next_suggestion(const std::vector<token_candidates> &token_candidates_vec,long long int n,std::vector<art_leaf *>& actual_query_suggestion,@@ -167,7 +171,8 @@ private:void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,- const uint32_t* result_ids, size_t results_size);+ size_t group_limit, const std::vector<std::string>& group_by_fields,+ const uint32_t* result_ids, size_t results_size) const;void search_field(const uint8_t & field_id,std::vector<token_t>& query_tokens,@@ -182,9 +187,11 @@ private:Topster* topster, spp::sparse_hash_set<uint64_t>& groups_processed,uint32_t** all_result_ids, size_t & all_result_ids_len,size_t& field_num_results,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,const token_ordering token_order = FREQUENCY, const bool prefix = false,const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,- const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD);+ const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD) const;void search_candidates(const uint8_t & field_id,uint32_t* filter_ids, size_t filter_ids_length,@@ -196,7 +203,8 @@ private:uint32_t** all_result_ids,size_t & all_result_ids_len,size_t& field_num_results,- const size_t typo_tokens_threshold);+ const size_t typo_tokens_threshold,+ const size_t group_limit, const std::vector<std::string>& group_by_fields) const;void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;@@ -213,13 +221,23 @@ private:void collate_included_ids(const std::vector<std::string>& q_included_tokens,const std::string & field, const uint8_t field_id,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,- Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);+ Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries) const;- uint64_t facet_token_hash(const field & a_field, const std::string &token);+ static uint64_t facet_token_hash(const field & a_field, const std::string &token);- void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);+ static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);public:+ // for limiting number of results on multiple candidates / query rewrites+ enum {TYPO_TOKENS_THRESHOLD = 100};++ // for limiting number of fields that can be searched on+ enum {FIELD_LIMIT_NUM = 100};++ // If the number of results found is less than this threshold, Typesense will attempt to drop the tokens+ // in the query that have the least individual hits one by one until enough results are found.+ static const int DROP_TOKENS_THRESHOLD = 10;+Index() = delete;Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,@@ -233,98 +251,29 @@ public:return combined;}- void run_search();-- void search(Option<uint32_t> & outcome,- const std::vector<std::string> & q_include_tokens,- const std::vector<std::string> & q_exclude_tokens,- const std::vector<std::vector<std::string>>& q_synonyms,- const std::vector<search_field_t> & search_fields,- const std::vector<filter> & filters, std::vector<facet> & facets,- facet_query_t & facet_query,- const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,- const std::vector<uint32_t> & excluded_ids,- const std::vector<sort_by> & sort_fields_std, const int num_typos,- Topster* topster, Topster* curated_topster,- const size_t per_page, const size_t page, const token_ordering token_order,- const bool prefix, const size_t drop_tokens_threshold,- size_t & all_result_ids_len,- spp::sparse_hash_set<uint64_t>& groups_processed,- std::vector<std::vector<art_leaf*>> & searched_queries,- std::vector<std::vector<KV*>> & raw_result_kvs,- std::vector<std::vector<KV*>> & override_result_kvs,- const size_t typo_tokens_threshold);-static void concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids);- Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);-- Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);-- art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);-- static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,- std::vector<uint32_t*>& leaf_to_indices,- size_t result_index,- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);-void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const uint32_t *result_ids, const size_t result_size, uint32_t token_bits);+ const uint32_t *result_ids, const size_t result_size,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,+ uint32_t token_bits) const;static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,- const std::string & default_sorting_field, bool is_update);-- static Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,- const std::string & default_sorting_field,- const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema,- bool is_update);-- static size_t batch_memory_index(Index *index,- std::vector<index_record> & iter_batch,- const std::string & default_sorting_field,- const std::unordered_map<std::string, field> & search_schema,- const std::map<std::string, field> & facet_schema);-- const spp::sparse_hash_map<std::string, art_tree *> &_get_search_index() const;+ const spp::sparse_hash_map<std::string, art_tree *>& _get_search_index() const;const spp::sparse_hash_map<std::string, num_tree_t*>& _get_numerical_index() const;- // for limiting number of results on multiple candidates / query rewrites- enum {TYPO_TOKENS_THRESHOLD = 100};-- // for limiting number of fields that can be searched on- enum {FIELD_LIMIT_NUM = 100};-- // If the number of results found is less than this threshold, Typesense will attempt to drop the tokens- // in the query that have the least individual hits one by one until enough results are found.- static const int DROP_TOKENS_THRESHOLD = 10;-- /*- * Concurrency Primitives- */-- // Used for passing control back and forth between main and worker threads- std::mutex m;- std::condition_variable cv;-- bool ready; // prevents spurious wake up of the worker thread- bool processed; // prevents spurious wake up of the main thread- bool terminate; // used for interrupting the thread during tear down-- search_args* search_params;-- int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;+ static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len);static int64_t float_to_in64_t(float n);- uint64_t get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id, const uint32_t seq_id) const;+ uint64_t get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id,+ const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;- void get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index);+ void get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) const;void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,const std::vector<art_leaf *> &query_suggestion,@@ -332,10 +281,10 @@ public:void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);- void tokenize_doc_field(const nlohmann::json& document, const field& search_field, std::vector<std::string>& tokens);+ static void tokenize_doc_field(const nlohmann::json& document, const field& search_field, std::vector<std::string>& tokens);template<typename T>- bool _arrays_match(std::vector<T> reindex_vals, std::vector<T> old_vals) {+ static bool _arrays_match(std::vector<T> reindex_vals, std::vector<T> old_vals) {if(old_vals.size() != reindex_vals.size()) {return false;}@@ -350,5 +299,60 @@ public:return true;}++ // Public operations++ void run_search(search_args* search_params);++ void search(const std::vector<std::string> & q_include_tokens,+ const std::vector<std::string> & q_exclude_tokens,+ const std::vector<std::vector<std::string>>& q_synonyms,+ const std::vector<search_field_t> & search_fields,+ const std::vector<filter> & filters, std::vector<facet> & facets,+ facet_query_t & facet_query,+ const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,+ const std::vector<uint32_t> & excluded_ids,+ const std::vector<sort_by> & sort_fields_std, const int num_typos,+ Topster* topster, Topster* curated_topster,+ const size_t per_page, const size_t page, const token_ordering token_order,+ const bool prefix, const size_t drop_tokens_threshold,+ size_t & all_result_ids_len,+ spp::sparse_hash_set<uint64_t>& groups_processed,+ std::vector<std::vector<art_leaf*>> & searched_queries,+ std::vector<std::vector<KV*>> & raw_result_kvs,+ std::vector<std::vector<KV*>> & override_result_kvs,+ const size_t typo_tokens_threshold,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields) const;++ Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);+++ Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,+ const std::string & default_sorting_field, bool is_update);++ static size_t batch_memory_index(Index *index,+ std::vector<index_record> & iter_batch,+ const std::string & default_sorting_field,+ const std::unordered_map<std::string, field> & search_schema,+ const std::map<std::string, field> & facet_schema);++ static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,+ const std::vector<uint32_t*>& leaf_to_indices,+ const size_t result_index,+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);++ // the following methods are not synchronized because their parent calls are synchronized++ art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);++ uint32_t do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const;++ static Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,+ const std::string & default_sorting_field,+ const std::unordered_map<std::string, field> & search_schema,+ const std::map<std::string, field> & facet_schema,+ bool is_update);+};diff --git a/src/auth_manager.cpp b/src/auth_manager.cppindex cb6fbb9f..de338956 100644--- a/src/auth_manager.cpp+++ b/src/auth_manager.cpp@@ -8,6 +8,7 @@ constexpr const uint64_t api_key_t::FAR_FUTURE_TIMESTAMP;Option<bool> AuthManager::init(Store *store) {// This function must be idempotent, i.e. when called multiple times, must produce the same state without leaks//LOG(INFO) << "AuthManager::init()";+ std::unique_lock lock(mutex);this->store = store;@@ -42,7 +43,9 @@ Option<bool> AuthManager::init(Store *store) {return Option<bool>(true);}-Option<std::vector<api_key_t>> AuthManager::list_keys() {+Option<std::vector<api_key_t>> AuthManager::list_keys() const {+ std::shared_lock lock(mutex);+std::vector<std::string> api_key_json_strs;store->scan_fill(API_KEYS_PREFIX, api_key_json_strs);@@ -61,7 +64,9 @@ Option<std::vector<api_key_t>> AuthManager::list_keys() {return Option<std::vector<api_key_t>>(stored_api_keys);}-Option<api_key_t> AuthManager::get_key(uint32_t id, bool truncate_value) {+Option<api_key_t> AuthManager::get_key(uint32_t id, bool truncate_value) const {+ std::shared_lock lock(mutex);+std::string api_key_store_key = std::string(API_KEYS_PREFIX) + "_" + std::to_string(id);std::string api_key_json_str;StoreStatus status = store->get(api_key_store_key, api_key_json_str);@@ -88,6 +93,7 @@ Option<api_key_t> AuthManager::get_key(uint32_t id, bool truncate_value) {Option<api_key_t> AuthManager::create_key(api_key_t& api_key) {//LOG(INFO) << "AuthManager::create_key()";+ std::unique_lock lock(mutex);if(api_keys.count(api_key.value) != 0) {return Option<api_key_t>(409, "API key generation conflict.");@@ -119,6 +125,8 @@ Option<api_key_t> AuthManager::remove_key(uint32_t id) {return Option<api_key_t>(500, "Could not delete API key.");}+ std::unique_lock lock(mutex);+api_key_t&& key = key_op.get();api_keys.erase(key.value);@@ -131,7 +139,9 @@ uint32_t AuthManager::get_next_api_key_id() {}bool AuthManager::authenticate(const std::string& req_api_key, const std::string& action,- const std::string& collection, std::map<std::string, std::string>& params) {+ const std::string& collection, std::map<std::string, std::string>& params) const {++ std::shared_lock lock(mutex);//LOG(INFO) << "AuthManager::authenticate()";@@ -168,7 +178,7 @@ bool AuthManager::authenticate(const std::string& req_api_key, const std::stringreturn false;}- const api_key_t& api_key = api_keys[req_api_key];+ const api_key_t& api_key = api_keys.at(req_api_key);return auth_against_key(collection, action, api_key, false);}@@ -212,7 +222,7 @@ bool AuthManager::auth_against_key(const std::string& collection, const std::str}Option<bool> AuthManager::authenticate_parse_params(const std::string& scoped_api_key, const std::string& action,- const std::string& collection, nlohmann::json& embedded_params) {+ const std::string& collection, nlohmann::json& embedded_params) const {// allow only searches from scoped keysif(action != DOCUMENTS_SEARCH_ACTION) {LOG(ERROR) << "Scoped API keys can only be used for searches.";diff --git a/src/collection.cpp b/src/collection.cppindex b643cabc..914cfc28 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -11,6 +11,7 @@#include <rocksdb/write_batch.h>#include <system_metrics.h>#include <tokenizer.h>+#include <collection_manager.h>#include "topster.h"#include "logger.h"@@ -39,51 +40,20 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, coconst uint32_t next_seq_id, Store *store, const std::vector<field> &fields,const std::string& default_sorting_field, const size_t num_memory_shards,const float max_memory_ratio):- name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),+ name(name), collection_id(collection_id), created_at(created_at),+ next_seq_id(next_seq_id), store(store),fields(fields), default_sorting_field(default_sorting_field),num_memory_shards(num_memory_shards),- max_memory_ratio(max_memory_ratio) {+ max_memory_ratio(max_memory_ratio),+ indices(init_indices()) {- for(const field& field: fields) {- search_schema.emplace(field.name, field);-- if(field.is_facet()) {- facet_schema.emplace(field.name, field);- }-- if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {- sort_schema.emplace(field.name, field);- }- }-- for(size_t i = 0; i < num_memory_shards; i++) {- Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);- indices.push_back(index);- std::thread* thread = new std::thread(&Index::run_search, index);- index_threads.push_back(thread);- }-- this->created_at = created_at;this->num_documents = 0;}Collection::~Collection() {for(size_t i = 0; i < indices.size(); i++) {- std::thread *t = index_threads[i];- Index* index = indices[i];- index->ready = true;- index->terminate = true;- index->cv.notify_one();- t->join();-- delete t;delete indices[i];- t = nullptr;- indices[i] = nullptr;}-- indices.clear();- index_threads.clear();}uint32_t Collection::get_next_seq_id() {@@ -91,14 +61,6 @@ uint32_t Collection::get_next_seq_id() {return next_seq_id++;}-void Collection::set_next_seq_id(uint32_t seq_id) {- next_seq_id = seq_id;-}--void Collection::increment_next_seq_id_field() {- next_seq_id++;-}-Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::json& document,const index_operation_t& operation, const std::string& id) {try {@@ -170,12 +132,14 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::}}-nlohmann::json Collection::get_summary_json() {+nlohmann::json Collection::get_summary_json() const {+ std::shared_lock lock(mutex);+nlohmann::json json_response;json_response["name"] = name;- json_response["num_memory_shards"] = num_memory_shards;- json_response["num_documents"] = num_documents;+ json_response["num_memory_shards"] = num_memory_shards.load();+ json_response["num_documents"] = num_documents.load();json_response["created_at"] = created_at;nlohmann::json fields_arr;@@ -373,6 +337,8 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch}Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id, bool is_update) {+ std::unique_lock lock(mutex);+if(!is_update) {// for update, validation should be done priorOption<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,@@ -392,21 +358,37 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinsize_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch,std::vector<size_t>& indexed_counts) {- std::vector<std::future<size_t>> futures;- size_t num_indexed = 0;+ std::unique_lock lock(mutex);++ size_t num_processed = 0;+ std::mutex m_process;+ std::condition_variable cv_process;- for(size_t i=0; i < num_memory_shards; i++) {- futures.push_back(- std::async(&Index::batch_memory_index, indices[i], std::ref(iter_batch[i]), default_sorting_field,- search_schema, facet_schema)- );+ std::vector<size_t> num_indexed_vec(indices.size());++ for(size_t index_id = 0; index_id < indices.size(); index_id++) {+ Index* index = indices[index_id];+ CollectionManager::get_instance().get_thread_pool()->enqueue(+ [index, index_id, &num_indexed_vec, &iter_batch, this, &m_process, &num_processed, &cv_process]() {+ size_t num_indexed = Index::batch_memory_index(index, std::ref(iter_batch[index_id]), default_sorting_field,+ search_schema, facet_schema);+ std::unique_lock<std::mutex> lock(m_process);+ num_indexed_vec[index_id] = num_indexed;+ num_processed++;+ cv_process.notify_one();+ });}- for(size_t i=0; i < futures.size(); i++) {- size_t num_indexed_future = futures[i].get();- num_documents += num_indexed_future;- num_indexed += num_indexed_future;- indexed_counts[i] = num_indexed_future;+ const size_t num_indices = indices.size();+ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_indices; });++ size_t num_indexed = 0;++ for(size_t index_id = 0; index_id < indices.size(); index_id++) {+ num_documents += num_indexed_vec[index_id];+ num_indexed += num_indexed_vec[index_id];+ indexed_counts[index_id] = num_indexed_vec[index_id];}return num_indexed;@@ -428,7 +410,7 @@ void Collection::populate_overrides(std::string query,const std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,std::map<size_t, std::vector<uint32_t>>& include_ids,- std::vector<uint32_t> & excluded_ids) {+ std::vector<uint32_t> & excluded_ids) const {StringUtils::tolowercase(query);std::set<uint32_t> excluded_set;@@ -513,7 +495,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const std::string& highlight_start_tag,const std::string& highlight_end_tag,std::vector<size_t> query_by_weights,- size_t limit_hits) {+ size_t limit_hits) const {++ std::shared_lock lock(mutex);if(query != "*" && search_fields.empty()) {return Option<nlohmann::json>(400, "No search fields specified for the query.");@@ -785,25 +769,39 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::vector<std::vector<std::string>> q_synonyms;synonym_reduction(q_include_tokens, q_synonyms);- // send data to individual index threads+ // search all indices+ std::vector<search_args*> search_args_vec;+ size_t num_processed = 0;+ std::mutex m_process;+ std::condition_variable cv_process;+size_t index_id = 0;for(Index* index: indices) {- index->search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,- filters, facets, index_to_included_ids[index_id],- index_to_excluded_ids[index_id],- sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,- per_page, page, token_order, prefix,- drop_tokens_threshold, typo_tokens_threshold,- group_by_fields, group_limit);- {- std::lock_guard<std::mutex> lk(index->m);- index->ready = true;- index->processed = false;- }- index->cv.notify_one();+ search_args* search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,+ filters, facets, index_to_included_ids[index_id],+ index_to_excluded_ids[index_id],+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,+ per_page, page, token_order, prefix,+ drop_tokens_threshold, typo_tokens_threshold,+ group_by_fields, group_limit);++ search_args_vec.push_back(search_params);++ CollectionManager::get_instance().get_thread_pool()->enqueue(+ [index, search_params, &m_process, &num_processed, &cv_process]() {+ index->run_search(search_params);+ std::unique_lock<std::mutex> lock(m_process);+ num_processed++;+ cv_process.notify_one();+ });+index_id++;}+ const size_t num_indices = indices.size();+ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_indices; });+Option<nlohmann::json> index_search_op({}); // stores the last error across all index threads// for grouping we have re-aggregate@@ -812,35 +810,19 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::Topster topster(topster_size, group_limit);Topster curated_topster(topster_size, group_limit);- for(Index* index: indices) {- // wait for the worker- {- std::unique_lock<std::mutex> lk(index->m);- index->cv.wait(lk, [index]{return index->processed;});- }+ for(const search_args* search_params: search_args_vec) {+ aggregate_topster(searched_queries.size(), topster, search_params->topster);+ aggregate_topster(searched_queries.size(), curated_topster, search_params->curated_topster);- if(!index->search_params->outcome.ok()) {- index_search_op = Option<nlohmann::json>(index->search_params->outcome.code(),- index->search_params->outcome.error());- }+ searched_queries.insert(searched_queries.end(), search_params->searched_queries.begin(),+ search_params->searched_queries.end());- if(!index_search_op.ok()) {- // we still need to iterate without breaking to release the locks- continue;- }-- aggregate_topster(searched_queries.size(), topster, index->search_params->topster);- aggregate_topster(searched_queries.size(), curated_topster, index->search_params->curated_topster);-- searched_queries.insert(searched_queries.end(), index->search_params->searched_queries.begin(),- index->search_params->searched_queries.end());-- for(size_t fi = 0; fi < index->search_params->facets.size(); fi++) {- auto & this_facet = index->search_params->facets[fi];+ for(size_t fi = 0; fi < search_params->facets.size(); fi++) {+ auto & this_facet = search_params->facets[fi];auto & acc_facet = facets[fi];for(auto & facet_kv: this_facet.result_map) {- if(index->search_params->group_limit) {+ if(search_params->group_limit) {// we have to add all group setsacc_facet.result_map[facet_kv.first].groups.insert(facet_kv.second.groups.begin(), facet_kv.second.groups.end()@@ -871,18 +853,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::if(group_limit) {groups_processed.insert(- index->search_params->groups_processed.begin(),- index->search_params->groups_processed.end()+ search_params->groups_processed.begin(),+ search_params->groups_processed.end());} else {- total_found += index->search_params->all_result_ids_len;+ total_found += search_params->all_result_ids_len;}}- if(!index_search_op.ok()) {- return index_search_op;- }-topster.sort();curated_topster.sort();@@ -946,7 +924,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::nlohmann::json result = nlohmann::json::object();result["found"] = total_found;- result["out_of"] = num_documents;+ result["out_of"] = num_documents.load();std::string hits_key = group_limit ? "grouped_hits" : "hits";result[hits_key] = nlohmann::json::array();@@ -1165,8 +1143,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}// free search params- for(Index* index: indices) {- delete index->search_params;+ for(auto search_params: search_args_vec) {+ delete search_params;}result["request_params"] = nlohmann::json::object();;@@ -1210,7 +1188,7 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s}}-void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) const {+void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) {if(topster->distinct) {for(auto &group_topster_entry: topster->group_kv_map) {Topster* group_topster = group_topster_entry.second;@@ -1226,7 +1204,7 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K}}-void Collection::aggregate_topster(size_t query_index, Topster& agg_topster, Topster* index_topster) const {+void Collection::aggregate_topster(size_t query_index, Topster& agg_topster, Topster* index_topster) {if(index_topster->distinct) {for(auto &group_topster_entry: index_topster->group_kv_map) {Topster* group_topster = group_topster_entry.second;@@ -1245,6 +1223,8 @@ void Collection::aggregate_topster(size_t query_index, Topster& agg_topster, TopOption<bool> Collection::get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids) {+ std::shared_lock lock(mutex);+std::vector<filter> filters;Option<bool> filter_op = parse_filter_query(simple_filter_query, filters);@@ -1254,12 +1234,7 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,for(auto& index: indices) {uint32_t* filter_ids = nullptr;- Option<uint32_t> op_filter_ids_length = index->do_filtering(&filter_ids, filters);- if(!op_filter_ids_length.ok()) {- return Option<bool>(op_filter_ids_length.code(), op_filter_ids_length.error());- }-- size_t filter_ids_len = op_filter_ids_length.get();+ size_t filter_ids_len = index->do_filtering(&filter_ids, filters);index_ids.emplace_back(filter_ids_len, filter_ids);}@@ -1267,7 +1242,7 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,}bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,- const nlohmann::json &document, std::string &value) {+ const nlohmann::json &document, std::string &value) const {if(document.count(a_facet.field_name) == 0) {LOG(ERROR) << "Could not find field " << a_facet.field_name << " in document during faceting.";@@ -1331,7 +1306,7 @@ void Collection::highlight_result(const field &search_field,bool highlighted_fully,const std::string& highlight_start_tag,const std::string& highlight_end_tag,- highlight_t & highlight) {+ highlight_t & highlight) const {if(searched_queries.size() <= field_order_kv->query_index) {return ;@@ -1494,11 +1469,7 @@ void Collection::free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) cons}}-bool Collection::doc_exists(const std::string & id) {- return store->contains(get_doc_id_key(id));-}--Option<nlohmann::json> Collection::get(const std::string & id) {+Option<nlohmann::json> Collection::get(const std::string & id) const {std::string seq_id_str;StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);@@ -1535,6 +1506,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) {}void Collection::remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {+ std::unique_lock lock(mutex);const std::string& id = document["id"];Index* index = indices[seq_id % num_memory_shards];@@ -1614,6 +1586,7 @@ Option<uint32_t> Collection::add_override(const override_t & override) {return Option<uint32_t>(500, "Error while storing the override on disk.");}+ std::unique_lock lock(mutex);overrides[override.id] = override;return Option<uint32_t>(200);}@@ -1624,6 +1597,8 @@ Option<uint32_t> Collection::remove_override(const std::string & id) {if(!removed) {return Option<uint32_t>(500, "Error while deleting the override from disk.");}++ std::unique_lock lock(mutex);overrides.erase(id);return Option<uint32_t>(200);}@@ -1645,34 +1620,34 @@ std::string Collection::get_next_seq_id_key(const std::string & collection_name)return std::string(COLLECTION_NEXT_SEQ_PREFIX) + "_" + collection_name;}-std::string Collection::get_seq_id_key(uint32_t seq_id) {+std::string Collection::get_seq_id_key(uint32_t seq_id) const {// We can't simply do std::to_string() because we want to preserve the byte order.// & 0xFF masks all but the lowest eight bits.const std::string & serialized_id = StringUtils::serialize_uint32_t(seq_id);return get_seq_id_collection_prefix() + "_" + serialized_id;}-std::string Collection::get_doc_id_key(const std::string & doc_id) {+std::string Collection::get_doc_id_key(const std::string & doc_id) const {return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_" + doc_id;}-std::string Collection::get_name() {+std::string Collection::get_name() const {return name;}-uint64_t Collection::get_created_at() {+uint64_t Collection::get_created_at() const {return created_at;}-size_t Collection::get_num_documents() {- return num_documents;+size_t Collection::get_num_documents() const {+ return num_documents.load();}-uint32_t Collection::get_collection_id() {+uint32_t Collection::get_collection_id() const {return collection_id;}-Option<uint32_t> Collection::doc_id_to_seq_id(const std::string & doc_id) {+Option<uint32_t> Collection::doc_id_to_seq_id(const std::string & doc_id) const {std::string seq_id_str;StoreStatus status = store->get(get_doc_id_key(doc_id), seq_id_str);if(status == StoreStatus::FOUND) {@@ -1725,7 +1700,7 @@ std::string Collection::get_synonym_key(const std::string & collection_name, conreturn std::string(COLLECTION_SYNONYM_PREFIX) + "_" + collection_name + "_" + synonym_id;}-std::string Collection::get_seq_id_collection_prefix() {+std::string Collection::get_seq_id_collection_prefix() const {return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);}@@ -1733,7 +1708,7 @@ std::string Collection::get_default_sorting_field() {return default_sorting_field;}-Option<bool> Collection::get_document_from_store(const std::string &seq_id_key, nlohmann::json & document) {+Option<bool> Collection::get_document_from_store(const std::string &seq_id_key, nlohmann::json & document) const {std::string json_doc_str;StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);@@ -1755,7 +1730,7 @@ const std::vector<Index *> &Collection::_get_indexes() const {}Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,- std::vector<filter>& filters) {+ std::vector<filter>& filters) const {std::vector<std::string> filter_blocks;StringUtils::split(simple_filter_query, filter_blocks, "&&");@@ -1947,7 +1922,8 @@ Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,return Option<bool>(true);}-void Collection::synonym_reduction(const std::vector<std::string>& tokens, std::vector<std::vector<std::string>>& results) {+void Collection::synonym_reduction(const std::vector<std::string>& tokens,+ std::vector<std::vector<std::string>>& results) const {std::set<uint64_t> processed_syn_hashes;synonym_reduction_internal(tokens, tokens.size(), 0, processed_syn_hashes, results);}@@ -1961,6 +1937,7 @@ Option<bool> Collection::add_synonym(const synonym_t& synonym) {}}+ std::unique_lock write_lock(mutex);synonym_definitions[synonym.id] = synonym;if(!synonym.root.empty()) {@@ -1973,6 +1950,8 @@ Option<bool> Collection::add_synonym(const synonym_t& synonym) {}}+ write_lock.unlock();+bool inserted = store->insert(Collection::get_synonym_key(name, synonym.id), synonym.to_json().dump());if(!inserted) {return Option<bool>(500, "Error while storing the synonym on disk.");@@ -1982,6 +1961,8 @@ Option<bool> Collection::add_synonym(const synonym_t& synonym) {}bool Collection::get_synonym(const std::string& id, synonym_t& synonym) {+ std::shared_lock lock(mutex);+if(synonym_definitions.count(id) != 0) {synonym = synonym_definitions[id];return true;@@ -1993,7 +1974,7 @@ bool Collection::get_synonym(const std::string& id, synonym_t& synonym) {void Collection::synonym_reduction_internal(const std::vector<std::string>& tokens,size_t start_window_size, size_t start_index_pos,std::set<uint64_t>& processed_syn_hashes,- std::vector<std::vector<std::string>>& results) {+ std::vector<std::vector<std::string>>& results) const {bool recursed = false;@@ -2021,7 +2002,7 @@ void Collection::synonym_reduction_internal(const std::vector<std::string>& tokeconst auto& syn_ids = syn_itr->second;for(const auto& syn_id: syn_ids) {- const auto &syn_def = synonym_definitions[syn_id];+ const auto &syn_def = synonym_definitions.at(syn_id);for (const auto &syn_def_tokens: syn_def.synonyms) {std::vector<std::string> new_tokens;@@ -2085,6 +2066,7 @@ void Collection::synonym_reduction_internal(const std::vector<std::string>& toke}Option<bool> Collection::remove_synonym(const std::string &id) {+ std::unique_lock lock(mutex);const auto& syn_iter = synonym_definitions.find(id);if(syn_iter != synonym_definitions.end()) {@@ -2111,6 +2093,30 @@ Option<bool> Collection::remove_synonym(const std::string &id) {return Option<bool>(404, "Could not find that `id`.");}-spp::sparse_hash_map<std::string, synonym_t> &Collection::get_synonyms() {+spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {+ std::shared_lock lock(mutex);return synonym_definitions;}++std::vector<Index *> Collection::init_indices() {+ std::vector<Index*> index_list;++ for(const field& field: fields) {+ search_schema.emplace(field.name, field);++ if(field.is_facet()) {+ facet_schema.emplace(field.name, field);+ }++ if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {+ sort_schema.emplace(field.name, field);+ }+ }++ for(size_t i = 0; i < num_memory_shards; i++) {+ Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);+ index_list.push_back(index);+ }++ return index_list;+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 08153a2d..85684246 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -58,14 +58,23 @@ void CollectionManager::add_to_collections(Collection* collection) {collection_id_names.emplace(collection->get_collection_id(), collection->get_name());}-void CollectionManager::init(Store *store,+void CollectionManager::init(Store *store, ThreadPool* thread_pool,const float max_memory_ratio,const std::string & auth_key) {+ std::unique_lock lock(mutex);+this->store = store;+ this->thread_pool = thread_pool;this->bootstrap_auth_key = auth_key;this->max_memory_ratio = max_memory_ratio;}+// used only in tests!+void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key) {+ ThreadPool* thread_pool = new ThreadPool(8);+ init(store, thread_pool, max_memory_ratio, auth_key);+}+Option<bool> CollectionManager::load(const size_t init_batch_size) {// This function must be idempotent, i.e. when called multiple times, must produce the same state without leaksLOG(INFO) << "CollectionManager::load()";@@ -120,12 +129,13 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {uint32_t collection_next_seq_id = next_seq_id_status == StoreStatus::NOT_FOUND ? 0 :StringUtils::deserialize_uint32_t(collection_next_seq_id_str);- if(get_collection(this_collection_name)) {+ if(get_collection(this_collection_name) != nullptr) {// To maintain idempotency, if the collection already exists in-memory, drop it from memoryLOG(WARNING) << "Dropping duplicate collection " << this_collection_name << " before loading it again.";drop_collection(this_collection_name, false);}+ std::unique_lock lock(mutex);Collection* collection = init_collection(collection_meta, collection_next_seq_id);LOG(INFO) << "Loading collection " << collection->get_name();@@ -252,6 +262,8 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {void CollectionManager::dispose() {+ std::unique_lock lock(mutex);+for(auto & name_collection: collections) {delete name_collection.second;name_collection.second = nullptr;@@ -264,7 +276,9 @@ void CollectionManager::dispose() {bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,const std::string& action,const std::string& collection,- std::map<std::string, std::string>& params) {+ std::map<std::string, std::string>& params) const {+ std::shared_lock lock(mutex);+if(auth_key_sent.empty()) {return false;}@@ -283,6 +297,8 @@ Option<Collection*> CollectionManager::create_collection(const std::string& nameconst std::vector<field> & fields,const std::string & default_sorting_field,const uint64_t created_at) {+ std::unique_lock lock(mutex);+if(store->contains(Collection::get_meta_key(name))) {return Option<Collection*>(409, std::string("A collection with name `") + name + "` already exists.");}@@ -328,7 +344,7 @@ Option<Collection*> CollectionManager::create_collection(const std::string& namenlohmann::json collection_meta;collection_meta[COLLECTION_NAME_KEY] = name;- collection_meta[COLLECTION_ID_KEY] = next_collection_id;+ collection_meta[COLLECTION_ID_KEY] = next_collection_id.load();collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;collection_meta[COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;collection_meta[COLLECTION_CREATED] = created_at;@@ -354,7 +370,7 @@ Option<Collection*> CollectionManager::create_collection(const std::string& namereturn Option<Collection*>(new_collection);}-Collection* CollectionManager::get_collection(const std::string & collection_name) {+Collection* CollectionManager::get_collection_unafe(const std::string & collection_name) const {if(collections.count(collection_name) != 0) {return collections.at(collection_name);}@@ -370,17 +386,27 @@ Collection* CollectionManager::get_collection(const std::string & collection_namreturn nullptr;}-Collection* CollectionManager::get_collection_with_id(uint32_t collection_id) {+locked_resource_view_t<Collection> CollectionManager::get_collection(const std::string & collection_name) const {+ std::shared_lock lock(mutex);+ Collection* coll = get_collection_unafe(collection_name);+ return locked_resource_view_t<Collection>(mutex, coll);+}++locked_resource_view_t<Collection> CollectionManager::get_collection_with_id(uint32_t collection_id) const {+ std::shared_lock lock(mutex);+if(collection_id_names.count(collection_id) != 0) {return get_collection(collection_id_names.at(collection_id));}- return nullptr;+ return locked_resource_view_t<Collection>(mutex, nullptr);}-std::vector<Collection*> CollectionManager::get_collections() {+std::vector<Collection*> CollectionManager::get_collections() const {+ std::shared_lock lock(mutex);+std::vector<Collection*> collection_vec;- for(auto kv: collections) {+ for(const auto& kv: collections) {collection_vec.push_back(kv.second);}@@ -392,12 +418,16 @@ std::vector<Collection*> CollectionManager::get_collections() {return collection_vec;}-Option<bool> CollectionManager::drop_collection(const std::string& collection_name, const bool remove_from_store) {- Collection* collection = get_collection(collection_name);+Option<nlohmann::json> CollectionManager::drop_collection(const std::string& collection_name, const bool remove_from_store) {+ std::unique_lock lock(mutex);++ Collection* collection = get_collection_unafe(collection_name);if(collection == nullptr) {- return Option<bool>(404, "No collection with name `" + collection_name + "` found.");+ return Option<nlohmann::json>(404, "No collection with name `" + collection_name + "` found.");}+ nlohmann::json collection_json = collection->get_summary_json();+if(remove_from_store) {const std::string &collection_id_str = std::to_string(collection->get_collection_id());@@ -418,10 +448,10 @@ Option<bool> CollectionManager::drop_collection(const std::string& collection_nadelete collection;- return Option<bool>(true);+ return Option<nlohmann::json>(collection_json);}-uint32_t CollectionManager::get_next_collection_id() {+uint32_t CollectionManager::get_next_collection_id() const {return next_collection_id;}@@ -429,15 +459,14 @@ std::string CollectionManager::get_symlink_key(const std::string & symlink_name)return std::string(SYMLINK_PREFIX) + "_" + symlink_name;}-void CollectionManager::set_next_collection_id(uint32_t next_id) {- next_collection_id = next_id;-}--spp::sparse_hash_map<std::string, std::string> & CollectionManager::get_symlinks() {+spp::sparse_hash_map<std::string, std::string> CollectionManager::get_symlinks() const {+ std::shared_lock lock(mutex);return collection_symlinks;}-Option<std::string> CollectionManager::resolve_symlink(const std::string & symlink_name) {+Option<std::string> CollectionManager::resolve_symlink(const std::string & symlink_name) const {+ std::shared_lock lock(mutex);+if(collection_symlinks.count(symlink_name) != 0) {return Option<std::string>(collection_symlinks.at(symlink_name));}@@ -446,6 +475,8 @@ Option<std::string> CollectionManager::resolve_symlink(const std::string & symli}Option<bool> CollectionManager::upsert_symlink(const std::string & symlink_name, const std::string & collection_name) {+ std::unique_lock lock(mutex);+if(collections.count(symlink_name) != 0) {return Option<bool>(500, "Name `" + symlink_name + "` conflicts with an existing collection name.");}@@ -460,6 +491,8 @@ Option<bool> CollectionManager::upsert_symlink(const std::string & symlink_name,}Option<bool> CollectionManager::delete_symlink(const std::string & symlink_name) {+ std::unique_lock lock(mutex);+bool removed = store->remove(get_symlink_key(symlink_name));if(!removed) {return Option<bool>(500, "Unable to delete from store.");@@ -711,7 +744,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req_params["collection"]);+ auto collection = collectionManager.get_collection(req_params["collection"]);if(collection == nullptr) {return Option<bool>(404, "Not found.");@@ -767,3 +800,21 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rereturn Option<bool>(true);}++ThreadPool* CollectionManager::get_thread_pool() const {+ return thread_pool;+}++nlohmann::json CollectionManager::get_collection_summaries() const {+ std::shared_lock lock(mutex);++ std::vector<Collection*> colls = get_collections();+ nlohmann::json json_summaries = nlohmann::json::array();++ for(Collection* collection: colls) {+ nlohmann::json collection_json = collection->get_summary_json();+ json_summaries.push_back(collection_json);+ }++ return json_summaries;+}diff --git a/src/core_api.cpp b/src/core_api.cppindex ffbe8f38..56adb91a 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -43,14 +43,7 @@ index_operation_t get_index_operation(const std::string& action) {bool get_collections(http_req & req, http_res & res) {CollectionManager & collectionManager = CollectionManager::get_instance();- std::vector<Collection*> collections = collectionManager.get_collections();- nlohmann::json json_response = nlohmann::json::array();-- for(Collection* collection: collections) {- nlohmann::json collection_json = collection->get_summary_json();- json_response.push_back(collection_json);- }-+ nlohmann::json json_response = collectionManager.get_collection_summaries();res.set_200(json_response.dump());return true;}@@ -170,22 +163,14 @@ bool post_create_collection(http_req & req, http_res & res) {bool del_drop_collection(http_req & req, http_res & res) {std::string doc_id = req.params["id"];CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ Option<nlohmann::json> drop_op = collectionManager.drop_collection(req.params["collection"]);- if(!collection) {- res.set_404();+ if(!drop_op.ok()) {+ res.set(drop_op.code(), drop_op.error());return false;}- nlohmann::json collection_json = collection->get_summary_json();- Option<bool> drop_result = collectionManager.drop_collection(req.params["collection"]);-- if(!drop_result.ok()) {- res.set(drop_result.code(), drop_result.error());- return false;- }-- res.set_200(collection_json.dump());+ res.set_200(drop_op.get().dump());return true;}@@ -349,8 +334,8 @@ bool post_multi_search(http_req& req, http_res& res) {}bool get_collection_summary(http_req & req, http_res & res) {- CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ CollectionManager& collectionManager = CollectionManager::get_instance();+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -366,7 +351,7 @@ bool get_collection_summary(http_req & req, http_res & res) {bool get_export_documents(http_req & req, http_res & res) {// NOTE: this is a streaming response end-point so this handler will be called multiple timesCollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {req.last_chunk_aggregate = true;@@ -459,7 +444,7 @@ bool post_import_documents(http_req& req, http_res& res) {}CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {req.last_chunk_aggregate = true;@@ -564,7 +549,7 @@ bool post_add_document(http_req & req, http_res & res) {}CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -587,7 +572,7 @@ bool patch_update_document(http_req & req, http_res & res) {std::string doc_id = req.params["id"];CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -609,7 +594,7 @@ bool get_fetch_document(http_req & req, http_res & res) {std::string doc_id = req.params["id"];CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();return false;@@ -630,7 +615,7 @@ bool del_remove_document(http_req & req, http_res & res) {std::string doc_id = req.params["id"];CollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();return false;@@ -662,7 +647,7 @@ bool del_remove_documents(http_req & req, http_res & res) {// NOTE: this is a streaming response end-point so this handler will be called multiple timesCollectionManager & collectionManager = CollectionManager::get_instance();- Collection* collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {req.last_chunk_aggregate = true;@@ -728,7 +713,7 @@ bool del_remove_documents(http_req & req, http_res & res) {for(size_t i=0; i<deletion_state->index_ids.size(); i++) {deletion_state->offsets.push_back(0);}- deletion_state->collection = collection;+ deletion_state->collection = collection.get();deletion_state->num_removed = 0;req.data = deletion_state;} else {@@ -855,7 +840,7 @@ bool del_alias(http_req & req, http_res & res) {bool get_overrides(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -865,7 +850,7 @@ bool get_overrides(http_req &req, http_res &res) {nlohmann::json res_json;res_json["overrides"] = nlohmann::json::array();- std::map<std::string, override_t>& overrides = collection->get_overrides();+ const std::map<std::string, override_t>& overrides = collection->get_overrides();for(const auto & kv: overrides) {nlohmann::json override = kv.second.to_json();res_json["overrides"].push_back(override);@@ -877,7 +862,7 @@ bool get_overrides(http_req &req, http_res &res) {bool get_override(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -886,10 +871,10 @@ bool get_override(http_req &req, http_res &res) {std::string override_id = req.params["id"];- std::map<std::string, override_t>& overrides = collection->get_overrides();+ const std::map<std::string, override_t>& overrides = collection->get_overrides();if(overrides.count(override_id) != 0) {- nlohmann::json override = overrides[override_id].to_json();+ nlohmann::json override = overrides.at(override_id).to_json();res.set_200(override.dump());return true;}@@ -900,7 +885,7 @@ bool get_override(http_req &req, http_res &res) {bool put_override(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);std::string override_id = req.params["id"];@@ -941,7 +926,7 @@ bool put_override(http_req &req, http_res &res) {bool del_override(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -1170,7 +1155,7 @@ bool post_config(http_req &req, http_res &res) {bool get_synonyms(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -1180,7 +1165,7 @@ bool get_synonyms(http_req &req, http_res &res) {nlohmann::json res_json;res_json["synonyms"] = nlohmann::json::array();- auto& synonyms = collection->get_synonyms();+ const auto& synonyms = collection->get_synonyms();for(const auto & kv: synonyms) {nlohmann::json synonym = kv.second.to_view_json();res_json["synonyms"].push_back(synonym);@@ -1192,7 +1177,7 @@ bool get_synonyms(http_req &req, http_res &res) {bool get_synonym(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();@@ -1216,7 +1201,7 @@ bool get_synonym(http_req &req, http_res &res) {bool put_synonym(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);std::string synonym_id = req.params["id"];@@ -1258,7 +1243,7 @@ bool put_synonym(http_req &req, http_res &res) {bool del_synonym(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection = collectionManager.get_collection(req.params["collection"]);+ auto collection = collectionManager.get_collection(req.params["collection"]);if(collection == nullptr) {res.set_404();diff --git a/src/http_server.cpp b/src/http_server.cppindex e9f71d8a..4db5e12d 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -13,9 +13,10 @@HttpServer::HttpServer(const std::string & version, const std::string & listen_address,uint32_t listen_port, const std::string & ssl_cert_path,- const std::string & ssl_cert_key_path, bool cors_enabled):+ const std::string & ssl_cert_key_path, bool cors_enabled, ThreadPool* thread_pool):version(version), listen_address(listen_address), listen_port(listen_port),- ssl_cert_path(ssl_cert_path), ssl_cert_key_path(ssl_cert_key_path), cors_enabled(cors_enabled) {+ ssl_cert_path(ssl_cert_path), ssl_cert_key_path(ssl_cert_key_path),+ cors_enabled(cors_enabled), thread_pool(thread_pool) {accept_ctx = new h2o_accept_ctx_t();h2o_config_init(&config);hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);@@ -544,6 +545,23 @@ int HttpServer::process_request(http_req* request, http_res* response, route_patreturn 0;}+ auto message_dispatcher = handler->http_server->get_message_dispatcher();+ auto api_handler = rpath->handler;++ handler->http_server->get_thread_pool()->enqueue([api_handler, message_dispatcher, request, response]() {+ (api_handler)(*request, *response);+ request_response req_res{request, response};+ response->auto_dispose = false;+ message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, &req_res);++ // wait until response is sent+ response->await.wait();++ delete request;+ delete response;+ });++ /*// for reads, we will invoke the request handler and handle response as well(rpath->handler)(*request, *response);@@ -552,6 +570,7 @@ int HttpServer::process_request(http_req* request, http_res* response, route_pat// otherwise, we send the whole body response on behalf of the handlerhandler->http_server->send_response(request, response);}+ */return 0;}@@ -861,3 +880,7 @@ void HttpServer::do_snapshot(const std::string& snapshot_path, http_req& req, htbool HttpServer::trigger_vote() {return replication_state->trigger_vote();}++ThreadPool* HttpServer::get_thread_pool() const {+ return thread_pool;+}diff --git a/src/index.cpp b/src/index.cppindex 2ec98834..9975a91a 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -39,10 +39,6 @@ Index::Index(const std::string name, const std::unordered_map<std::string, field}num_documents = 0;-- ready = false;- processed = false;- terminate = false;}Index::~Index() {@@ -98,6 +94,8 @@ int64_t Index::float_to_in64_t(float f) {Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,const std::string & default_sorting_field, bool is_update) {+ std::unique_lock lock(mutex);+int64_t points = 0;if(is_update && document.count(default_sorting_field) == 0) {@@ -640,7 +638,8 @@ void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::s}void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,- const uint32_t* result_ids, size_t results_size) {+ size_t group_limit, const std::vector<std::string>& group_by_fields,+ const uint32_t* result_ids, size_t results_size) const {std::unordered_map<std::string, size_t> facet_to_index;get_facet_to_index(facet_to_index);@@ -723,7 +722,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}const std::vector<std::vector<uint64_t>>& doc_facet_index = doc_facet_index_it->second;- const uint64_t distinct_id = search_params->group_limit ? get_distinct_id(facet_to_index, doc_seq_id) : 0;+ const uint64_t distinct_id = group_limit ? get_distinct_id(facet_to_index, group_by_fields, doc_seq_id) : 0;// assumed that facet fields have already been validated upstreamfor(size_t findex=0; findex < facets.size(); findex++) {@@ -797,7 +796,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,facet_count.doc_id = doc_seq_id;facet_count.array_pos = array_pos;- if(search_params->group_limit) {+ if(group_limit) {facet_count.groups.emplace(distinct_id);} else {facet_count.count += 1;@@ -819,7 +818,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}}-void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) {+void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) const {size_t i_facet = 0;for(const auto & facet_kv: facet_schema) {facet_to_index.emplace(facet_kv.first, i_facet);@@ -837,7 +836,8 @@ void Index::search_candidates(const uint8_t & field_id,spp::sparse_hash_set<uint64_t>& groups_processed,uint32_t** all_result_ids, size_t & all_result_ids_len,size_t& field_num_results,- const size_t typo_tokens_threshold) {+ const size_t typo_tokens_threshold,+ const size_t group_limit, const std::vector<std::string>& group_by_fields) const {const long long combination_limit = 10;auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };@@ -921,7 +921,8 @@ void Index::search_candidates(const uint8_t & field_id,// go through each matching document id and calculate match scorescore_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,- groups_processed, filtered_result_ids, filtered_results_size, token_bits);+ groups_processed, filtered_result_ids, filtered_results_size,+ group_limit, group_by_fields, token_bits);field_num_results += filtered_results_size;@@ -939,7 +940,7 @@ void Index::search_candidates(const uint8_t & field_id,}*/score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,- groups_processed, result_ids, result_size, token_bits);+ groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits);field_num_results += result_size;@@ -955,7 +956,7 @@ void Index::search_candidates(const uint8_t & field_id,}}-Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {+uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const {//auto begin = std::chrono::high_resolution_clock::now();uint32_t* filter_ids = nullptr;@@ -1078,7 +1079,7 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectosize_t facet_id = facet_to_index[f.name];for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {uint32_t seq_id = strt_ids[strt_ids_index];- const std::vector<uint64_t> &fvalues = facet_index_v2[seq_id][facet_id];+ const std::vector<uint64_t> &fvalues = facet_index_v2.at(seq_id).at(facet_id);bool found_filter = false;if(!f.is_array()) {@@ -1156,7 +1157,7 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectoLOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/*filter_ids_out = filter_ids;- return Option<>(filter_ids_length);+ return filter_ids_length;}void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,@@ -1212,46 +1213,28 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,}}-void Index::run_search() {- while(true) {- // wait until main thread sends data- std::unique_lock<std::mutex> lk(m);- cv.wait(lk, [this]{return ready;});-- if(terminate) {- break;- }-- // after the wait, we own the lock.- search(search_params->outcome, search_params->q_include_tokens, search_params->q_exclude_tokens,- search_params->q_synonyms,- search_params->search_fields,- search_params->filters, search_params->facets, search_params->facet_query,- search_params->included_ids, search_params->excluded_ids,- search_params->sort_fields_std, search_params->num_typos,- search_params->topster, search_params->curated_topster,- search_params->per_page, search_params->page, search_params->token_order,- search_params->prefix, search_params->drop_tokens_threshold,- search_params->all_result_ids_len, search_params->groups_processed,- search_params->searched_queries,- search_params->raw_result_kvs, search_params->override_result_kvs,- search_params->typo_tokens_threshold);-- // hand control back to main thread- processed = true;- ready = false;-- // manual unlocking is done before notifying, to avoid waking up the waiting thread only to block again- lk.unlock();- cv.notify_one();- }+void Index::run_search(search_args* search_params) {+ search(search_params->q_include_tokens, search_params->q_exclude_tokens,+ search_params->q_synonyms,+ search_params->search_fields,+ search_params->filters, search_params->facets, search_params->facet_query,+ search_params->included_ids, search_params->excluded_ids,+ search_params->sort_fields_std, search_params->num_typos,+ search_params->topster, search_params->curated_topster,+ search_params->per_page, search_params->page, search_params->token_order,+ search_params->prefix, search_params->drop_tokens_threshold,+ search_params->all_result_ids_len, search_params->groups_processed,+ search_params->searched_queries,+ search_params->raw_result_kvs, search_params->override_result_kvs,+ search_params->typo_tokens_threshold,+ search_params->group_limit, search_params->group_by_fields);}void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,const std::string & field, const uint8_t field_id,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,Topster* curated_topster,- std::vector<std::vector<art_leaf*>> & searched_queries) {+ std::vector<std::vector<art_leaf*>> & searched_queries) const {if(included_ids_map.empty()) {return;@@ -1316,8 +1299,7 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,}}-void Index::search(Option<uint32_t> & outcome,- const std::vector<std::string>& q_include_tokens,+void Index::search(const std::vector<std::string>& q_include_tokens,const std::vector<std::string>& q_exclude_tokens,const std::vector<std::vector<std::string>>& q_synonyms,const std::vector<search_field_t>& search_fields,@@ -1335,20 +1317,18 @@ void Index::search(Option<uint32_t> & outcome,std::vector<std::vector<art_leaf*>>& searched_queries,std::vector<std::vector<KV*>> & raw_result_kvs,std::vector<std::vector<KV*>> & override_result_kvs,- const size_t typo_tokens_threshold) {+ const size_t typo_tokens_threshold,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields) const {++ std::shared_lock lock(mutex);//auto begin = std::chrono::high_resolution_clock::now();// process the filtersuint32_t* filter_ids = nullptr;- Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, filters);- if(!op_filter_ids_length.ok()) {- outcome = Option<uint32_t>(op_filter_ids_length);- return ;- }-- uint32_t filter_ids_length = op_filter_ids_length.get();+ uint32_t filter_ids_length = do_filtering(&filter_ids, filters);// we will be removing all curated IDs from organic result ids before running topsterstd::set<uint32_t> curated_ids;@@ -1413,7 +1393,7 @@ void Index::search(Option<uint32_t> & outcome,}}- const spp::sparse_hash_map<uint32_t, int64_t> *kvs = sort_index[all_records_field];+ const spp::sparse_hash_map<uint32_t, int64_t> *kvs = sort_index.at(all_records_field);filter_ids_length = kvs->size();filter_ids = new uint32_t[filter_ids_length];@@ -1445,7 +1425,7 @@ void Index::search(Option<uint32_t> & outcome,uint32_t token_bits = 255;score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},- groups_processed, filter_ids, filter_ids_length, token_bits);+ groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits);collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);all_result_ids_len = filter_ids_length;@@ -1494,7 +1474,8 @@ void Index::search(Option<uint32_t> & outcome,search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,- field_num_results, token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);+ field_num_results, group_limit, group_by_fields, token_order, prefix,+ drop_tokens_threshold, typo_tokens_threshold);// do synonym based searchesfor(const auto& syn_tokens: q_pos_synonyms) {@@ -1505,7 +1486,8 @@ void Index::search(Option<uint32_t> & outcome,search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,- field_num_results, token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);+ field_num_results, group_limit, group_by_fields, token_order, prefix,+ drop_tokens_threshold, typo_tokens_threshold);}concat_topster_ids(ftopster, topster_ids);@@ -1613,8 +1595,8 @@ void Index::search(Option<uint32_t> & outcome,delete [] exclude_token_ids;- do_facets(facets, facet_query, all_result_ids, all_result_ids_len);- do_facets(facets, facet_query, &included_ids[0], included_ids.size());+ do_facets(facets, facet_query, group_limit, group_by_fields, all_result_ids, all_result_ids_len);+ do_facets(facets, facet_query, group_limit, group_by_fields, &included_ids[0], included_ids.size());all_result_ids_len += curated_topster->size;@@ -1625,10 +1607,9 @@ void Index::search(Option<uint32_t> & outcome,delete ftopster;}+ //LOG(INFO) << "all_result_ids_len " << all_result_ids_len << " for index " << name;//long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();//LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";-- outcome = Option<uint32_t>(1);}/*@@ -1652,10 +1633,10 @@ void Index::search_field(const uint8_t & field_id,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,std::vector<std::vector<art_leaf*>> & searched_queries,Topster* topster, spp::sparse_hash_set<uint64_t>& groups_processed,- uint32_t** all_result_ids, size_t & all_result_ids_len,- size_t& field_num_results,+ uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,const token_ordering token_order, const bool prefix,- const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) {+ const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) const {const size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;@@ -1758,7 +1739,7 @@ void Index::search_field(const uint8_t & field_id,search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,groups_processed, all_result_ids, all_result_ids_len, field_num_results,- typo_tokens_threshold);+ typo_tokens_threshold, group_limit, group_by_fields);}resume_typo_loop:@@ -1796,11 +1777,12 @@ void Index::search_field(const uint8_t & field_id,return search_field(field_id, query_tokens, truncated_tokens, exclude_token_ids, exclude_token_ids_size,num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids,facets,sort_fields, num_typos,searched_queries, topster, groups_processed, all_result_ids,- all_result_ids_len, field_num_results, token_order, prefix);+ all_result_ids_len, field_num_results, group_limit, group_by_fields,+ token_order, prefix);}}-int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const {+int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) {int bounded_cost = max_cost;if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {bounded_cost = token_len - 1;@@ -1825,7 +1807,9 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const uint8_t & field_id, const uint32_t total_cost, Topster* topster,const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const uint32_t *result_ids, const size_t result_size, uint32_t token_bits) {+ const uint32_t *result_ids, const size_t result_size,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,+ uint32_t token_bits) const {std::vector<uint32_t*> leaf_to_indices;for (art_leaf *token_leaf: query_suggestion) {@@ -1935,8 +1919,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16uint64_t distinct_id = seq_id;- if(search_params->group_limit != 0) {- distinct_id = get_distinct_id(facet_to_index, seq_id);+ if(group_limit != 0) {+ distinct_id = get_distinct_id(facet_to_index, group_by_fields, seq_id);groups_processed.emplace(distinct_id);}@@ -1953,11 +1937,12 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}uint64_t Index::get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id,+ const std::vector<std::string>& group_by_fields,const uint32_t seq_id) const {uint64_t distinct_id = 1; // some constant initial value// calculate hash from group_by_fields- for(const auto& field: search_params->group_by_fields) {+ for(const auto& field: group_by_fields) {if(facet_to_id.count(field) == 0 || facet_index_v2.count(seq_id) == 0) {continue;}@@ -1973,8 +1958,8 @@ uint64_t Index::get_distinct_id(const std::unordered_map<std::string, size_t> &f}void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,- std::vector<uint32_t*>& leaf_to_indices,- size_t result_index,+ const std::vector<uint32_t*>& leaf_to_indices,+ const size_t result_index,std::unordered_map<size_t, std::vector<std::vector<uint16_t>>>& array_token_positions) {if(query_suggestion.empty()) {return ;@@ -2102,6 +2087,8 @@ void Index::remove_and_shift_offset_index(sorted_array& offset_index, const uint}Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document) {+ std::unique_lock lock(mutex);+std::unordered_map<std::string, size_t> facet_to_index;get_facet_to_index(facet_to_index);diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex d24be610..ee405d83 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -360,9 +360,11 @@ int run_server(const Config & config, const std::string & version, void (*mastercreate_init_db_snapshot = true;}+ ThreadPool thread_pool(32);Store store(db_dir);+CollectionManager & collectionManager = CollectionManager::get_instance();- collectionManager.init(&store, config.get_max_memory_ratio(), config.get_api_key());+ collectionManager.init(&store, &thread_pool, config.get_max_memory_ratio(), config.get_api_key());curl_global_init(CURL_GLOBAL_SSL);HttpClient & httpClient = HttpClient::get_instance();@@ -374,7 +376,8 @@ int run_server(const Config & config, const std::string & version, void (*masterconfig.get_api_port(),config.get_ssl_cert(),config.get_ssl_cert_key(),- config.get_enable_cors()+ config.get_enable_cors(),+ &thread_pool);server->set_auth_handler(handle_authentication);@@ -388,7 +391,6 @@ int run_server(const Config & config, const std::string & version, void (*master// first we start the peering service- ThreadPool thread_pool(32);ReplicationState replication_state(&store, &thread_pool, server->get_message_dispatcher(),ssl_enabled, config.get_catch_up_threshold_percentage(),create_init_db_snapshot, quit_raft_service);diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex e192a2c2..d1d90763 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -47,7 +47,7 @@ TEST_F(CollectionFacetingTest, FacetFieldStringFiltering) {std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };- coll_str = collectionManager.get_collection("coll_str");+ coll_str = collectionManager.get_collection("coll_str").get();if(coll_str == nullptr) {coll_str = collectionManager.create_collection("coll_str", 1, fields, "points").get();}@@ -112,7 +112,7 @@ TEST_F(CollectionFacetingTest, FacetFieldStringArrayFiltering) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if(coll_array_fields == nullptr) {coll_array_fields = collectionManager.create_collection("coll_array_fields", 1, fields, "age").get();}@@ -194,7 +194,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if(coll_array_fields == nullptr) {coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();}@@ -504,7 +504,7 @@ TEST_F(CollectionFacetingTest, FacetCountsBool) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -558,7 +558,7 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -694,7 +694,7 @@ TEST_F(CollectionFacetingTest, FacetStatOnFloatFields) {std::vector<sort_by> sort_fields_desc = { sort_by("average", "DESC") };- coll_float_fields = collectionManager.get_collection("coll_float_fields");+ coll_float_fields = collectionManager.get_collection("coll_float_fields").get();if(coll_float_fields == nullptr) {coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "average").get();}@@ -756,7 +756,7 @@ TEST_F(CollectionFacetingTest, FacetCountOnSimilarStrings) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 62982f19..81dcd370 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -47,7 +47,7 @@ TEST_F(CollectionFilteringTest, FilterOnTextFields) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if(coll_array_fields == nullptr) {coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();}@@ -127,7 +127,7 @@ TEST_F(CollectionFilteringTest, FilterOnTextFieldWithColon) {std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -165,7 +165,7 @@ TEST_F(CollectionFilteringTest, HandleBadlyFormedFilterQuery) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if(coll_array_fields == nullptr) {coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();}@@ -219,7 +219,7 @@ TEST_F(CollectionFilteringTest, FilterAndQueryFieldRestrictions) {field("points", field_types::INT32, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -332,7 +332,7 @@ TEST_F(CollectionFilteringTest, FilterOnNumericFields) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if(coll_array_fields == nullptr) {// ensure that default_sorting_field is a non-array numerical fieldauto coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "years");@@ -529,7 +529,7 @@ TEST_F(CollectionFilteringTest, FilterOnFloatFields) {std::vector<sort_by> sort_fields_desc = { sort_by("rating", "DESC") };std::vector<sort_by> sort_fields_asc = { sort_by("rating", "ASC") };- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if(coll_array_fields == nullptr) {coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();}@@ -669,7 +669,7 @@ TEST_F(CollectionFilteringTest, ComparatorsOnMultiValuedNumericalField) {std::vector<sort_by> sort_fields_desc = {sort_by("rating", "DESC")};- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if (coll_array_fields == nullptr) {coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();}@@ -724,7 +724,7 @@ TEST_F(CollectionFilteringTest, FilteringWithPrefixSearch) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 6f35b2da..06f2bde8 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -29,7 +29,7 @@ protected:field("rating", field_types::FLOAT, true, false)};- coll_group = collectionManager.get_collection("coll_group");+ coll_group = collectionManager.get_collection("coll_group").get();if(coll_group == nullptr) {coll_group = collectionManager.create_collection("coll_group", 4, fields, "rating").get();}@@ -256,7 +256,7 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {field("genre", field_types::STRING, true),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex bc642d97..b1faa27e 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -47,7 +47,7 @@ protected:TEST_F(CollectionManagerTest, CollectionCreation) {CollectionManager & collectionManager2 = CollectionManager::get_instance();- collection1 = collectionManager2.get_collection("collection1");+ collection1 = collectionManager2.get_collection("collection1").get();ASSERT_NE(nullptr, collection1);std::unordered_map<std::string, field> schema = collection1->get_schema();@@ -239,7 +239,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_TRUE(load_op.ok());- collection1 = collectionManager2.get_collection("collection1");+ collection1 = collectionManager2.get_collection("collection1").get();ASSERT_NE(nullptr, collection1);std::vector<std::string> facet_fields_expected = {"cast"};@@ -262,16 +262,16 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_STREQ("exclude-rule", collection1->get_overrides()["exclude-rule"].id.c_str());ASSERT_STREQ("include-rule", collection1->get_overrides()["include-rule"].id.c_str());- auto& synonyms = collection1->get_synonyms();+ const auto& synonyms = collection1->get_synonyms();ASSERT_EQ(2, synonyms.size());- ASSERT_STREQ("id1", synonyms["id1"].id.c_str());- ASSERT_EQ(2, synonyms["id1"].root.size());- ASSERT_EQ(1, synonyms["id1"].synonyms.size());+ ASSERT_STREQ("id1", synonyms.at("id1").id.c_str());+ ASSERT_EQ(2, synonyms.at("id1").root.size());+ ASSERT_EQ(1, synonyms.at("id1").synonyms.size());- ASSERT_STREQ("id3", synonyms["id3"].id.c_str());- ASSERT_EQ(0, synonyms["id3"].root.size());- ASSERT_EQ(2, synonyms["id3"].synonyms.size());+ ASSERT_STREQ("id3", synonyms.at("id3").id.c_str());+ ASSERT_EQ(0, synonyms.at("id3").root.size());+ ASSERT_EQ(2, synonyms.at("id3").synonyms.size());results = collection1->search("thomas", search_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(4, results["hits"].size());@@ -287,8 +287,8 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {infile.close();- ASSERT_FALSE(nullptr == collectionManager.get_collection_with_id(0));- ASSERT_FALSE(nullptr == collectionManager.get_collection("collection1"));+ ASSERT_FALSE(nullptr == collectionManager.get_collection_with_id(0).get());+ ASSERT_FALSE(nullptr == collectionManager.get_collection("collection1").get());collectionManager.drop_collection("collection1");@@ -303,8 +303,8 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {ASSERT_EQ(1, num_keys);ASSERT_TRUE(it->status().ok());- ASSERT_EQ(nullptr, collectionManager.get_collection("collection1"));- ASSERT_EQ(nullptr, collectionManager.get_collection_with_id(0));+ ASSERT_EQ(nullptr, collectionManager.get_collection("collection1").get());+ ASSERT_EQ(nullptr, collectionManager.get_collection_with_id(0).get());ASSERT_EQ(1, collectionManager.get_next_collection_id());delete it;diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex 37120de4..ac49e6e9 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -29,7 +29,7 @@ protected:field("points", field_types::INT32, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -628,7 +628,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWithWildCardQuery) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 3, fields, "points").get();}@@ -675,7 +675,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsIdsHavingColon) {std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex b488ea16..3ad37630 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -43,7 +43,7 @@ TEST_F(CollectionSortingTest, SortingOrder) {field("points", field_types::INT32, false),field("cast", field_types::STRING_ARRAY, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -150,7 +150,7 @@ TEST_F(CollectionSortingTest, Int64AsDefaultSortingField) {field("points", field_types::INT64, false),field("cast", field_types::STRING_ARRAY, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -209,7 +209,7 @@ TEST_F(CollectionSortingTest, SortOnFloatFields) {std::vector<sort_by> sort_fields_desc = { sort_by("score", "DESC"), sort_by("average", "DESC") };- coll_float_fields = collectionManager.get_collection("coll_float_fields");+ coll_float_fields = collectionManager.get_collection("coll_float_fields").get();if(coll_float_fields == nullptr) {coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "score").get();}@@ -277,7 +277,7 @@ TEST_F(CollectionSortingTest, ThreeSortFieldsLimit) {field("max", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -316,7 +316,7 @@ TEST_F(CollectionSortingTest, NegativeInt64Value) {field("points", field_types::INT64, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex b87e7335..322b0f9a 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -29,7 +29,7 @@ protected:field("points", field_types::INT32, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -376,7 +376,7 @@ TEST_F(CollectionSynonymsTest, ExactMatchRankedSameAsSynonymMatch) {field("description", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -431,7 +431,7 @@ TEST_F(CollectionSynonymsTest, SynonymFieldOrdering) {field("description", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex c21c07f3..0f0ff121 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -35,7 +35,7 @@ protected:query_fields = {"title"};sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };- collection = collectionManager.get_collection("collection");+ collection = collectionManager.get_collection("collection").get();if(collection == nullptr) {collection = collectionManager.create_collection("collection", 4, search_fields, "points").get();}@@ -478,6 +478,9 @@ TEST_F(CollectionTest, Pagination) {TEST_F(CollectionTest, WildcardQuery) {nlohmann::json results = collection->search("*", query_fields, "points:>0", {}, sort_fields, 0, 3, 1, FREQUENCY,false).get();++ LOG(INFO) << results;+ASSERT_EQ(3, results["hits"].size());ASSERT_EQ(25, results["found"].get<uint32_t>());@@ -626,7 +629,7 @@ TEST_F(CollectionTest, MultiOccurrenceString) {field("points", field_types::INT32, false)};- coll_multi_string = collectionManager.get_collection("coll_multi_string");+ coll_multi_string = collectionManager.get_collection("coll_multi_string").get();if (coll_multi_string == nullptr) {coll_multi_string = collectionManager.create_collection("coll_multi_string", 4, fields, "points").get();}@@ -654,7 +657,7 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {field("points", field_types::INT32, false)};- coll_array_text = collectionManager.get_collection("coll_array_text");+ coll_array_text = collectionManager.get_collection("coll_array_text").get();if (coll_array_text == nullptr) {coll_array_text = collectionManager.create_collection("coll_array_text", 4, fields, "points").get();}@@ -837,7 +840,7 @@ TEST_F(CollectionTest, MultipleFields) {field("points", field_types::INT32, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -956,7 +959,7 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {field("points", field_types::INT32, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -1044,7 +1047,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {field("points", field_types::INT32, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 1, fields, "points").get();}@@ -1195,7 +1198,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsertOptional) {field("points", field_types::INT32, false)};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -1289,7 +1292,7 @@ TEST_F(CollectionTest, ImportDocuments) {field("points", field_types::INT32, false)};- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();if(coll_mul_fields == nullptr) {coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();}@@ -1422,7 +1425,7 @@ TEST_F(CollectionTest, QueryBoolFields) {std::vector<sort_by> sort_fields = { sort_by("popular", "DESC"), sort_by("rating", "DESC") };- coll_bool = collectionManager.get_collection("coll_bool");+ coll_bool = collectionManager.get_collection("coll_bool").get();if(coll_bool == nullptr) {coll_bool = collectionManager.create_collection("coll_bool", 4, fields, "rating").get();}@@ -1531,7 +1534,7 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };- coll_array_fields = collectionManager.get_collection("coll_array_fields");+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();if(coll_array_fields == nullptr) {coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();}@@ -1586,7 +1589,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };- sample_collection = collectionManager.get_collection("sample_collection");+ sample_collection = collectionManager.get_collection("sample_collection").get();if(sample_collection == nullptr) {sample_collection = collectionManager.create_collection("sample_collection", 4, fields, "age").get();}@@ -1668,7 +1671,7 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };- empty_coll = collectionManager.get_collection("empty_coll");+ empty_coll = collectionManager.get_collection("empty_coll").get();if(empty_coll == nullptr) {empty_coll = collectionManager.create_collection("empty_coll", 4, fields, "age").get();}@@ -1688,7 +1691,7 @@ TEST_F(CollectionTest, IdFieldShouldBeAString) {std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "age").get();}@@ -1716,7 +1719,7 @@ TEST_F(CollectionTest, AnIntegerCanBePassedToAFloatField) {std::vector<sort_by> sort_fields = { sort_by("average", "DESC") };- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "average").get();}@@ -1744,7 +1747,7 @@ TEST_F(CollectionTest, DeletionOfADocument) {std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };Collection *collection_for_del;- collection_for_del = collectionManager.get_collection("collection_for_del");+ collection_for_del = collectionManager.get_collection("collection_for_del").get();if(collection_for_del == nullptr) {collection_for_del = collectionManager.create_collection("collection_for_del", 4, search_fields, "points").get();}@@ -1825,7 +1828,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -1898,7 +1901,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("age", "DESC") };- coll_large_text = collectionManager.get_collection("coll_large_text");+ coll_large_text = collectionManager.get_collection("coll_large_text").get();if(coll_large_text == nullptr) {coll_large_text = collectionManager.create_collection("coll_large_text", 4, fields, "age").get();}@@ -1991,7 +1994,7 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -2016,7 +2019,7 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -2074,7 +2077,7 @@ TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -2108,7 +2111,7 @@ TEST_F(CollectionTest, SearchHighlightWithNewLine) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -2142,7 +2145,7 @@ TEST_F(CollectionTest, UpdateDocument) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -2307,7 +2310,7 @@ TEST_F(CollectionTest, UpdateDocumentSorting) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -2365,7 +2368,7 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -2443,7 +2446,7 @@ TEST_F(CollectionTest, OptionalFields) {field("is_valid", field_types::BOOL, false, true),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "max").get();}@@ -2588,7 +2591,7 @@ TEST_F(CollectionTest, RemoveIfFound) {std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -2650,7 +2653,7 @@ TEST_F(CollectionTest, MultiFieldRelevance) {field("artist", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}@@ -2675,6 +2678,8 @@ TEST_F(CollectionTest, MultiFieldRelevance) {auto results = coll1->search("Dustin Kensrue Down There by the Train",{"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();+ LOG(INFO) << results;+ASSERT_EQ(3, results["found"].get<size_t>());ASSERT_EQ(3, results["hits"].size());@@ -2750,7 +2755,7 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {field("artist", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -2802,7 +2807,7 @@ TEST_F(CollectionTest, FieldWeightsNotProper) {field("artist", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -2847,7 +2852,7 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {field("artist", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -2890,7 +2895,7 @@ TEST_F(CollectionTest, MultiFieldMatchRanking) {field("artist", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -2940,7 +2945,7 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnArray) {field("skills", field_types::STRING_ARRAY, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -2981,7 +2986,7 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnFieldOrder) {field("artist", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -3020,7 +3025,7 @@ TEST_F(CollectionTest, PrefixRankedAfterExactMatch) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();}@@ -3060,7 +3065,7 @@ TEST_F(CollectionTest, HighlightWithAccentedCharacters) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if (coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();}diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 3160636a..ea1bfb42 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -39,7 +39,7 @@ TEST_F(CoreAPIUtilsTest, StatefulRemoveDocs) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};- coll1 = collectionManager.get_collection("coll1");+ coll1 = collectionManager.get_collection("coll1").get();if(coll1 == nullptr) {coll1 = collectionManager.create_collection("coll1", 2, fields, "points").get();}