Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor concurrency model.

kishorenc17fbbd083887
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e147b41..3c0ea9fa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,8 +3,8 @@ project(typesense)
cmake_policy(SET CMP0074 NEW)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++11 -O0 -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -O2 -g -DNDEBUG")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-class-memaccess -Wno-unused-parameter -std=c++17 -O0 -g")
set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
diff --git a/docker-build.sh b/docker-build.sh
index 5bab1fe5..59b81e07 100755
--- a/docker-build.sh
+++ b/docker-build.sh
@@ -22,10 +22,10 @@ if [[ "$@" == *"--depclean"* ]]; then
fi
echo "Building Typesense $TYPESENSE_VERSION..."
-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
-DCMAKE_BUILD_TYPE=Release -H/typesense -B/typesense/$BUILD_DIR
-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIR
+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIR
if [[ "$@" == *"--build-deploy-image"* ]]; then
echo "Creating deployment image for Typesense $TYPESENSE_VERSION server ..."
@@ -45,7 +45,7 @@ fi
#
#if [[ "$@" == *"--create-deb-upload"* ]]; then
-# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
+# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
# -DCMAKE_BUILD_TYPE=Debug -H/typesense -B/typesense/$BUILD_DIR
#fi
diff --git a/docker/development.Dockerfile b/docker/development.Dockerfile
index bb480975..8840674f 100644
--- a/docker/development.Dockerfile
+++ b/docker/development.Dockerfile
@@ -2,10 +2,10 @@
#
# $ docker push typesense/typesense-development:latest
-FROM typesense/ubuntu-10-04-gcc:6.4.0
+FROM typesense/ubuntu-12-04-gcc:v10.1.0
-ENV PATH /usr/local/gcc-6.4.0/bin/:$PATH
-ENV LD_LIBRARY_PATH /usr/local/gcc-6.4.0/lib64
+ENV PATH /usr/local/gcc-10.1.0/bin/:$PATH
+ENV LD_LIBRARY_PATH /usr/local/gcc-10.1.0/lib64
RUN apt-get install -y python-software-properties \
&& add-apt-repository ppa:git-core/ppa \
@@ -16,9 +16,9 @@ RUN apt-get install -y python-software-properties \
libidn11 \
git
-ADD https://gcc.gnu.org/pub/binutils/snapshots/binutils-2.30.0.tar.xz /opt/binutils-2.30.0.tar.xz
-RUN tar -C /opt -xf /opt/binutils-2.30.0.tar.xz
-RUN cd /opt/binutils-2.30.0 && ./configure --prefix=/usr && make tooldir=/usr && make check && \
+ADD https://ftp.gnu.org/gnu/binutils/binutils-2.36.tar.xz /opt/binutils-2.36.tar.xz
+RUN tar -C /opt -xf /opt/binutils-2.36.tar.xz
+RUN cd /opt/binutils-2.36 && ./configure --prefix=/usr && make tooldir=/usr && make check && \
make -j8 tooldir=/usr install && cp include/libiberty.h /usr/include
ADD https://github.com/Kitware/CMake/releases/download/v3.15.2/cmake-3.15.2-Linux-x86_64.tar.gz /opt/cmake-3.15.2-Linux-x86_64.tar.gz
@@ -29,8 +29,8 @@ ADD https://launchpad.net/ubuntu/+archive/primary/+files/snappy_1.1.3.orig.tar.g
RUN tar -C /opt -xf /opt/snappy_1.1.3.orig.tar.gz
RUN mkdir -p /opt/snappy-1.1.3/build && cd /opt/snappy-1.1.3/build && ../configure && make -j8 && make install
-ADD https://github.com/unicode-org/icu/releases/download/release-61-1/icu4c-61_1-src.tgz /opt/icu4c-61_1-src.tgz
-RUN tar -C /opt -xf /opt/icu4c-61_1-src.tgz
+ADD https://github.com/unicode-org/icu/releases/download/release-68-2/icu4c-68_2-src.tgz /opt/icu4c-68_2-src.tgz
+RUN tar -C /opt -xf /opt/icu4c-68_2-src.tgz
RUN cd /opt/icu/source && echo "#define U_DISABLE_RENAMING 1" >> common/unicode/uconfig.h && \
echo "#define U_STATIC_IMPLEMENTATION 1" >> common/unicode/uconfig.h && \
echo "#define U_USING_ICU_NAMESPACE 0" >> common/unicode/uconfig.h
@@ -96,5 +96,5 @@ RUN tar -C /opt -xf /opt/elfutils-0.182.tar.bz2
RUN cd /opt/elfutils-0.182 && ./configure --disable-libdebuginfod --disable-debuginfod && \
make -j8 && make install && rm -rf /usr/local/lib/*.so*
-ENV CC /usr/local/gcc-6.4.0/bin/gcc
-ENV CXX /usr/local/gcc-6.4.0/bin/g++
+ENV CC /usr/local/gcc-10.1.0/bin/gcc
+ENV CXX /usr/local/gcc-10.1.0/bin/g++
diff --git a/docker/ubuntu-12-04-gcc.Dockerfile b/docker/ubuntu-12-04-gcc.Dockerfile
new file mode 100644
index 00000000..73f78191
--- /dev/null
+++ b/docker/ubuntu-12-04-gcc.Dockerfile
@@ -0,0 +1,37 @@
+# $ docker build --file $PROJECT_DIR/docker/ubuntu-10-04-gcc.Dockerfile --tag typesense/ubuntu-12.04-gcc:latest --tag typesense/ubuntu-12.04-gcc:6.4 $PROJECT_DIR/docker
+#
+# NOTE: Default build image is bloated. Before publishing, export from a container to squash the image:
+# $ docker run -it typesense/ubuntu-12.04-gcc:latest bash -c "exit"
+# $ docker ps -a | grep typesense
+# $ docker export <container_id> | docker import - typesense/ubuntu-12.04-gcc:latest
+# $ docker push typesense/ubuntu-12.04-gcc
+
+FROM ubuntu:12.04
+
+RUN sed -i -re 's/([a-z]{2}\.)?archive.ubuntu.com|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list
+RUN apt-get update && apt-get install -y \
+ build-essential \
+ tar \
+ libmpfr-dev \
+ libgmp3-dev \
+ libmpc-dev \
+ flex \
+ bison \
+ zlib1g-dev
+
+ADD https://ftp.gnu.org/gnu/gcc/gcc-10.1.0/gcc-10.1.0.tar.gz /opt/
+RUN tar -C /opt -xf /opt/gcc-10.1.0.tar.gz
+
+RUN mkdir /opt/gcc-10.1.0/build && cd /opt/gcc-10.1.0/build && ../configure --disable-checking --enable-languages=c,c++ \
+ --enable-multiarch --disable-multilib --enable-shared --enable-threads=posix \
+ --with-gmp=/usr/lib --with-mpc=/usr/lib --with-mpfr=/usr/lib \
+ --build=x86_64-linux-gnu --host=x86_64-linux-gnu --target=x86_64-linux-gnu \
+ --without-included-gettext --with-tune=generic --prefix=/usr/local/gcc-10.1.0
+RUN cd /opt/gcc-10.1.0/build && make -j8
+RUN cd /opt/gcc-10.1.0/build && make install
+
+RUN apt-get remove -y gcc
+RUN rm -rf /opt/gcc-10.1.0 /opt/gcc-10.1.0.tar.gz
+
+ENV PATH /usr/local/gcc-10.1.0/bin/:$PATH
+ENV LD_LIBRARY_PATH /usr/local/gcc-10.1.0/lib64
diff --git a/include/auth_manager.h b/include/auth_manager.h
index 2cb10262..d1e5c571 100644
--- a/include/auth_manager.h
+++ b/include/auth_manager.h
@@ -3,6 +3,8 @@
#include <string>
#include <vector>
#include <map>
+#include <mutex>
+#include <shared_mutex>
#include "json.hpp"
#include "option.h"
#include "store.h"
@@ -78,11 +80,13 @@ class AuthManager {
private:
+ mutable std::shared_mutex mutex;
+
std::map<std::string, api_key_t> api_keys; // stores key_value => key mapping
Store *store;
// Auto incrementing API KEY ID
- uint32_t next_api_key_id;
+ std::atomic<uint32_t> next_api_key_id;
// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
static constexpr const char* API_KEY_NEXT_ID_KEY = "$KN";
@@ -94,39 +98,31 @@ private:
static std::string fmt_error(std::string&& error, const std::string& key);
+ Option<bool> authenticate_parse_params(const std::string& scoped_api_key, const std::string& action,
+ const std::string& collection, nlohmann::json& embedded_params) const;
+
+ bool auth_against_key(const std::string &collection, const std::string& action,
+ const api_key_t &api_key, const bool search_only) const;
+
public:
+ static const size_t KEY_LEN = 32;
+ static const size_t HMAC_BASE64_LEN = 44;
+
AuthManager() = default;
Option<bool> init(Store *store);
- Option<std::vector<api_key_t>> list_keys();
+ Option<std::vector<api_key_t>> list_keys() const;
- Option<api_key_t> get_key(uint32_t id, bool truncate_value = true);
+ Option<api_key_t> get_key(uint32_t id, bool truncate_value = true) const;
Option<api_key_t> create_key(api_key_t& api_key);
Option<api_key_t> remove_key(uint32_t id);
- bool authenticate(
- const std::string& req_api_key,
- const std::string& action,
- const std::string& collection,
- std::map<std::string, std::string>& params
- );
-
- Option<bool> authenticate_parse_params(
- const std::string& scoped_api_key,
- const std::string& action,
- const std::string& collection,
- nlohmann::json& embedded_params
- );
-
- static const size_t KEY_LEN = 32;
- static const size_t HMAC_BASE64_LEN = 44;
-
- bool auth_against_key(const std::string &collection, const std::string& action,
- const api_key_t &api_key, const bool search_only) const;
+ bool authenticate(const std::string& req_api_key, const std::string& action, const std::string& collection,
+ std::map<std::string, std::string>& params) const;
static bool populate_req_params(std::map<std::string, std::string> &req_params,
nlohmann::detail::iteration_proxy_value<nlohmann::json::iterator>& item);
diff --git a/include/collection.h b/include/collection.h
index 29984509..2f60b51d 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -7,6 +7,7 @@
#include <thread>
#include <mutex>
#include <condition_variable>
+#include <shared_mutex>
#include <art.h>
#include <index.h>
#include <number.h>
@@ -267,6 +268,8 @@ struct synonym_t {
class Collection {
private:
+ mutable std::shared_mutex mutex;
+
const uint8_t CURATED_RECORD_IDENTIFIER = 100;
struct highlight_t {
@@ -286,26 +289,20 @@ private:
}
};
- std::map<std::string, override_t> overrides;
-
- std::string name;
-
- uint32_t collection_id;
+ const std::string name;
- uint64_t created_at;
+ const uint32_t collection_id;
- size_t num_documents;
+ const uint64_t created_at;
- std::vector<Index*> indices;
-
- std::vector<std::thread*> index_threads;
+ std::atomic<size_t> num_documents;
// Auto incrementing record ID used internally for indexing - not exposed to the client
- uint32_t next_seq_id;
+ std::atomic<uint32_t> next_seq_id;
Store* store;
- std::vector<field> fields;
+ const std::vector<field> fields;
std::unordered_map<std::string, field> search_schema;
@@ -313,18 +310,24 @@ private:
std::unordered_map<std::string, field> sort_schema;
+ std::map<std::string, override_t> overrides;
+
spp::sparse_hash_map<std::string, synonym_t> synonym_definitions;
spp::sparse_hash_map<uint64_t, std::vector<std::string>> synonym_index;
- std::string default_sorting_field;
+ const std::string default_sorting_field;
- size_t num_memory_shards;
+ std::atomic<size_t> num_memory_shards;
const float max_memory_ratio;
- std::string get_doc_id_key(const std::string & doc_id);
+ const std::vector<Index*> indices;
- std::string get_seq_id_key(uint32_t seq_id);
+ // methods
+
+ std::string get_doc_id_key(const std::string & doc_id) const;
+
+ std::string get_seq_id_key(uint32_t seq_id) const;
void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,
const KV* field_order_kv, const nlohmann::json &document,
@@ -334,7 +337,7 @@ private:
bool highlighted_fully,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
- highlight_t &highlight);
+ highlight_t &highlight) const;
void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
@@ -342,7 +345,7 @@ private:
const std::map<size_t, std::vector<std::string>>& pinned_hits,
const std::vector<std::string>& hidden_hits,
std::map<size_t, std::vector<uint32_t>>& include_ids,
- std::vector<uint32_t> & excluded_ids);
+ std::vector<uint32_t> & excluded_ids) const;
static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,
const std::pair<uint64_t, facet_count_t>& b) {
@@ -356,18 +359,37 @@ private:
void free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const;
- Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters);
+ Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters) const;
- Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,
+ static Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,
std::map<size_t, std::vector<std::string>>& pinned_hits);
void synonym_reduction_internal(const std::vector<std::string>& tokens,
size_t start_window_size,
size_t start_index_pos,
std::set<uint64_t>& processed_syn_hashes,
- std::vector<std::vector<std::string>>& results);
+ std::vector<std::vector<std::string>>& results) const;
+
+ std::vector<Index *> init_indices();
public:
+
+ enum {MAX_ARRAY_MATCHES = 5};
+
+ const size_t PER_PAGE_MAX = 250;
+
+ const size_t GROUP_LIMIT_MAX = 99;
+
+ // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
+ static constexpr const char* COLLECTION_META_PREFIX = "$CM";
+ static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
+ static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";
+ static constexpr const char* COLLECTION_SYNONYM_PREFIX = "$CY";
+ static constexpr const char* SEQ_ID_PREFIX = "$SI";
+ static constexpr const char* DOC_ID_PREFIX = "$DI";
+
+ // methods
+
Collection() = delete;
Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,
@@ -384,23 +406,17 @@ public:
static std::string get_synonym_key(const std::string & collection_name, const std::string & synonym_id);
- std::string get_seq_id_collection_prefix();
-
- std::string get_name();
+ std::string get_seq_id_collection_prefix() const;
- uint64_t get_created_at();
+ std::string get_name() const;
- size_t get_num_documents();
+ uint64_t get_created_at() const;
- uint32_t get_collection_id();
+ uint32_t get_collection_id() const;
uint32_t get_next_seq_id();
- void set_next_seq_id(uint32_t seq_id);
-
- void increment_next_seq_id_field();
-
- Option<uint32_t> doc_id_to_seq_id(const std::string & doc_id);
+ Option<uint32_t> doc_id_to_seq_id(const std::string & doc_id) const;
std::vector<std::string> get_facet_fields();
@@ -415,110 +431,106 @@ public:
Option<doc_seq_id_t> to_doc(const std::string& json_str, nlohmann::json& document,
const index_operation_t& operation, const std::string& id="");
- nlohmann::json get_summary_json();
-
- Option<nlohmann::json> add(const std::string & json_str,
- const index_operation_t& operation=CREATE, const std::string& id="");
-
- nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,
- const index_operation_t& operation=CREATE, const std::string& id="");
+ static uint32_t get_seq_id_from_key(const std::string & key);
- Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,
- const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
- const std::vector<sort_by> & sort_fields, int num_typos,
- size_t per_page = 10, size_t page = 1,
- token_ordering token_order = FREQUENCY, bool prefix = false,
- size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
- const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),
- const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),
- size_t max_facet_values=10,
- const std::string & simple_facet_query = "",
- const size_t snippet_threshold = 30,
- const size_t highlight_affix_num_tokens = 4,
- const std::string & highlight_full_fields = "",
- size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,
- const std::string& pinned_hits_str="",
- const std::string& hidden_hits="",
- const std::vector<std::string>& group_by_fields={},
- const size_t group_limit = 0,
- const std::string& highlight_start_tag="<mark>",
- const std::string& highlight_end_tag="</mark>",
- std::vector<size_t> query_by_weights={},
- size_t limit_hits=UINT32_MAX);
+ Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document) const;
- Option<bool> get_filter_ids(const std::string & simple_filter_query,
- std::vector<std::pair<size_t, uint32_t*>>& index_ids);
+ Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, bool is_update);
- Option<nlohmann::json> get(const std::string & id);
+ static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,
+ const spp::sparse_hash_set<std::string> & exclude_fields);
- bool doc_exists(const std::string & id);
+ const std::vector<Index *> &_get_indexes() const;
- Option<std::string> remove(const std::string & id, bool remove_from_store = true);
+ bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,
+ std::string &value) const;
- Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);
+ static void aggregate_topster(size_t query_index, Topster &topster, Topster *index_topster);
- Option<uint32_t> add_override(const override_t & override);
+ static void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs);
- Option<uint32_t> remove_override(const std::string & id);
+ void batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,
+ size_t &num_indexed);
- std::map<std::string, override_t>& get_overrides() {
- return overrides;
- };
+ bool is_exceeding_memory_threshold() const;
- size_t get_num_memory_shards();
+ static void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,
+ nlohmann::json &new_doc,
+ nlohmann::json &del_doc);
- static uint32_t get_seq_id_from_key(const std::string & key);
+ void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
+ std::vector<std::string>& q_exclude_tokens) const;
- Option<bool> get_document_from_store(const std::string & seq_id_key, nlohmann::json & document);
+ // PUBLIC OPERATIONS
- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, bool is_update);
+ nlohmann::json get_summary_json() const;
size_t par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch, std::vector<size_t>& indexed_counts);
- static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,
- const spp::sparse_hash_set<std::string> & exclude_fields);
+ Option<nlohmann::json> add(const std::string & json_str,
+ const index_operation_t& operation=CREATE, const std::string& id="");
- const std::vector<Index *> &_get_indexes() const;
+ nlohmann::json add_many(std::vector<std::string>& json_lines, nlohmann::json& document,
+ const index_operation_t& operation=CREATE, const std::string& id="");
- enum {MAX_ARRAY_MATCHES = 5};
+ Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,
+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
+ const std::vector<sort_by> & sort_fields, int num_typos,
+ size_t per_page = 10, size_t page = 1,
+ token_ordering token_order = FREQUENCY, bool prefix = false,
+ size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
+ const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),
+ const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),
+ size_t max_facet_values=10,
+ const std::string & simple_facet_query = "",
+ const size_t snippet_threshold = 30,
+ const size_t highlight_affix_num_tokens = 4,
+ const std::string & highlight_full_fields = "",
+ size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,
+ const std::string& pinned_hits_str="",
+ const std::string& hidden_hits="",
+ const std::vector<std::string>& group_by_fields={},
+ const size_t group_limit = 0,
+ const std::string& highlight_start_tag="<mark>",
+ const std::string& highlight_end_tag="</mark>",
+ std::vector<size_t> query_by_weights={},
+ size_t limit_hits=UINT32_MAX) const;
- const size_t PER_PAGE_MAX = 250;
+ Option<bool> get_filter_ids(const std::string & simple_filter_query,
+ std::vector<std::pair<size_t, uint32_t*>>& index_ids);
- const size_t GROUP_LIMIT_MAX = 99;
+ Option<nlohmann::json> get(const std::string & id) const;
- // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
- static constexpr const char* COLLECTION_META_PREFIX = "$CM";
- static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
- static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";
- static constexpr const char* COLLECTION_SYNONYM_PREFIX = "$CY";
- static constexpr const char* SEQ_ID_PREFIX = "$SI";
- static constexpr const char* DOC_ID_PREFIX = "$DI";
+ Option<std::string> remove(const std::string & id, bool remove_from_store = true);
- bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,
- std::string &value);
+ Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);
- void aggregate_topster(size_t query_index, Topster &topster, Topster *index_topster) const;
+ size_t get_num_memory_shards();
- void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) const;
+ size_t get_num_documents() const;
- void batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,
- size_t &num_indexed);
+ // Override operations
- bool is_exceeding_memory_threshold() const;
+ Option<uint32_t> add_override(const override_t & override);
- void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,
- nlohmann::json &new_doc,
- nlohmann::json &del_doc);
+ Option<uint32_t> remove_override(const std::string & id);
- void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
- std::vector<std::string>& q_exclude_tokens) const;
+ std::map<std::string, override_t> get_overrides() {
+ std::shared_lock lock(mutex);
+ return overrides;
+ };
// synonym operations
- void synonym_reduction(const std::vector<std::string>& tokens, std::vector<std::vector<std::string>>& results);
- spp::sparse_hash_map<std::string, synonym_t>& get_synonyms();
+ void synonym_reduction(const std::vector<std::string>& tokens,
+ std::vector<std::vector<std::string>>& results) const;
+
+ spp::sparse_hash_map<std::string, synonym_t> get_synonyms();
+
bool get_synonym(const std::string& id, synonym_t& synonym);
+
Option<bool> add_synonym(const synonym_t& synonym);
+
Option<bool> remove_synonym(const std::string & id);
};
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 75a30172..147601d9 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -7,11 +7,54 @@
#include "field.h"
#include "collection.h"
#include "auth_manager.h"
+#include "threadpool.h"
+
+template<typename ResourceType>
+struct locked_resource_view_t {
+ locked_resource_view_t(std::shared_mutex &mutex, ResourceType &resource) : _lock(mutex), _resource(&resource) {}
+
+ locked_resource_view_t(std::shared_mutex &mutex, ResourceType *resource) : _lock(mutex), _resource(resource) {}
+
+ locked_resource_view_t(ResourceType &&) = delete;
+
+ locked_resource_view_t(const ResourceType &) = delete;
+
+ locked_resource_view_t &operator=(const ResourceType &) = delete;
+
+ ResourceType* operator->() {
+ return _resource;
+ }
+
+ bool operator==(const ResourceType* other) {
+ return other == _resource;
+ }
+
+ bool operator!=(const ResourceType* other) {
+ return other != _resource;
+ }
+
+ void unlock() {
+ _lock.unlock();
+ }
+
+ ResourceType* get() {
+ return _resource;
+ }
+
+private:
+ std::shared_lock<std::shared_mutex> _lock;
+ ResourceType* _resource;
+};
+
// Singleton, for managing meta information of all collections and house keeping
class CollectionManager {
private:
+ mutable std::shared_mutex mutex;
+
Store *store;
+ ThreadPool* thread_pool;
+
AuthManager auth_manager;
spp::sparse_hash_map<std::string, Collection*> collections;
@@ -22,7 +65,7 @@ private:
// Auto incrementing ID assigned to each collection
// Using a ID instead of a collection's name makes renaming possible
- uint32_t next_collection_id;
+ std::atomic<uint32_t> next_collection_id;
static constexpr const char* COLLECTION_NAME_KEY = "name";
static constexpr const char* COLLECTION_ID_KEY = "id";
@@ -42,7 +85,7 @@ private:
~CollectionManager() = default;
- Option<std::string> get_first_index_error(const std::vector<index_record>& index_records) {
+ static Option<std::string> get_first_index_error(const std::vector<index_record>& index_records) {
for(const auto & index_record: index_records) {
if(!index_record.indexed.ok()) {
return Option<std::string>(index_record.indexed.error());
@@ -53,6 +96,11 @@ private:
}
public:
+ static const size_t DEFAULT_NUM_MEMORY_SHARDS = 4;
+
+ static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
+ static constexpr const char* SYMLINK_PREFIX = "$SL";
+
static CollectionManager & get_instance() {
static CollectionManager instance;
return instance;
@@ -61,6 +109,19 @@ public:
CollectionManager(CollectionManager const&) = delete;
void operator=(CollectionManager const&) = delete;
+ Collection* init_collection(const nlohmann::json & collection_meta, const uint32_t collection_next_seq_id);
+
+ void add_to_collections(Collection* collection);
+
+ std::vector<Collection*> get_collections() const;
+
+ Collection* get_collection_unafe(const std::string & collection_name) const;
+
+ // PUBLICLY EXPOSED API
+
+ void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio, const std::string & auth_key);
+
+ // only for tests!
void init(Store *store, const float max_memory_ratio, const std::string & auth_key);
Option<bool> load(const size_t init_batch_size=1000);
@@ -68,49 +129,40 @@ public:
// frees in-memory data structures when server is shutdown - helps us run a memory leak detector properly
void dispose();
- Collection* init_collection(const nlohmann::json & collection_meta, const uint32_t collection_next_seq_id);
-
- void add_to_collections(Collection* collection);
-
bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,
- const std::string& collection, std::map<std::string, std::string>& params);
+ const std::string& collection, std::map<std::string, std::string>& params) const;
Option<Collection*> create_collection(const std::string& name, const size_t num_memory_shards,
const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)));
- Collection* get_collection(const std::string & collection_name);
+ locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;
- Collection* get_collection_with_id(uint32_t collection_id);
+ locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;
- std::vector<Collection*> get_collections();
+ nlohmann::json get_collection_summaries() const;
- Option<bool> drop_collection(const std::string& collection_name, const bool remove_from_store = true);
+ Option<nlohmann::json> drop_collection(const std::string& collection_name, const bool remove_from_store = true);
- uint32_t get_next_collection_id();
+ uint32_t get_next_collection_id() const;
static std::string get_symlink_key(const std::string & symlink_name);
- void set_next_collection_id(uint32_t next_id);
-
Store* get_store();
+ ThreadPool* get_thread_pool() const;
+
AuthManager& getAuthManager();
static Option<bool> do_search(std::map<std::string, std::string>& req_params, std::string& results_json_str);
// symlinks
- Option<std::string> resolve_symlink(const std::string & symlink_name);
+ Option<std::string> resolve_symlink(const std::string & symlink_name) const;
- spp::sparse_hash_map<std::string, std::string> & get_symlinks();
+ spp::sparse_hash_map<std::string, std::string> get_symlinks() const;
Option<bool> upsert_symlink(const std::string & symlink_name, const std::string & collection_name);
Option<bool> delete_symlink(const std::string & symlink_name);
-
- static const size_t DEFAULT_NUM_MEMORY_SHARDS = 4;
-
- static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
- static constexpr const char* SYMLINK_PREFIX = "$SL";
};
\ No newline at end of file
diff --git a/include/http_server.h b/include/http_server.h
index 914d58a2..01ce190e 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -14,6 +14,7 @@ extern "C" {
#include <cstdio>
#include "http_data.h"
#include "option.h"
+#include "threadpool.h"
class ReplicationState;
class HttpServer;
@@ -75,6 +76,8 @@ private:
bool cors_enabled;
+ ThreadPool* thread_pool;
+
bool (*auth_handler)(std::map<std::string, std::string>& params, const route_path& rpath,
const std::string& auth_key);
@@ -108,7 +111,9 @@ private:
public:
HttpServer(const std::string & version,
const std::string & listen_address, uint32_t listen_port,
- const std::string & ssl_cert_path, const std::string & ssl_cert_key_path, bool cors_enabled);
+ const std::string & ssl_cert_path,
+ const std::string & ssl_cert_key_path,
+ bool cors_enabled, ThreadPool* thread_pool);
~HttpServer();
@@ -164,6 +169,8 @@ public:
std::string get_version();
+ ThreadPool* get_thread_pool() const;
+
static constexpr const char* STOP_SERVER_MESSAGE = "STOP_SERVER";
static constexpr const char* STREAM_RESPONSE_MESSAGE = "STREAM_RESPONSE";
static constexpr const char* REQUEST_PROCEED_MESSAGE = "REQUEST_PROCEED";
diff --git a/include/index.h b/include/index.h
index d7ab0a9e..abcc17c7 100644
--- a/include/index.h
+++ b/include/index.h
@@ -4,6 +4,7 @@
#include <unordered_map>
#include <vector>
#include <mutex>
+#include <shared_mutex>
#include <condition_variable>
#include <art.h>
#include <number.h>
@@ -61,9 +62,8 @@ struct search_args {
Topster* curated_topster;
std::vector<std::vector<KV*>> raw_result_kvs;
std::vector<std::vector<KV*>> override_result_kvs;
- Option<uint32_t> outcome;
- search_args(): outcome(0) {
+ search_args() {
}
@@ -83,7 +83,7 @@ struct search_args {
page(page), token_order(token_order), prefix(prefix),
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
group_by_fields(group_by_fields), group_limit(group_limit),
- all_result_ids_len(0), outcome(0) {
+ all_result_ids_len(0) {
const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
topster = new Topster(topster_size, group_limit);
@@ -134,6 +134,8 @@ struct index_record {
class Index {
private:
+ mutable std::shared_mutex mutex;
+
const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();
std::string name;
@@ -158,6 +160,8 @@ private:
StringUtils string_utils;
+ // Internal utility functions
+
static inline uint32_t next_suggestion(const std::vector<token_candidates> &token_candidates_vec,
long long int n,
std::vector<art_leaf *>& actual_query_suggestion,
@@ -167,7 +171,8 @@ private:
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
- const uint32_t* result_ids, size_t results_size);
+ size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const uint32_t* result_ids, size_t results_size) const;
void search_field(const uint8_t & field_id,
std::vector<token_t>& query_tokens,
@@ -182,9 +187,11 @@ private:
Topster* topster, spp::sparse_hash_set<uint64_t>& groups_processed,
uint32_t** all_result_ids, size_t & all_result_ids_len,
size_t& field_num_results,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
const token_ordering token_order = FREQUENCY, const bool prefix = false,
const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
- const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD);
+ const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD) const;
void search_candidates(const uint8_t & field_id,
uint32_t* filter_ids, size_t filter_ids_length,
@@ -196,7 +203,8 @@ private:
uint32_t** all_result_ids,
size_t & all_result_ids_len,
size_t& field_num_results,
- const size_t typo_tokens_threshold);
+ const size_t typo_tokens_threshold,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields) const;
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
@@ -213,13 +221,23 @@ private:
void collate_included_ids(const std::vector<std::string>& q_included_tokens,
const std::string & field, const uint8_t field_id,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
- Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);
+ Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries) const;
- uint64_t facet_token_hash(const field & a_field, const std::string &token);
+ static uint64_t facet_token_hash(const field & a_field, const std::string &token);
- void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
+ static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
public:
+ // for limiting number of results on multiple candidates / query rewrites
+ enum {TYPO_TOKENS_THRESHOLD = 100};
+
+ // for limiting number of fields that can be searched on
+ enum {FIELD_LIMIT_NUM = 100};
+
+ // If the number of results found is less than this threshold, Typesense will attempt to drop the tokens
+ // in the query that have the least individual hits one by one until enough results are found.
+ static const int DROP_TOKENS_THRESHOLD = 10;
+
Index() = delete;
Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,
@@ -233,98 +251,29 @@ public:
return combined;
}
- void run_search();
-
- void search(Option<uint32_t> & outcome,
- const std::vector<std::string> & q_include_tokens,
- const std::vector<std::string> & q_exclude_tokens,
- const std::vector<std::vector<std::string>>& q_synonyms,
- const std::vector<search_field_t> & search_fields,
- const std::vector<filter> & filters, std::vector<facet> & facets,
- facet_query_t & facet_query,
- const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
- const std::vector<uint32_t> & excluded_ids,
- const std::vector<sort_by> & sort_fields_std, const int num_typos,
- Topster* topster, Topster* curated_topster,
- const size_t per_page, const size_t page, const token_ordering token_order,
- const bool prefix, const size_t drop_tokens_threshold,
- size_t & all_result_ids_len,
- spp::sparse_hash_set<uint64_t>& groups_processed,
- std::vector<std::vector<art_leaf*>> & searched_queries,
- std::vector<std::vector<KV*>> & raw_result_kvs,
- std::vector<std::vector<KV*>> & override_result_kvs,
- const size_t typo_tokens_threshold);
-
static void concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids);
- Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);
-
- Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);
-
- art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
-
- static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
- std::vector<uint32_t*>& leaf_to_indices,
- size_t result_index,
- std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);
-
void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const uint32_t *result_ids, const size_t result_size, uint32_t token_bits);
+ const uint32_t *result_ids, const size_t result_size,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ uint32_t token_bits) const;
static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);
- Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,
- const std::string & default_sorting_field, bool is_update);
-
- static Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,
- const std::string & default_sorting_field,
- const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema,
- bool is_update);
-
- static size_t batch_memory_index(Index *index,
- std::vector<index_record> & iter_batch,
- const std::string & default_sorting_field,
- const std::unordered_map<std::string, field> & search_schema,
- const std::map<std::string, field> & facet_schema);
-
- const spp::sparse_hash_map<std::string, art_tree *> &_get_search_index() const;
+ const spp::sparse_hash_map<std::string, art_tree *>& _get_search_index() const;
const spp::sparse_hash_map<std::string, num_tree_t*>& _get_numerical_index() const;
- // for limiting number of results on multiple candidates / query rewrites
- enum {TYPO_TOKENS_THRESHOLD = 100};
-
- // for limiting number of fields that can be searched on
- enum {FIELD_LIMIT_NUM = 100};
-
- // If the number of results found is less than this threshold, Typesense will attempt to drop the tokens
- // in the query that have the least individual hits one by one until enough results are found.
- static const int DROP_TOKENS_THRESHOLD = 10;
-
- /*
- * Concurrency Primitives
- */
-
- // Used for passing control back and forth between main and worker threads
- std::mutex m;
- std::condition_variable cv;
-
- bool ready; // prevents spurious wake up of the worker thread
- bool processed; // prevents spurious wake up of the main thread
- bool terminate; // used for interrupting the thread during tear down
-
- search_args* search_params;
-
- int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;
+ static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len);
static int64_t float_to_in64_t(float n);
- uint64_t get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id, const uint32_t seq_id) const;
+ uint64_t get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id,
+ const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;
- void get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index);
+ void get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) const;
void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
const std::vector<art_leaf *> &query_suggestion,
@@ -332,10 +281,10 @@ public:
void scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doc, nlohmann::json& old_doc);
- void tokenize_doc_field(const nlohmann::json& document, const field& search_field, std::vector<std::string>& tokens);
+ static void tokenize_doc_field(const nlohmann::json& document, const field& search_field, std::vector<std::string>& tokens);
template<typename T>
- bool _arrays_match(std::vector<T> reindex_vals, std::vector<T> old_vals) {
+ static bool _arrays_match(std::vector<T> reindex_vals, std::vector<T> old_vals) {
if(old_vals.size() != reindex_vals.size()) {
return false;
}
@@ -350,5 +299,60 @@ public:
return true;
}
+
+ // Public operations
+
+ void run_search(search_args* search_params);
+
+ void search(const std::vector<std::string> & q_include_tokens,
+ const std::vector<std::string> & q_exclude_tokens,
+ const std::vector<std::vector<std::string>>& q_synonyms,
+ const std::vector<search_field_t> & search_fields,
+ const std::vector<filter> & filters, std::vector<facet> & facets,
+ facet_query_t & facet_query,
+ const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
+ const std::vector<uint32_t> & excluded_ids,
+ const std::vector<sort_by> & sort_fields_std, const int num_typos,
+ Topster* topster, Topster* curated_topster,
+ const size_t per_page, const size_t page, const token_ordering token_order,
+ const bool prefix, const size_t drop_tokens_threshold,
+ size_t & all_result_ids_len,
+ spp::sparse_hash_set<uint64_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>> & searched_queries,
+ std::vector<std::vector<KV*>> & raw_result_kvs,
+ std::vector<std::vector<KV*>> & override_result_kvs,
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields) const;
+
+ Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);
+
+
+ Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id,
+ const std::string & default_sorting_field, bool is_update);
+
+ static size_t batch_memory_index(Index *index,
+ std::vector<index_record> & iter_batch,
+ const std::string & default_sorting_field,
+ const std::unordered_map<std::string, field> & search_schema,
+ const std::map<std::string, field> & facet_schema);
+
+ static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
+ const std::vector<uint32_t*>& leaf_to_indices,
+ const size_t result_index,
+ std::unordered_map<size_t, std::vector<std::vector<uint16_t>>> &array_token_positions);
+
+ // the following methods are not synchronized because their parent calls are synchronized
+
+ art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
+
+ uint32_t do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const;
+
+ static Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id,
+ const std::string & default_sorting_field,
+ const std::unordered_map<std::string, field> & search_schema,
+ const std::map<std::string, field> & facet_schema,
+ bool is_update);
+
};
diff --git a/src/auth_manager.cpp b/src/auth_manager.cpp
index cb6fbb9f..de338956 100644
--- a/src/auth_manager.cpp
+++ b/src/auth_manager.cpp
@@ -8,6 +8,7 @@ constexpr const uint64_t api_key_t::FAR_FUTURE_TIMESTAMP;
Option<bool> AuthManager::init(Store *store) {
// This function must be idempotent, i.e. when called multiple times, must produce the same state without leaks
//LOG(INFO) << "AuthManager::init()";
+ std::unique_lock lock(mutex);
this->store = store;
@@ -42,7 +43,9 @@ Option<bool> AuthManager::init(Store *store) {
return Option<bool>(true);
}
-Option<std::vector<api_key_t>> AuthManager::list_keys() {
+Option<std::vector<api_key_t>> AuthManager::list_keys() const {
+ std::shared_lock lock(mutex);
+
std::vector<std::string> api_key_json_strs;
store->scan_fill(API_KEYS_PREFIX, api_key_json_strs);
@@ -61,7 +64,9 @@ Option<std::vector<api_key_t>> AuthManager::list_keys() {
return Option<std::vector<api_key_t>>(stored_api_keys);
}
-Option<api_key_t> AuthManager::get_key(uint32_t id, bool truncate_value) {
+Option<api_key_t> AuthManager::get_key(uint32_t id, bool truncate_value) const {
+ std::shared_lock lock(mutex);
+
std::string api_key_store_key = std::string(API_KEYS_PREFIX) + "_" + std::to_string(id);
std::string api_key_json_str;
StoreStatus status = store->get(api_key_store_key, api_key_json_str);
@@ -88,6 +93,7 @@ Option<api_key_t> AuthManager::get_key(uint32_t id, bool truncate_value) {
Option<api_key_t> AuthManager::create_key(api_key_t& api_key) {
//LOG(INFO) << "AuthManager::create_key()";
+ std::unique_lock lock(mutex);
if(api_keys.count(api_key.value) != 0) {
return Option<api_key_t>(409, "API key generation conflict.");
@@ -119,6 +125,8 @@ Option<api_key_t> AuthManager::remove_key(uint32_t id) {
return Option<api_key_t>(500, "Could not delete API key.");
}
+ std::unique_lock lock(mutex);
+
api_key_t&& key = key_op.get();
api_keys.erase(key.value);
@@ -131,7 +139,9 @@ uint32_t AuthManager::get_next_api_key_id() {
}
bool AuthManager::authenticate(const std::string& req_api_key, const std::string& action,
- const std::string& collection, std::map<std::string, std::string>& params) {
+ const std::string& collection, std::map<std::string, std::string>& params) const {
+
+ std::shared_lock lock(mutex);
//LOG(INFO) << "AuthManager::authenticate()";
@@ -168,7 +178,7 @@ bool AuthManager::authenticate(const std::string& req_api_key, const std::string
return false;
}
- const api_key_t& api_key = api_keys[req_api_key];
+ const api_key_t& api_key = api_keys.at(req_api_key);
return auth_against_key(collection, action, api_key, false);
}
@@ -212,7 +222,7 @@ bool AuthManager::auth_against_key(const std::string& collection, const std::str
}
Option<bool> AuthManager::authenticate_parse_params(const std::string& scoped_api_key, const std::string& action,
- const std::string& collection, nlohmann::json& embedded_params) {
+ const std::string& collection, nlohmann::json& embedded_params) const {
// allow only searches from scoped keys
if(action != DOCUMENTS_SEARCH_ACTION) {
LOG(ERROR) << "Scoped API keys can only be used for searches.";
diff --git a/src/collection.cpp b/src/collection.cpp
index b643cabc..914cfc28 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -11,6 +11,7 @@
#include <rocksdb/write_batch.h>
#include <system_metrics.h>
#include <tokenizer.h>
+#include <collection_manager.h>
#include "topster.h"
#include "logger.h"
@@ -39,51 +40,20 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, co
const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,
const std::string& default_sorting_field, const size_t num_memory_shards,
const float max_memory_ratio):
- name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
+ name(name), collection_id(collection_id), created_at(created_at),
+ next_seq_id(next_seq_id), store(store),
fields(fields), default_sorting_field(default_sorting_field),
num_memory_shards(num_memory_shards),
- max_memory_ratio(max_memory_ratio) {
+ max_memory_ratio(max_memory_ratio),
+ indices(init_indices()) {
- for(const field& field: fields) {
- search_schema.emplace(field.name, field);
-
- if(field.is_facet()) {
- facet_schema.emplace(field.name, field);
- }
-
- if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {
- sort_schema.emplace(field.name, field);
- }
- }
-
- for(size_t i = 0; i < num_memory_shards; i++) {
- Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);
- indices.push_back(index);
- std::thread* thread = new std::thread(&Index::run_search, index);
- index_threads.push_back(thread);
- }
-
- this->created_at = created_at;
this->num_documents = 0;
}
Collection::~Collection() {
for(size_t i = 0; i < indices.size(); i++) {
- std::thread *t = index_threads[i];
- Index* index = indices[i];
- index->ready = true;
- index->terminate = true;
- index->cv.notify_one();
- t->join();
-
- delete t;
delete indices[i];
- t = nullptr;
- indices[i] = nullptr;
}
-
- indices.clear();
- index_threads.clear();
}
uint32_t Collection::get_next_seq_id() {
@@ -91,14 +61,6 @@ uint32_t Collection::get_next_seq_id() {
return next_seq_id++;
}
-void Collection::set_next_seq_id(uint32_t seq_id) {
- next_seq_id = seq_id;
-}
-
-void Collection::increment_next_seq_id_field() {
- next_seq_id++;
-}
-
Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::json& document,
const index_operation_t& operation, const std::string& id) {
try {
@@ -170,12 +132,14 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::
}
}
-nlohmann::json Collection::get_summary_json() {
+nlohmann::json Collection::get_summary_json() const {
+ std::shared_lock lock(mutex);
+
nlohmann::json json_response;
json_response["name"] = name;
- json_response["num_memory_shards"] = num_memory_shards;
- json_response["num_documents"] = num_documents;
+ json_response["num_memory_shards"] = num_memory_shards.load();
+ json_response["num_documents"] = num_documents.load();
json_response["created_at"] = created_at;
nlohmann::json fields_arr;
@@ -373,6 +337,8 @@ void Collection::batch_index(std::vector<std::vector<index_record>> &index_batch
}
Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id, bool is_update) {
+ std::unique_lock lock(mutex);
+
if(!is_update) {
// for update, validation should be done prior
Option<uint32_t> validation_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,
@@ -392,21 +358,37 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
size_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch,
std::vector<size_t>& indexed_counts) {
- std::vector<std::future<size_t>> futures;
- size_t num_indexed = 0;
+ std::unique_lock lock(mutex);
+
+ size_t num_processed = 0;
+ std::mutex m_process;
+ std::condition_variable cv_process;
- for(size_t i=0; i < num_memory_shards; i++) {
- futures.push_back(
- std::async(&Index::batch_memory_index, indices[i], std::ref(iter_batch[i]), default_sorting_field,
- search_schema, facet_schema)
- );
+ std::vector<size_t> num_indexed_vec(indices.size());
+
+ for(size_t index_id = 0; index_id < indices.size(); index_id++) {
+ Index* index = indices[index_id];
+ CollectionManager::get_instance().get_thread_pool()->enqueue(
+ [index, index_id, &num_indexed_vec, &iter_batch, this, &m_process, &num_processed, &cv_process]() {
+ size_t num_indexed = Index::batch_memory_index(index, std::ref(iter_batch[index_id]), default_sorting_field,
+ search_schema, facet_schema);
+ std::unique_lock<std::mutex> lock(m_process);
+ num_indexed_vec[index_id] = num_indexed;
+ num_processed++;
+ cv_process.notify_one();
+ });
}
- for(size_t i=0; i < futures.size(); i++) {
- size_t num_indexed_future = futures[i].get();
- num_documents += num_indexed_future;
- num_indexed += num_indexed_future;
- indexed_counts[i] = num_indexed_future;
+ const size_t num_indices = indices.size();
+ std::unique_lock<std::mutex> lock_process(m_process);
+ cv_process.wait(lock_process, [&](){ return num_processed == num_indices; });
+
+ size_t num_indexed = 0;
+
+ for(size_t index_id = 0; index_id < indices.size(); index_id++) {
+ num_documents += num_indexed_vec[index_id];
+ num_indexed += num_indexed_vec[index_id];
+ indexed_counts[index_id] = num_indexed_vec[index_id];
}
return num_indexed;
@@ -428,7 +410,7 @@ void Collection::populate_overrides(std::string query,
const std::map<size_t, std::vector<std::string>>& pinned_hits,
const std::vector<std::string>& hidden_hits,
std::map<size_t, std::vector<uint32_t>>& include_ids,
- std::vector<uint32_t> & excluded_ids) {
+ std::vector<uint32_t> & excluded_ids) const {
StringUtils::tolowercase(query);
std::set<uint32_t> excluded_set;
@@ -513,7 +495,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
std::vector<size_t> query_by_weights,
- size_t limit_hits) {
+ size_t limit_hits) const {
+
+ std::shared_lock lock(mutex);
if(query != "*" && search_fields.empty()) {
return Option<nlohmann::json>(400, "No search fields specified for the query.");
@@ -785,25 +769,39 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::vector<std::vector<std::string>> q_synonyms;
synonym_reduction(q_include_tokens, q_synonyms);
- // send data to individual index threads
+ // search all indices
+ std::vector<search_args*> search_args_vec;
+ size_t num_processed = 0;
+ std::mutex m_process;
+ std::condition_variable cv_process;
+
size_t index_id = 0;
for(Index* index: indices) {
- index->search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,
- filters, facets, index_to_included_ids[index_id],
- index_to_excluded_ids[index_id],
- sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
- per_page, page, token_order, prefix,
- drop_tokens_threshold, typo_tokens_threshold,
- group_by_fields, group_limit);
- {
- std::lock_guard<std::mutex> lk(index->m);
- index->ready = true;
- index->processed = false;
- }
- index->cv.notify_one();
+ search_args* search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,
+ filters, facets, index_to_included_ids[index_id],
+ index_to_excluded_ids[index_id],
+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
+ per_page, page, token_order, prefix,
+ drop_tokens_threshold, typo_tokens_threshold,
+ group_by_fields, group_limit);
+
+ search_args_vec.push_back(search_params);
+
+ CollectionManager::get_instance().get_thread_pool()->enqueue(
+ [index, search_params, &m_process, &num_processed, &cv_process]() {
+ index->run_search(search_params);
+ std::unique_lock<std::mutex> lock(m_process);
+ num_processed++;
+ cv_process.notify_one();
+ });
+
index_id++;
}
+ const size_t num_indices = indices.size();
+ std::unique_lock<std::mutex> lock_process(m_process);
+ cv_process.wait(lock_process, [&](){ return num_processed == num_indices; });
+
Option<nlohmann::json> index_search_op({}); // stores the last error across all index threads
// for grouping we have re-aggregate
@@ -812,35 +810,19 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
Topster topster(topster_size, group_limit);
Topster curated_topster(topster_size, group_limit);
- for(Index* index: indices) {
- // wait for the worker
- {
- std::unique_lock<std::mutex> lk(index->m);
- index->cv.wait(lk, [index]{return index->processed;});
- }
+ for(const search_args* search_params: search_args_vec) {
+ aggregate_topster(searched_queries.size(), topster, search_params->topster);
+ aggregate_topster(searched_queries.size(), curated_topster, search_params->curated_topster);
- if(!index->search_params->outcome.ok()) {
- index_search_op = Option<nlohmann::json>(index->search_params->outcome.code(),
- index->search_params->outcome.error());
- }
+ searched_queries.insert(searched_queries.end(), search_params->searched_queries.begin(),
+ search_params->searched_queries.end());
- if(!index_search_op.ok()) {
- // we still need to iterate without breaking to release the locks
- continue;
- }
-
- aggregate_topster(searched_queries.size(), topster, index->search_params->topster);
- aggregate_topster(searched_queries.size(), curated_topster, index->search_params->curated_topster);
-
- searched_queries.insert(searched_queries.end(), index->search_params->searched_queries.begin(),
- index->search_params->searched_queries.end());
-
- for(size_t fi = 0; fi < index->search_params->facets.size(); fi++) {
- auto & this_facet = index->search_params->facets[fi];
+ for(size_t fi = 0; fi < search_params->facets.size(); fi++) {
+ auto & this_facet = search_params->facets[fi];
auto & acc_facet = facets[fi];
for(auto & facet_kv: this_facet.result_map) {
- if(index->search_params->group_limit) {
+ if(search_params->group_limit) {
// we have to add all group sets
acc_facet.result_map[facet_kv.first].groups.insert(
facet_kv.second.groups.begin(), facet_kv.second.groups.end()
@@ -871,18 +853,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
if(group_limit) {
groups_processed.insert(
- index->search_params->groups_processed.begin(),
- index->search_params->groups_processed.end()
+ search_params->groups_processed.begin(),
+ search_params->groups_processed.end()
);
} else {
- total_found += index->search_params->all_result_ids_len;
+ total_found += search_params->all_result_ids_len;
}
}
- if(!index_search_op.ok()) {
- return index_search_op;
- }
-
topster.sort();
curated_topster.sort();
@@ -946,7 +924,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
nlohmann::json result = nlohmann::json::object();
result["found"] = total_found;
- result["out_of"] = num_documents;
+ result["out_of"] = num_documents.load();
std::string hits_key = group_limit ? "grouped_hits" : "hits";
result[hits_key] = nlohmann::json::array();
@@ -1165,8 +1143,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
// free search params
- for(Index* index: indices) {
- delete index->search_params;
+ for(auto search_params: search_args_vec) {
+ delete search_params;
}
result["request_params"] = nlohmann::json::object();;
@@ -1210,7 +1188,7 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
}
}
-void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) const {
+void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) {
if(topster->distinct) {
for(auto &group_topster_entry: topster->group_kv_map) {
Topster* group_topster = group_topster_entry.second;
@@ -1226,7 +1204,7 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K
}
}
-void Collection::aggregate_topster(size_t query_index, Topster& agg_topster, Topster* index_topster) const {
+void Collection::aggregate_topster(size_t query_index, Topster& agg_topster, Topster* index_topster) {
if(index_topster->distinct) {
for(auto &group_topster_entry: index_topster->group_kv_map) {
Topster* group_topster = group_topster_entry.second;
@@ -1245,6 +1223,8 @@ void Collection::aggregate_topster(size_t query_index, Topster& agg_topster, Top
Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids) {
+ std::shared_lock lock(mutex);
+
std::vector<filter> filters;
Option<bool> filter_op = parse_filter_query(simple_filter_query, filters);
@@ -1254,12 +1234,7 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
for(auto& index: indices) {
uint32_t* filter_ids = nullptr;
- Option<uint32_t> op_filter_ids_length = index->do_filtering(&filter_ids, filters);
- if(!op_filter_ids_length.ok()) {
- return Option<bool>(op_filter_ids_length.code(), op_filter_ids_length.error());
- }
-
- size_t filter_ids_len = op_filter_ids_length.get();
+ size_t filter_ids_len = index->do_filtering(&filter_ids, filters);
index_ids.emplace_back(filter_ids_len, filter_ids);
}
@@ -1267,7 +1242,7 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
}
bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,
- const nlohmann::json &document, std::string &value) {
+ const nlohmann::json &document, std::string &value) const {
if(document.count(a_facet.field_name) == 0) {
LOG(ERROR) << "Could not find field " << a_facet.field_name << " in document during faceting.";
@@ -1331,7 +1306,7 @@ void Collection::highlight_result(const field &search_field,
bool highlighted_fully,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
- highlight_t & highlight) {
+ highlight_t & highlight) const {
if(searched_queries.size() <= field_order_kv->query_index) {
return ;
@@ -1494,11 +1469,7 @@ void Collection::free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) cons
}
}
-bool Collection::doc_exists(const std::string & id) {
- return store->contains(get_doc_id_key(id));
-}
-
-Option<nlohmann::json> Collection::get(const std::string & id) {
+Option<nlohmann::json> Collection::get(const std::string & id) const {
std::string seq_id_str;
StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);
@@ -1535,6 +1506,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) {
}
void Collection::remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {
+ std::unique_lock lock(mutex);
const std::string& id = document["id"];
Index* index = indices[seq_id % num_memory_shards];
@@ -1614,6 +1586,7 @@ Option<uint32_t> Collection::add_override(const override_t & override) {
return Option<uint32_t>(500, "Error while storing the override on disk.");
}
+ std::unique_lock lock(mutex);
overrides[override.id] = override;
return Option<uint32_t>(200);
}
@@ -1624,6 +1597,8 @@ Option<uint32_t> Collection::remove_override(const std::string & id) {
if(!removed) {
return Option<uint32_t>(500, "Error while deleting the override from disk.");
}
+
+ std::unique_lock lock(mutex);
overrides.erase(id);
return Option<uint32_t>(200);
}
@@ -1645,34 +1620,34 @@ std::string Collection::get_next_seq_id_key(const std::string & collection_name)
return std::string(COLLECTION_NEXT_SEQ_PREFIX) + "_" + collection_name;
}
-std::string Collection::get_seq_id_key(uint32_t seq_id) {
+std::string Collection::get_seq_id_key(uint32_t seq_id) const {
// We can't simply do std::to_string() because we want to preserve the byte order.
// & 0xFF masks all but the lowest eight bits.
const std::string & serialized_id = StringUtils::serialize_uint32_t(seq_id);
return get_seq_id_collection_prefix() + "_" + serialized_id;
}
-std::string Collection::get_doc_id_key(const std::string & doc_id) {
+std::string Collection::get_doc_id_key(const std::string & doc_id) const {
return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_" + doc_id;
}
-std::string Collection::get_name() {
+std::string Collection::get_name() const {
return name;
}
-uint64_t Collection::get_created_at() {
+uint64_t Collection::get_created_at() const {
return created_at;
}
-size_t Collection::get_num_documents() {
- return num_documents;
+size_t Collection::get_num_documents() const {
+ return num_documents.load();
}
-uint32_t Collection::get_collection_id() {
+uint32_t Collection::get_collection_id() const {
return collection_id;
}
-Option<uint32_t> Collection::doc_id_to_seq_id(const std::string & doc_id) {
+Option<uint32_t> Collection::doc_id_to_seq_id(const std::string & doc_id) const {
std::string seq_id_str;
StoreStatus status = store->get(get_doc_id_key(doc_id), seq_id_str);
if(status == StoreStatus::FOUND) {
@@ -1725,7 +1700,7 @@ std::string Collection::get_synonym_key(const std::string & collection_name, con
return std::string(COLLECTION_SYNONYM_PREFIX) + "_" + collection_name + "_" + synonym_id;
}
-std::string Collection::get_seq_id_collection_prefix() {
+std::string Collection::get_seq_id_collection_prefix() const {
return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);
}
@@ -1733,7 +1708,7 @@ std::string Collection::get_default_sorting_field() {
return default_sorting_field;
}
-Option<bool> Collection::get_document_from_store(const std::string &seq_id_key, nlohmann::json & document) {
+Option<bool> Collection::get_document_from_store(const std::string &seq_id_key, nlohmann::json & document) const {
std::string json_doc_str;
StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);
@@ -1755,7 +1730,7 @@ const std::vector<Index *> &Collection::_get_indexes() const {
}
Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,
- std::vector<filter>& filters) {
+ std::vector<filter>& filters) const {
std::vector<std::string> filter_blocks;
StringUtils::split(simple_filter_query, filter_blocks, "&&");
@@ -1947,7 +1922,8 @@ Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,
return Option<bool>(true);
}
-void Collection::synonym_reduction(const std::vector<std::string>& tokens, std::vector<std::vector<std::string>>& results) {
+void Collection::synonym_reduction(const std::vector<std::string>& tokens,
+ std::vector<std::vector<std::string>>& results) const {
std::set<uint64_t> processed_syn_hashes;
synonym_reduction_internal(tokens, tokens.size(), 0, processed_syn_hashes, results);
}
@@ -1961,6 +1937,7 @@ Option<bool> Collection::add_synonym(const synonym_t& synonym) {
}
}
+ std::unique_lock write_lock(mutex);
synonym_definitions[synonym.id] = synonym;
if(!synonym.root.empty()) {
@@ -1973,6 +1950,8 @@ Option<bool> Collection::add_synonym(const synonym_t& synonym) {
}
}
+ write_lock.unlock();
+
bool inserted = store->insert(Collection::get_synonym_key(name, synonym.id), synonym.to_json().dump());
if(!inserted) {
return Option<bool>(500, "Error while storing the synonym on disk.");
@@ -1982,6 +1961,8 @@ Option<bool> Collection::add_synonym(const synonym_t& synonym) {
}
bool Collection::get_synonym(const std::string& id, synonym_t& synonym) {
+ std::shared_lock lock(mutex);
+
if(synonym_definitions.count(id) != 0) {
synonym = synonym_definitions[id];
return true;
@@ -1993,7 +1974,7 @@ bool Collection::get_synonym(const std::string& id, synonym_t& synonym) {
void Collection::synonym_reduction_internal(const std::vector<std::string>& tokens,
size_t start_window_size, size_t start_index_pos,
std::set<uint64_t>& processed_syn_hashes,
- std::vector<std::vector<std::string>>& results) {
+ std::vector<std::vector<std::string>>& results) const {
bool recursed = false;
@@ -2021,7 +2002,7 @@ void Collection::synonym_reduction_internal(const std::vector<std::string>& toke
const auto& syn_ids = syn_itr->second;
for(const auto& syn_id: syn_ids) {
- const auto &syn_def = synonym_definitions[syn_id];
+ const auto &syn_def = synonym_definitions.at(syn_id);
for (const auto &syn_def_tokens: syn_def.synonyms) {
std::vector<std::string> new_tokens;
@@ -2085,6 +2066,7 @@ void Collection::synonym_reduction_internal(const std::vector<std::string>& toke
}
Option<bool> Collection::remove_synonym(const std::string &id) {
+ std::unique_lock lock(mutex);
const auto& syn_iter = synonym_definitions.find(id);
if(syn_iter != synonym_definitions.end()) {
@@ -2111,6 +2093,30 @@ Option<bool> Collection::remove_synonym(const std::string &id) {
return Option<bool>(404, "Could not find that `id`.");
}
-spp::sparse_hash_map<std::string, synonym_t> &Collection::get_synonyms() {
+spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {
+ std::shared_lock lock(mutex);
return synonym_definitions;
}
+
+std::vector<Index *> Collection::init_indices() {
+ std::vector<Index*> index_list;
+
+ for(const field& field: fields) {
+ search_schema.emplace(field.name, field);
+
+ if(field.is_facet()) {
+ facet_schema.emplace(field.name, field);
+ }
+
+ if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {
+ sort_schema.emplace(field.name, field);
+ }
+ }
+
+ for(size_t i = 0; i < num_memory_shards; i++) {
+ Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema);
+ index_list.push_back(index);
+ }
+
+ return index_list;
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 08153a2d..85684246 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -58,14 +58,23 @@ void CollectionManager::add_to_collections(Collection* collection) {
collection_id_names.emplace(collection->get_collection_id(), collection->get_name());
}
-void CollectionManager::init(Store *store,
+void CollectionManager::init(Store *store, ThreadPool* thread_pool,
const float max_memory_ratio,
const std::string & auth_key) {
+ std::unique_lock lock(mutex);
+
this->store = store;
+ this->thread_pool = thread_pool;
this->bootstrap_auth_key = auth_key;
this->max_memory_ratio = max_memory_ratio;
}
+// used only in tests!
+void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key) {
+ ThreadPool* thread_pool = new ThreadPool(8);
+ init(store, thread_pool, max_memory_ratio, auth_key);
+}
+
Option<bool> CollectionManager::load(const size_t init_batch_size) {
// This function must be idempotent, i.e. when called multiple times, must produce the same state without leaks
LOG(INFO) << "CollectionManager::load()";
@@ -120,12 +129,13 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
uint32_t collection_next_seq_id = next_seq_id_status == StoreStatus::NOT_FOUND ? 0 :
StringUtils::deserialize_uint32_t(collection_next_seq_id_str);
- if(get_collection(this_collection_name)) {
+ if(get_collection(this_collection_name) != nullptr) {
// To maintain idempotency, if the collection already exists in-memory, drop it from memory
LOG(WARNING) << "Dropping duplicate collection " << this_collection_name << " before loading it again.";
drop_collection(this_collection_name, false);
}
+ std::unique_lock lock(mutex);
Collection* collection = init_collection(collection_meta, collection_next_seq_id);
LOG(INFO) << "Loading collection " << collection->get_name();
@@ -252,6 +262,8 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
void CollectionManager::dispose() {
+ std::unique_lock lock(mutex);
+
for(auto & name_collection: collections) {
delete name_collection.second;
name_collection.second = nullptr;
@@ -264,7 +276,9 @@ void CollectionManager::dispose() {
bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,
const std::string& action,
const std::string& collection,
- std::map<std::string, std::string>& params) {
+ std::map<std::string, std::string>& params) const {
+ std::shared_lock lock(mutex);
+
if(auth_key_sent.empty()) {
return false;
}
@@ -283,6 +297,8 @@ Option<Collection*> CollectionManager::create_collection(const std::string& name
const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at) {
+ std::unique_lock lock(mutex);
+
if(store->contains(Collection::get_meta_key(name))) {
return Option<Collection*>(409, std::string("A collection with name `") + name + "` already exists.");
}
@@ -328,7 +344,7 @@ Option<Collection*> CollectionManager::create_collection(const std::string& name
nlohmann::json collection_meta;
collection_meta[COLLECTION_NAME_KEY] = name;
- collection_meta[COLLECTION_ID_KEY] = next_collection_id;
+ collection_meta[COLLECTION_ID_KEY] = next_collection_id.load();
collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
collection_meta[COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;
collection_meta[COLLECTION_CREATED] = created_at;
@@ -354,7 +370,7 @@ Option<Collection*> CollectionManager::create_collection(const std::string& name
return Option<Collection*>(new_collection);
}
-Collection* CollectionManager::get_collection(const std::string & collection_name) {
+Collection* CollectionManager::get_collection_unafe(const std::string & collection_name) const {
if(collections.count(collection_name) != 0) {
return collections.at(collection_name);
}
@@ -370,17 +386,27 @@ Collection* CollectionManager::get_collection(const std::string & collection_nam
return nullptr;
}
-Collection* CollectionManager::get_collection_with_id(uint32_t collection_id) {
+locked_resource_view_t<Collection> CollectionManager::get_collection(const std::string & collection_name) const {
+ std::shared_lock lock(mutex);
+ Collection* coll = get_collection_unafe(collection_name);
+ return locked_resource_view_t<Collection>(mutex, coll);
+}
+
+locked_resource_view_t<Collection> CollectionManager::get_collection_with_id(uint32_t collection_id) const {
+ std::shared_lock lock(mutex);
+
if(collection_id_names.count(collection_id) != 0) {
return get_collection(collection_id_names.at(collection_id));
}
- return nullptr;
+ return locked_resource_view_t<Collection>(mutex, nullptr);
}
-std::vector<Collection*> CollectionManager::get_collections() {
+std::vector<Collection*> CollectionManager::get_collections() const {
+ std::shared_lock lock(mutex);
+
std::vector<Collection*> collection_vec;
- for(auto kv: collections) {
+ for(const auto& kv: collections) {
collection_vec.push_back(kv.second);
}
@@ -392,12 +418,16 @@ std::vector<Collection*> CollectionManager::get_collections() {
return collection_vec;
}
-Option<bool> CollectionManager::drop_collection(const std::string& collection_name, const bool remove_from_store) {
- Collection* collection = get_collection(collection_name);
+Option<nlohmann::json> CollectionManager::drop_collection(const std::string& collection_name, const bool remove_from_store) {
+ std::unique_lock lock(mutex);
+
+ Collection* collection = get_collection_unafe(collection_name);
if(collection == nullptr) {
- return Option<bool>(404, "No collection with name `" + collection_name + "` found.");
+ return Option<nlohmann::json>(404, "No collection with name `" + collection_name + "` found.");
}
+ nlohmann::json collection_json = collection->get_summary_json();
+
if(remove_from_store) {
const std::string &collection_id_str = std::to_string(collection->get_collection_id());
@@ -418,10 +448,10 @@ Option<bool> CollectionManager::drop_collection(const std::string& collection_na
delete collection;
- return Option<bool>(true);
+ return Option<nlohmann::json>(collection_json);
}
-uint32_t CollectionManager::get_next_collection_id() {
+uint32_t CollectionManager::get_next_collection_id() const {
return next_collection_id;
}
@@ -429,15 +459,14 @@ std::string CollectionManager::get_symlink_key(const std::string & symlink_name)
return std::string(SYMLINK_PREFIX) + "_" + symlink_name;
}
-void CollectionManager::set_next_collection_id(uint32_t next_id) {
- next_collection_id = next_id;
-}
-
-spp::sparse_hash_map<std::string, std::string> & CollectionManager::get_symlinks() {
+spp::sparse_hash_map<std::string, std::string> CollectionManager::get_symlinks() const {
+ std::shared_lock lock(mutex);
return collection_symlinks;
}
-Option<std::string> CollectionManager::resolve_symlink(const std::string & symlink_name) {
+Option<std::string> CollectionManager::resolve_symlink(const std::string & symlink_name) const {
+ std::shared_lock lock(mutex);
+
if(collection_symlinks.count(symlink_name) != 0) {
return Option<std::string>(collection_symlinks.at(symlink_name));
}
@@ -446,6 +475,8 @@ Option<std::string> CollectionManager::resolve_symlink(const std::string & symli
}
Option<bool> CollectionManager::upsert_symlink(const std::string & symlink_name, const std::string & collection_name) {
+ std::unique_lock lock(mutex);
+
if(collections.count(symlink_name) != 0) {
return Option<bool>(500, "Name `" + symlink_name + "` conflicts with an existing collection name.");
}
@@ -460,6 +491,8 @@ Option<bool> CollectionManager::upsert_symlink(const std::string & symlink_name,
}
Option<bool> CollectionManager::delete_symlink(const std::string & symlink_name) {
+ std::unique_lock lock(mutex);
+
bool removed = store->remove(get_symlink_key(symlink_name));
if(!removed) {
return Option<bool>(500, "Unable to delete from store.");
@@ -711,7 +744,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
}
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req_params["collection"]);
+ auto collection = collectionManager.get_collection(req_params["collection"]);
if(collection == nullptr) {
return Option<bool>(404, "Not found.");
@@ -767,3 +800,21 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
return Option<bool>(true);
}
+
+ThreadPool* CollectionManager::get_thread_pool() const {
+ return thread_pool;
+}
+
+nlohmann::json CollectionManager::get_collection_summaries() const {
+ std::shared_lock lock(mutex);
+
+ std::vector<Collection*> colls = get_collections();
+ nlohmann::json json_summaries = nlohmann::json::array();
+
+ for(Collection* collection: colls) {
+ nlohmann::json collection_json = collection->get_summary_json();
+ json_summaries.push_back(collection_json);
+ }
+
+ return json_summaries;
+}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index ffbe8f38..56adb91a 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -43,14 +43,7 @@ index_operation_t get_index_operation(const std::string& action) {
bool get_collections(http_req & req, http_res & res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- std::vector<Collection*> collections = collectionManager.get_collections();
- nlohmann::json json_response = nlohmann::json::array();
-
- for(Collection* collection: collections) {
- nlohmann::json collection_json = collection->get_summary_json();
- json_response.push_back(collection_json);
- }
-
+ nlohmann::json json_response = collectionManager.get_collection_summaries();
res.set_200(json_response.dump());
return true;
}
@@ -170,22 +163,14 @@ bool post_create_collection(http_req & req, http_res & res) {
bool del_drop_collection(http_req & req, http_res & res) {
std::string doc_id = req.params["id"];
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ Option<nlohmann::json> drop_op = collectionManager.drop_collection(req.params["collection"]);
- if(!collection) {
- res.set_404();
+ if(!drop_op.ok()) {
+ res.set(drop_op.code(), drop_op.error());
return false;
}
- nlohmann::json collection_json = collection->get_summary_json();
- Option<bool> drop_result = collectionManager.drop_collection(req.params["collection"]);
-
- if(!drop_result.ok()) {
- res.set(drop_result.code(), drop_result.error());
- return false;
- }
-
- res.set_200(collection_json.dump());
+ res.set_200(drop_op.get().dump());
return true;
}
@@ -349,8 +334,8 @@ bool post_multi_search(http_req& req, http_res& res) {
}
bool get_collection_summary(http_req & req, http_res & res) {
- CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ CollectionManager& collectionManager = CollectionManager::get_instance();
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -366,7 +351,7 @@ bool get_collection_summary(http_req & req, http_res & res) {
bool get_export_documents(http_req & req, http_res & res) {
// NOTE: this is a streaming response end-point so this handler will be called multiple times
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
req.last_chunk_aggregate = true;
@@ -459,7 +444,7 @@ bool post_import_documents(http_req& req, http_res& res) {
}
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
req.last_chunk_aggregate = true;
@@ -564,7 +549,7 @@ bool post_add_document(http_req & req, http_res & res) {
}
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -587,7 +572,7 @@ bool patch_update_document(http_req & req, http_res & res) {
std::string doc_id = req.params["id"];
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -609,7 +594,7 @@ bool get_fetch_document(http_req & req, http_res & res) {
std::string doc_id = req.params["id"];
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
return false;
@@ -630,7 +615,7 @@ bool del_remove_document(http_req & req, http_res & res) {
std::string doc_id = req.params["id"];
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
return false;
@@ -662,7 +647,7 @@ bool del_remove_documents(http_req & req, http_res & res) {
// NOTE: this is a streaming response end-point so this handler will be called multiple times
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
req.last_chunk_aggregate = true;
@@ -728,7 +713,7 @@ bool del_remove_documents(http_req & req, http_res & res) {
for(size_t i=0; i<deletion_state->index_ids.size(); i++) {
deletion_state->offsets.push_back(0);
}
- deletion_state->collection = collection;
+ deletion_state->collection = collection.get();
deletion_state->num_removed = 0;
req.data = deletion_state;
} else {
@@ -855,7 +840,7 @@ bool del_alias(http_req & req, http_res & res) {
bool get_overrides(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -865,7 +850,7 @@ bool get_overrides(http_req &req, http_res &res) {
nlohmann::json res_json;
res_json["overrides"] = nlohmann::json::array();
- std::map<std::string, override_t>& overrides = collection->get_overrides();
+ const std::map<std::string, override_t>& overrides = collection->get_overrides();
for(const auto & kv: overrides) {
nlohmann::json override = kv.second.to_json();
res_json["overrides"].push_back(override);
@@ -877,7 +862,7 @@ bool get_overrides(http_req &req, http_res &res) {
bool get_override(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -886,10 +871,10 @@ bool get_override(http_req &req, http_res &res) {
std::string override_id = req.params["id"];
- std::map<std::string, override_t>& overrides = collection->get_overrides();
+ const std::map<std::string, override_t>& overrides = collection->get_overrides();
if(overrides.count(override_id) != 0) {
- nlohmann::json override = overrides[override_id].to_json();
+ nlohmann::json override = overrides.at(override_id).to_json();
res.set_200(override.dump());
return true;
}
@@ -900,7 +885,7 @@ bool get_override(http_req &req, http_res &res) {
bool put_override(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
std::string override_id = req.params["id"];
@@ -941,7 +926,7 @@ bool put_override(http_req &req, http_res &res) {
bool del_override(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -1170,7 +1155,7 @@ bool post_config(http_req &req, http_res &res) {
bool get_synonyms(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -1180,7 +1165,7 @@ bool get_synonyms(http_req &req, http_res &res) {
nlohmann::json res_json;
res_json["synonyms"] = nlohmann::json::array();
- auto& synonyms = collection->get_synonyms();
+ const auto& synonyms = collection->get_synonyms();
for(const auto & kv: synonyms) {
nlohmann::json synonym = kv.second.to_view_json();
res_json["synonyms"].push_back(synonym);
@@ -1192,7 +1177,7 @@ bool get_synonyms(http_req &req, http_res &res) {
bool get_synonym(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
@@ -1216,7 +1201,7 @@ bool get_synonym(http_req &req, http_res &res) {
bool put_synonym(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
std::string synonym_id = req.params["id"];
@@ -1258,7 +1243,7 @@ bool put_synonym(http_req &req, http_res &res) {
bool del_synonym(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection = collectionManager.get_collection(req.params["collection"]);
+ auto collection = collectionManager.get_collection(req.params["collection"]);
if(collection == nullptr) {
res.set_404();
diff --git a/src/http_server.cpp b/src/http_server.cpp
index e9f71d8a..4db5e12d 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -13,9 +13,10 @@
HttpServer::HttpServer(const std::string & version, const std::string & listen_address,
uint32_t listen_port, const std::string & ssl_cert_path,
- const std::string & ssl_cert_key_path, bool cors_enabled):
+ const std::string & ssl_cert_key_path, bool cors_enabled, ThreadPool* thread_pool):
version(version), listen_address(listen_address), listen_port(listen_port),
- ssl_cert_path(ssl_cert_path), ssl_cert_key_path(ssl_cert_key_path), cors_enabled(cors_enabled) {
+ ssl_cert_path(ssl_cert_path), ssl_cert_key_path(ssl_cert_key_path),
+ cors_enabled(cors_enabled), thread_pool(thread_pool) {
accept_ctx = new h2o_accept_ctx_t();
h2o_config_init(&config);
hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);
@@ -544,6 +545,23 @@ int HttpServer::process_request(http_req* request, http_res* response, route_pat
return 0;
}
+ auto message_dispatcher = handler->http_server->get_message_dispatcher();
+ auto api_handler = rpath->handler;
+
+ handler->http_server->get_thread_pool()->enqueue([api_handler, message_dispatcher, request, response]() {
+ (api_handler)(*request, *response);
+ request_response req_res{request, response};
+ response->auto_dispose = false;
+ message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, &req_res);
+
+ // wait until response is sent
+ response->await.wait();
+
+ delete request;
+ delete response;
+ });
+
+ /*
// for reads, we will invoke the request handler and handle response as well
(rpath->handler)(*request, *response);
@@ -552,6 +570,7 @@ int HttpServer::process_request(http_req* request, http_res* response, route_pat
// otherwise, we send the whole body response on behalf of the handler
handler->http_server->send_response(request, response);
}
+ */
return 0;
}
@@ -861,3 +880,7 @@ void HttpServer::do_snapshot(const std::string& snapshot_path, http_req& req, ht
bool HttpServer::trigger_vote() {
return replication_state->trigger_vote();
}
+
+ThreadPool* HttpServer::get_thread_pool() const {
+ return thread_pool;
+}
diff --git a/src/index.cpp b/src/index.cpp
index 2ec98834..9975a91a 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -39,10 +39,6 @@ Index::Index(const std::string name, const std::unordered_map<std::string, field
}
num_documents = 0;
-
- ready = false;
- processed = false;
- terminate = false;
}
Index::~Index() {
@@ -98,6 +94,8 @@ int64_t Index::float_to_in64_t(float f) {
Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,
const std::string & default_sorting_field, bool is_update) {
+ std::unique_lock lock(mutex);
+
int64_t points = 0;
if(is_update && document.count(default_sorting_field) == 0) {
@@ -640,7 +638,8 @@ void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::s
}
void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
- const uint32_t* result_ids, size_t results_size) {
+ size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const uint32_t* result_ids, size_t results_size) const {
std::unordered_map<std::string, size_t> facet_to_index;
get_facet_to_index(facet_to_index);
@@ -723,7 +722,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
}
const std::vector<std::vector<uint64_t>>& doc_facet_index = doc_facet_index_it->second;
- const uint64_t distinct_id = search_params->group_limit ? get_distinct_id(facet_to_index, doc_seq_id) : 0;
+ const uint64_t distinct_id = group_limit ? get_distinct_id(facet_to_index, group_by_fields, doc_seq_id) : 0;
// assumed that facet fields have already been validated upstream
for(size_t findex=0; findex < facets.size(); findex++) {
@@ -797,7 +796,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
facet_count.doc_id = doc_seq_id;
facet_count.array_pos = array_pos;
- if(search_params->group_limit) {
+ if(group_limit) {
facet_count.groups.emplace(distinct_id);
} else {
facet_count.count += 1;
@@ -819,7 +818,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
}
}
-void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) {
+void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) const {
size_t i_facet = 0;
for(const auto & facet_kv: facet_schema) {
facet_to_index.emplace(facet_kv.first, i_facet);
@@ -837,7 +836,8 @@ void Index::search_candidates(const uint8_t & field_id,
spp::sparse_hash_set<uint64_t>& groups_processed,
uint32_t** all_result_ids, size_t & all_result_ids_len,
size_t& field_num_results,
- const size_t typo_tokens_threshold) {
+ const size_t typo_tokens_threshold,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields) const {
const long long combination_limit = 10;
auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };
@@ -921,7 +921,8 @@ void Index::search_candidates(const uint8_t & field_id,
// go through each matching document id and calculate match score
score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
- groups_processed, filtered_result_ids, filtered_results_size, token_bits);
+ groups_processed, filtered_result_ids, filtered_results_size,
+ group_limit, group_by_fields, token_bits);
field_num_results += filtered_results_size;
@@ -939,7 +940,7 @@ void Index::search_candidates(const uint8_t & field_id,
}*/
score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster, query_suggestion,
- groups_processed, result_ids, result_size, token_bits);
+ groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits);
field_num_results += result_size;
@@ -955,7 +956,7 @@ void Index::search_candidates(const uint8_t & field_id,
}
}
-Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {
+uint32_t Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) const {
//auto begin = std::chrono::high_resolution_clock::now();
uint32_t* filter_ids = nullptr;
@@ -1078,7 +1079,7 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
size_t facet_id = facet_to_index[f.name];
for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {
uint32_t seq_id = strt_ids[strt_ids_index];
- const std::vector<uint64_t> &fvalues = facet_index_v2[seq_id][facet_id];
+ const std::vector<uint64_t> &fvalues = facet_index_v2.at(seq_id).at(facet_id);
bool found_filter = false;
if(!f.is_array()) {
@@ -1156,7 +1157,7 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
*filter_ids_out = filter_ids;
- return Option<>(filter_ids_length);
+ return filter_ids_length;
}
void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
@@ -1212,46 +1213,28 @@ void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
}
}
-void Index::run_search() {
- while(true) {
- // wait until main thread sends data
- std::unique_lock<std::mutex> lk(m);
- cv.wait(lk, [this]{return ready;});
-
- if(terminate) {
- break;
- }
-
- // after the wait, we own the lock.
- search(search_params->outcome, search_params->q_include_tokens, search_params->q_exclude_tokens,
- search_params->q_synonyms,
- search_params->search_fields,
- search_params->filters, search_params->facets, search_params->facet_query,
- search_params->included_ids, search_params->excluded_ids,
- search_params->sort_fields_std, search_params->num_typos,
- search_params->topster, search_params->curated_topster,
- search_params->per_page, search_params->page, search_params->token_order,
- search_params->prefix, search_params->drop_tokens_threshold,
- search_params->all_result_ids_len, search_params->groups_processed,
- search_params->searched_queries,
- search_params->raw_result_kvs, search_params->override_result_kvs,
- search_params->typo_tokens_threshold);
-
- // hand control back to main thread
- processed = true;
- ready = false;
-
- // manual unlocking is done before notifying, to avoid waking up the waiting thread only to block again
- lk.unlock();
- cv.notify_one();
- }
+void Index::run_search(search_args* search_params) {
+ search(search_params->q_include_tokens, search_params->q_exclude_tokens,
+ search_params->q_synonyms,
+ search_params->search_fields,
+ search_params->filters, search_params->facets, search_params->facet_query,
+ search_params->included_ids, search_params->excluded_ids,
+ search_params->sort_fields_std, search_params->num_typos,
+ search_params->topster, search_params->curated_topster,
+ search_params->per_page, search_params->page, search_params->token_order,
+ search_params->prefix, search_params->drop_tokens_threshold,
+ search_params->all_result_ids_len, search_params->groups_processed,
+ search_params->searched_queries,
+ search_params->raw_result_kvs, search_params->override_result_kvs,
+ search_params->typo_tokens_threshold,
+ search_params->group_limit, search_params->group_by_fields);
}
void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,
const std::string & field, const uint8_t field_id,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
Topster* curated_topster,
- std::vector<std::vector<art_leaf*>> & searched_queries) {
+ std::vector<std::vector<art_leaf*>> & searched_queries) const {
if(included_ids_map.empty()) {
return;
@@ -1316,8 +1299,7 @@ void Index::concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t,
}
}
-void Index::search(Option<uint32_t> & outcome,
- const std::vector<std::string>& q_include_tokens,
+void Index::search(const std::vector<std::string>& q_include_tokens,
const std::vector<std::string>& q_exclude_tokens,
const std::vector<std::vector<std::string>>& q_synonyms,
const std::vector<search_field_t>& search_fields,
@@ -1335,20 +1317,18 @@ void Index::search(Option<uint32_t> & outcome,
std::vector<std::vector<art_leaf*>>& searched_queries,
std::vector<std::vector<KV*>> & raw_result_kvs,
std::vector<std::vector<KV*>> & override_result_kvs,
- const size_t typo_tokens_threshold) {
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields) const {
+
+ std::shared_lock lock(mutex);
//auto begin = std::chrono::high_resolution_clock::now();
// process the filters
uint32_t* filter_ids = nullptr;
- Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, filters);
- if(!op_filter_ids_length.ok()) {
- outcome = Option<uint32_t>(op_filter_ids_length);
- return ;
- }
-
- uint32_t filter_ids_length = op_filter_ids_length.get();
+ uint32_t filter_ids_length = do_filtering(&filter_ids, filters);
// we will be removing all curated IDs from organic result ids before running topster
std::set<uint32_t> curated_ids;
@@ -1413,7 +1393,7 @@ void Index::search(Option<uint32_t> & outcome,
}
}
- const spp::sparse_hash_map<uint32_t, int64_t> *kvs = sort_index[all_records_field];
+ const spp::sparse_hash_map<uint32_t, int64_t> *kvs = sort_index.at(all_records_field);
filter_ids_length = kvs->size();
filter_ids = new uint32_t[filter_ids_length];
@@ -1445,7 +1425,7 @@ void Index::search(Option<uint32_t> & outcome,
uint32_t token_bits = 255;
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
- groups_processed, filter_ids, filter_ids_length, token_bits);
+ groups_processed, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits);
collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
all_result_ids_len = filter_ids_length;
@@ -1494,7 +1474,8 @@ void Index::search(Option<uint32_t> & outcome,
search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
- field_num_results, token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
+ field_num_results, group_limit, group_by_fields, token_order, prefix,
+ drop_tokens_threshold, typo_tokens_threshold);
// do synonym based searches
for(const auto& syn_tokens: q_pos_synonyms) {
@@ -1505,7 +1486,8 @@ void Index::search(Option<uint32_t> & outcome,
search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
- field_num_results, token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
+ field_num_results, group_limit, group_by_fields, token_order, prefix,
+ drop_tokens_threshold, typo_tokens_threshold);
}
concat_topster_ids(ftopster, topster_ids);
@@ -1613,8 +1595,8 @@ void Index::search(Option<uint32_t> & outcome,
delete [] exclude_token_ids;
- do_facets(facets, facet_query, all_result_ids, all_result_ids_len);
- do_facets(facets, facet_query, &included_ids[0], included_ids.size());
+ do_facets(facets, facet_query, group_limit, group_by_fields, all_result_ids, all_result_ids_len);
+ do_facets(facets, facet_query, group_limit, group_by_fields, &included_ids[0], included_ids.size());
all_result_ids_len += curated_topster->size;
@@ -1625,10 +1607,9 @@ void Index::search(Option<uint32_t> & outcome,
delete ftopster;
}
+ //LOG(INFO) << "all_result_ids_len " << all_result_ids_len << " for index " << name;
//long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";
-
- outcome = Option<uint32_t>(1);
}
/*
@@ -1652,10 +1633,10 @@ void Index::search_field(const uint8_t & field_id,
std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,
std::vector<std::vector<art_leaf*>> & searched_queries,
Topster* topster, spp::sparse_hash_set<uint64_t>& groups_processed,
- uint32_t** all_result_ids, size_t & all_result_ids_len,
- size_t& field_num_results,
+ uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
const token_ordering token_order, const bool prefix,
- const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) {
+ const size_t drop_tokens_threshold, const size_t typo_tokens_threshold) const {
const size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
@@ -1758,7 +1739,7 @@ void Index::search_field(const uint8_t & field_id,
search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,
curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,
groups_processed, all_result_ids, all_result_ids_len, field_num_results,
- typo_tokens_threshold);
+ typo_tokens_threshold, group_limit, group_by_fields);
}
resume_typo_loop:
@@ -1796,11 +1777,12 @@ void Index::search_field(const uint8_t & field_id,
return search_field(field_id, query_tokens, truncated_tokens, exclude_token_ids, exclude_token_ids_size,
num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids,facets,
sort_fields, num_typos,searched_queries, topster, groups_processed, all_result_ids,
- all_result_ids_len, field_num_results, token_order, prefix);
+ all_result_ids_len, field_num_results, group_limit, group_by_fields,
+ token_order, prefix);
}
}
-int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const {
+int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) {
int bounded_cost = max_cost;
if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {
bounded_cost = token_len - 1;
@@ -1825,7 +1807,9 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
const uint8_t & field_id, const uint32_t total_cost, Topster* topster,
const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const uint32_t *result_ids, const size_t result_size, uint32_t token_bits) {
+ const uint32_t *result_ids, const size_t result_size,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ uint32_t token_bits) const {
std::vector<uint32_t*> leaf_to_indices;
for (art_leaf *token_leaf: query_suggestion) {
@@ -1935,8 +1919,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
uint64_t distinct_id = seq_id;
- if(search_params->group_limit != 0) {
- distinct_id = get_distinct_id(facet_to_index, seq_id);
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(facet_to_index, group_by_fields, seq_id);
groups_processed.emplace(distinct_id);
}
@@ -1953,11 +1937,12 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
uint64_t Index::get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id,
+ const std::vector<std::string>& group_by_fields,
const uint32_t seq_id) const {
uint64_t distinct_id = 1; // some constant initial value
// calculate hash from group_by_fields
- for(const auto& field: search_params->group_by_fields) {
+ for(const auto& field: group_by_fields) {
if(facet_to_id.count(field) == 0 || facet_index_v2.count(seq_id) == 0) {
continue;
}
@@ -1973,8 +1958,8 @@ uint64_t Index::get_distinct_id(const std::unordered_map<std::string, size_t> &f
}
void Index::populate_token_positions(const std::vector<art_leaf *>& query_suggestion,
- std::vector<uint32_t*>& leaf_to_indices,
- size_t result_index,
+ const std::vector<uint32_t*>& leaf_to_indices,
+ const size_t result_index,
std::unordered_map<size_t, std::vector<std::vector<uint16_t>>>& array_token_positions) {
if(query_suggestion.empty()) {
return ;
@@ -2102,6 +2087,8 @@ void Index::remove_and_shift_offset_index(sorted_array& offset_index, const uint
}
Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document) {
+ std::unique_lock lock(mutex);
+
std::unordered_map<std::string, size_t> facet_to_index;
get_facet_to_index(facet_to_index);
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index d24be610..ee405d83 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -360,9 +360,11 @@ int run_server(const Config & config, const std::string & version, void (*master
create_init_db_snapshot = true;
}
+ ThreadPool thread_pool(32);
Store store(db_dir);
+
CollectionManager & collectionManager = CollectionManager::get_instance();
- collectionManager.init(&store, config.get_max_memory_ratio(), config.get_api_key());
+ collectionManager.init(&store, &thread_pool, config.get_max_memory_ratio(), config.get_api_key());
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
@@ -374,7 +376,8 @@ int run_server(const Config & config, const std::string & version, void (*master
config.get_api_port(),
config.get_ssl_cert(),
config.get_ssl_cert_key(),
- config.get_enable_cors()
+ config.get_enable_cors(),
+ &thread_pool
);
server->set_auth_handler(handle_authentication);
@@ -388,7 +391,6 @@ int run_server(const Config & config, const std::string & version, void (*master
// first we start the peering service
- ThreadPool thread_pool(32);
ReplicationState replication_state(&store, &thread_pool, server->get_message_dispatcher(),
ssl_enabled, config.get_catch_up_threshold_percentage(),
create_init_db_snapshot, quit_raft_service);
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index e192a2c2..d1d90763 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -47,7 +47,7 @@ TEST_F(CollectionFacetingTest, FacetFieldStringFiltering) {
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
- coll_str = collectionManager.get_collection("coll_str");
+ coll_str = collectionManager.get_collection("coll_str").get();
if(coll_str == nullptr) {
coll_str = collectionManager.create_collection("coll_str", 1, fields, "points").get();
}
@@ -112,7 +112,7 @@ TEST_F(CollectionFacetingTest, FacetFieldStringArrayFiltering) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", 1, fields, "age").get();
}
@@ -194,7 +194,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
@@ -504,7 +504,7 @@ TEST_F(CollectionFacetingTest, FacetCountsBool) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -558,7 +558,7 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -694,7 +694,7 @@ TEST_F(CollectionFacetingTest, FacetStatOnFloatFields) {
std::vector<sort_by> sort_fields_desc = { sort_by("average", "DESC") };
- coll_float_fields = collectionManager.get_collection("coll_float_fields");
+ coll_float_fields = collectionManager.get_collection("coll_float_fields").get();
if(coll_float_fields == nullptr) {
coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "average").get();
}
@@ -756,7 +756,7 @@ TEST_F(CollectionFacetingTest, FacetCountOnSimilarStrings) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 62982f19..81dcd370 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -47,7 +47,7 @@ TEST_F(CollectionFilteringTest, FilterOnTextFields) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
@@ -127,7 +127,7 @@ TEST_F(CollectionFilteringTest, FilterOnTextFieldWithColon) {
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -165,7 +165,7 @@ TEST_F(CollectionFilteringTest, HandleBadlyFormedFilterQuery) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
@@ -219,7 +219,7 @@ TEST_F(CollectionFilteringTest, FilterAndQueryFieldRestrictions) {
field("points", field_types::INT32, false)
};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -332,7 +332,7 @@ TEST_F(CollectionFilteringTest, FilterOnNumericFields) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if(coll_array_fields == nullptr) {
// ensure that default_sorting_field is a non-array numerical field
auto coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "years");
@@ -529,7 +529,7 @@ TEST_F(CollectionFilteringTest, FilterOnFloatFields) {
std::vector<sort_by> sort_fields_desc = { sort_by("rating", "DESC") };
std::vector<sort_by> sort_fields_asc = { sort_by("rating", "ASC") };
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
@@ -669,7 +669,7 @@ TEST_F(CollectionFilteringTest, ComparatorsOnMultiValuedNumericalField) {
std::vector<sort_by> sort_fields_desc = {sort_by("rating", "DESC")};
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if (coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
@@ -724,7 +724,7 @@ TEST_F(CollectionFilteringTest, FilteringWithPrefixSearch) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 6f35b2da..06f2bde8 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -29,7 +29,7 @@ protected:
field("rating", field_types::FLOAT, true, false)
};
- coll_group = collectionManager.get_collection("coll_group");
+ coll_group = collectionManager.get_collection("coll_group").get();
if(coll_group == nullptr) {
coll_group = collectionManager.create_collection("coll_group", 4, fields, "rating").get();
}
@@ -256,7 +256,7 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {
field("genre", field_types::STRING, true),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index bc642d97..b1faa27e 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -47,7 +47,7 @@ protected:
TEST_F(CollectionManagerTest, CollectionCreation) {
CollectionManager & collectionManager2 = CollectionManager::get_instance();
- collection1 = collectionManager2.get_collection("collection1");
+ collection1 = collectionManager2.get_collection("collection1").get();
ASSERT_NE(nullptr, collection1);
std::unordered_map<std::string, field> schema = collection1->get_schema();
@@ -239,7 +239,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
ASSERT_TRUE(load_op.ok());
- collection1 = collectionManager2.get_collection("collection1");
+ collection1 = collectionManager2.get_collection("collection1").get();
ASSERT_NE(nullptr, collection1);
std::vector<std::string> facet_fields_expected = {"cast"};
@@ -262,16 +262,16 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
ASSERT_STREQ("exclude-rule", collection1->get_overrides()["exclude-rule"].id.c_str());
ASSERT_STREQ("include-rule", collection1->get_overrides()["include-rule"].id.c_str());
- auto& synonyms = collection1->get_synonyms();
+ const auto& synonyms = collection1->get_synonyms();
ASSERT_EQ(2, synonyms.size());
- ASSERT_STREQ("id1", synonyms["id1"].id.c_str());
- ASSERT_EQ(2, synonyms["id1"].root.size());
- ASSERT_EQ(1, synonyms["id1"].synonyms.size());
+ ASSERT_STREQ("id1", synonyms.at("id1").id.c_str());
+ ASSERT_EQ(2, synonyms.at("id1").root.size());
+ ASSERT_EQ(1, synonyms.at("id1").synonyms.size());
- ASSERT_STREQ("id3", synonyms["id3"].id.c_str());
- ASSERT_EQ(0, synonyms["id3"].root.size());
- ASSERT_EQ(2, synonyms["id3"].synonyms.size());
+ ASSERT_STREQ("id3", synonyms.at("id3").id.c_str());
+ ASSERT_EQ(0, synonyms.at("id3").root.size());
+ ASSERT_EQ(2, synonyms.at("id3").synonyms.size());
results = collection1->search("thomas", search_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(4, results["hits"].size());
@@ -287,8 +287,8 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {
infile.close();
- ASSERT_FALSE(nullptr == collectionManager.get_collection_with_id(0));
- ASSERT_FALSE(nullptr == collectionManager.get_collection("collection1"));
+ ASSERT_FALSE(nullptr == collectionManager.get_collection_with_id(0).get());
+ ASSERT_FALSE(nullptr == collectionManager.get_collection("collection1").get());
collectionManager.drop_collection("collection1");
@@ -303,8 +303,8 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {
ASSERT_EQ(1, num_keys);
ASSERT_TRUE(it->status().ok());
- ASSERT_EQ(nullptr, collectionManager.get_collection("collection1"));
- ASSERT_EQ(nullptr, collectionManager.get_collection_with_id(0));
+ ASSERT_EQ(nullptr, collectionManager.get_collection("collection1").get());
+ ASSERT_EQ(nullptr, collectionManager.get_collection_with_id(0).get());
ASSERT_EQ(1, collectionManager.get_next_collection_id());
delete it;
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index 37120de4..ac49e6e9 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -29,7 +29,7 @@ protected:
field("points", field_types::INT32, false)
};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -628,7 +628,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWithWildCardQuery) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 3, fields, "points").get();
}
@@ -675,7 +675,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsIdsHavingColon) {
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index b488ea16..3ad37630 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -43,7 +43,7 @@ TEST_F(CollectionSortingTest, SortingOrder) {
field("points", field_types::INT32, false),
field("cast", field_types::STRING_ARRAY, false)};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -150,7 +150,7 @@ TEST_F(CollectionSortingTest, Int64AsDefaultSortingField) {
field("points", field_types::INT64, false),
field("cast", field_types::STRING_ARRAY, false)};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -209,7 +209,7 @@ TEST_F(CollectionSortingTest, SortOnFloatFields) {
std::vector<sort_by> sort_fields_desc = { sort_by("score", "DESC"), sort_by("average", "DESC") };
- coll_float_fields = collectionManager.get_collection("coll_float_fields");
+ coll_float_fields = collectionManager.get_collection("coll_float_fields").get();
if(coll_float_fields == nullptr) {
coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "score").get();
}
@@ -277,7 +277,7 @@ TEST_F(CollectionSortingTest, ThreeSortFieldsLimit) {
field("max", field_types::INT32, false),
};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -316,7 +316,7 @@ TEST_F(CollectionSortingTest, NegativeInt64Value) {
field("points", field_types::INT64, false),
};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index b87e7335..322b0f9a 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -29,7 +29,7 @@ protected:
field("points", field_types::INT32, false)
};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -376,7 +376,7 @@ TEST_F(CollectionSynonymsTest, ExactMatchRankedSameAsSynonymMatch) {
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -431,7 +431,7 @@ TEST_F(CollectionSynonymsTest, SynonymFieldOrdering) {
field("description", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index c21c07f3..0f0ff121 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -35,7 +35,7 @@ protected:
query_fields = {"title"};
sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };
- collection = collectionManager.get_collection("collection");
+ collection = collectionManager.get_collection("collection").get();
if(collection == nullptr) {
collection = collectionManager.create_collection("collection", 4, search_fields, "points").get();
}
@@ -478,6 +478,9 @@ TEST_F(CollectionTest, Pagination) {
TEST_F(CollectionTest, WildcardQuery) {
nlohmann::json results = collection->search("*", query_fields, "points:>0", {}, sort_fields, 0, 3, 1, FREQUENCY,
false).get();
+
+ LOG(INFO) << results;
+
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ(25, results["found"].get<uint32_t>());
@@ -626,7 +629,7 @@ TEST_F(CollectionTest, MultiOccurrenceString) {
field("points", field_types::INT32, false)
};
- coll_multi_string = collectionManager.get_collection("coll_multi_string");
+ coll_multi_string = collectionManager.get_collection("coll_multi_string").get();
if (coll_multi_string == nullptr) {
coll_multi_string = collectionManager.create_collection("coll_multi_string", 4, fields, "points").get();
}
@@ -654,7 +657,7 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
field("points", field_types::INT32, false)
};
- coll_array_text = collectionManager.get_collection("coll_array_text");
+ coll_array_text = collectionManager.get_collection("coll_array_text").get();
if (coll_array_text == nullptr) {
coll_array_text = collectionManager.create_collection("coll_array_text", 4, fields, "points").get();
}
@@ -837,7 +840,7 @@ TEST_F(CollectionTest, MultipleFields) {
field("points", field_types::INT32, false)
};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -956,7 +959,7 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {
field("points", field_types::INT32, false)
};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -1044,7 +1047,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
field("points", field_types::INT32, false)
};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 1, fields, "points").get();
}
@@ -1195,7 +1198,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsertOptional) {
field("points", field_types::INT32, false)
};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -1289,7 +1292,7 @@ TEST_F(CollectionTest, ImportDocuments) {
field("points", field_types::INT32, false)
};
- coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields").get();
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
@@ -1422,7 +1425,7 @@ TEST_F(CollectionTest, QueryBoolFields) {
std::vector<sort_by> sort_fields = { sort_by("popular", "DESC"), sort_by("rating", "DESC") };
- coll_bool = collectionManager.get_collection("coll_bool");
+ coll_bool = collectionManager.get_collection("coll_bool").get();
if(coll_bool == nullptr) {
coll_bool = collectionManager.create_collection("coll_bool", 4, fields, "rating").get();
}
@@ -1531,7 +1534,7 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
- coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
@@ -1586,7 +1589,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
- sample_collection = collectionManager.get_collection("sample_collection");
+ sample_collection = collectionManager.get_collection("sample_collection").get();
if(sample_collection == nullptr) {
sample_collection = collectionManager.create_collection("sample_collection", 4, fields, "age").get();
}
@@ -1668,7 +1671,7 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
- empty_coll = collectionManager.get_collection("empty_coll");
+ empty_coll = collectionManager.get_collection("empty_coll").get();
if(empty_coll == nullptr) {
empty_coll = collectionManager.create_collection("empty_coll", 4, fields, "age").get();
}
@@ -1688,7 +1691,7 @@ TEST_F(CollectionTest, IdFieldShouldBeAString) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "age").get();
}
@@ -1716,7 +1719,7 @@ TEST_F(CollectionTest, AnIntegerCanBePassedToAFloatField) {
std::vector<sort_by> sort_fields = { sort_by("average", "DESC") };
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "average").get();
}
@@ -1744,7 +1747,7 @@ TEST_F(CollectionTest, DeletionOfADocument) {
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
Collection *collection_for_del;
- collection_for_del = collectionManager.get_collection("collection_for_del");
+ collection_for_del = collectionManager.get_collection("collection_for_del").get();
if(collection_for_del == nullptr) {
collection_for_del = collectionManager.create_collection("collection_for_del", 4, search_fields, "points").get();
}
@@ -1825,7 +1828,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -1898,7 +1901,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {
std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("age", "DESC") };
- coll_large_text = collectionManager.get_collection("coll_large_text");
+ coll_large_text = collectionManager.get_collection("coll_large_text").get();
if(coll_large_text == nullptr) {
coll_large_text = collectionManager.create_collection("coll_large_text", 4, fields, "age").get();
}
@@ -1991,7 +1994,7 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -2016,7 +2019,7 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -2074,7 +2077,7 @@ TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -2108,7 +2111,7 @@ TEST_F(CollectionTest, SearchHighlightWithNewLine) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -2142,7 +2145,7 @@ TEST_F(CollectionTest, UpdateDocument) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -2307,7 +2310,7 @@ TEST_F(CollectionTest, UpdateDocumentSorting) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -2365,7 +2368,7 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -2443,7 +2446,7 @@ TEST_F(CollectionTest, OptionalFields) {
field("is_valid", field_types::BOOL, false, true),
};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "max").get();
}
@@ -2588,7 +2591,7 @@ TEST_F(CollectionTest, RemoveIfFound) {
std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -2650,7 +2653,7 @@ TEST_F(CollectionTest, MultiFieldRelevance) {
field("artist", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
@@ -2675,6 +2678,8 @@ TEST_F(CollectionTest, MultiFieldRelevance) {
auto results = coll1->search("Dustin Kensrue Down There by the Train",
{"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+ LOG(INFO) << results;
+
ASSERT_EQ(3, results["found"].get<size_t>());
ASSERT_EQ(3, results["hits"].size());
@@ -2750,7 +2755,7 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {
field("artist", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -2802,7 +2807,7 @@ TEST_F(CollectionTest, FieldWeightsNotProper) {
field("artist", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -2847,7 +2852,7 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {
field("artist", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -2890,7 +2895,7 @@ TEST_F(CollectionTest, MultiFieldMatchRanking) {
field("artist", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -2940,7 +2945,7 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnArray) {
field("skills", field_types::STRING_ARRAY, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -2981,7 +2986,7 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnFieldOrder) {
field("artist", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -3020,7 +3025,7 @@ TEST_F(CollectionTest, PrefixRankedAfterExactMatch) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
}
@@ -3060,7 +3065,7 @@ TEST_F(CollectionTest, HighlightWithAccentedCharacters) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index 3160636a..ea1bfb42 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -39,7 +39,7 @@ TEST_F(CoreAPIUtilsTest, StatefulRemoveDocs) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};
- coll1 = collectionManager.get_collection("coll1");
+ coll1 = collectionManager.get_collection("coll1").get();
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", 2, fields, "points").get();
}