Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Create new collection.h and collection.cpp as a clone of index.h and index.cpp.

Kishore Nallan9856c3888a4f
diff --git a/include/collection.h b/include/collection.h
new file mode 100644
index 00000000..21954c0b
--- /dev/null
+++ b/include/collection.h
@@ -0,0 +1,191 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <art.h>
+#include <number.h>
+#include <sparsepp.h>
+#include <store.h>
+#include <topster.h>
+#include <json.hpp>
+#include <field.h>
+#include <option.h>
+
+struct facet_value {
+ // use string to int mapping for saving memory
+ spp::sparse_hash_map<std::string, uint32_t> value_index;
+ spp::sparse_hash_map<uint32_t, std::string> index_value;
+
+ spp::sparse_hash_map<uint32_t, std::vector<uint32_t>> doc_values;
+
+ uint32_t get_value_index(const std::string & value) {
+ if(value_index.count(value) != 0) {
+ return value_index[value];
+ }
+
+ uint32_t new_index = value_index.size();
+ value_index.emplace(value, new_index);
+ index_value.emplace(new_index, value);
+ return new_index;
+ }
+
+ void index_values(uint32_t doc_seq_id, const std::vector<std::string> & values) {
+ std::vector<uint32_t> value_vec(values.size());
+ for(auto i = 0; i < values.size(); i++) {
+ value_vec[i] = get_value_index(values[i]);
+ }
+ doc_values.emplace(doc_seq_id, value_vec);
+ }
+};
+
+class Collection {
+private:
+ std::string name;
+
+ uint32_t collection_id;
+
+ size_t num_documents;
+
+ // Auto incrementing record ID used internally for indexing - not exposed to the client
+ uint32_t next_seq_id;
+
+ spp::sparse_hash_map<std::string, field> search_schema;
+
+ spp::sparse_hash_map<std::string, field> facet_schema;
+
+ spp::sparse_hash_map<std::string, field> sort_schema;
+
+ Store* store;
+
+ spp::sparse_hash_map<std::string, art_tree*> search_index;
+
+ spp::sparse_hash_map<std::string, facet_value> facet_index;
+
+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;
+
+ std::string token_ranking_field;
+
+ std::string get_doc_id_key(const std::string & doc_id);
+
+ std::string get_seq_id_key(uint32_t seq_id);
+
+ static inline std::vector<art_leaf *> next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
+ long long int n);
+
+ void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
+
+ size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);
+
+ Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str);
+
+ void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);
+
+ void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
+ spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
+ size_t result_index, std::vector<std::vector<uint16_t>> &token_positions) const;
+
+ void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
+ std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,
+ const int num_typos, const size_t num_results,
+ std::vector<std::vector<art_leaf*>> & searched_queries, int & searched_queries_index,
+ Topster<100> & topster, uint32_t** all_result_ids,
+ size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY, const bool prefix = false);
+
+ void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
+ const std::vector<sort_by> & sort_fields, int & candidate_rank,
+ std::vector<std::vector<art_leaf*>> & token_to_candidates,
+ std::vector<std::vector<art_leaf*>> & searched_queries, Topster<100> & topster,
+ size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ const size_t & max_results, const bool prefix);
+
+ void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,
+ const bool verbatim) const;
+
+ void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
+ uint32_t seq_id, const bool verbatim) const;
+
+ void index_int32_field(const int32_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_int64_field(const int64_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_float_field(const float value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
+ const uint32_t indices_length);
+ Option<uint32_t> validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id);
+
+public:
+ Collection() = delete;
+
+ Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
+ const std::vector<field> & fields, const std::string & token_ranking_field);
+
+ ~Collection();
+
+ static std::string get_next_seq_id_key(const std::string & collection_name);
+
+ static std::string get_meta_key(const std::string & collection_name);
+
+ std::string get_seq_id_collection_prefix();
+
+ std::string get_name();
+
+ size_t get_num_documents();
+
+ uint32_t get_collection_id();
+
+ uint32_t get_next_seq_id();
+
+ void set_next_seq_id(uint32_t seq_id);
+
+ void increment_next_seq_id_field();
+
+ static uint32_t deserialize_seq_id_key(std::string serialized_seq_id);
+
+ uint32_t doc_id_to_seq_id(std::string doc_id);
+
+ std::vector<std::string> get_facet_fields();
+
+ std::vector<field> get_sort_fields();
+
+ spp::sparse_hash_map<std::string, field> get_schema();
+
+ std::string get_token_ranking_field();
+
+ Option<std::string> add(const std::string & json_str);
+
+ Option<nlohmann::json> search(std::string query, const std::vector<std::string> search_fields,
+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
+ const std::vector<sort_by> & sort_fields, const int num_typos,
+ const size_t per_page = 10, const size_t page = 1,
+ const token_ordering token_order = FREQUENCY, const bool prefix = false);
+
+ Option<nlohmann::json> get(const std::string & id);
+
+ Option<std::string> remove(const std::string & id, const bool remove_from_store = true);
+
+ void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
+ Topster<100> &topster, const std::vector<art_leaf *> & query_suggestion, const uint32_t *result_ids,
+ const size_t result_size) const;
+
+ Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);
+
+ static const int MAX_SEARCH_TOKENS = 10;
+ static const int MAX_RESULTS = 100;
+
+ // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
+ enum {SNIPPET_STR_ABOVE_LEN = 30};
+
+ // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
+ static constexpr const char* COLLECTION_META_PREFIX = "$CM";
+ static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
+ static constexpr const char* SEQ_ID_PREFIX = "$SI";
+ static constexpr const char* DOC_ID_PREFIX = "$DI";
+};
+
diff --git a/src/collection.cpp b/src/collection.cpp
new file mode 100644
index 00000000..2c548e5b
--- /dev/null
+++ b/src/collection.cpp
@@ -0,0 +1,1590 @@
+#include "index.h"
+
+#include <numeric>
+#include <chrono>
+#include <array_utils.h>
+#include <match_score.h>
+#include <string_utils.h>
+#include <art.h>
+
+Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
+ const std::vector<field> &fields, const std::string & token_ranking_field):
+ name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
+ token_ranking_field(token_ranking_field) {
+
+ for(const field& field: fields) {
+ art_tree *t = new art_tree;
+ art_tree_init(t);
+ search_index.emplace(field.name, t);
+ search_schema.emplace(field.name, field);
+
+ if(field.is_facet()) {
+ facet_value fvalue;
+ facet_index.emplace(field.name, fvalue);
+ facet_schema.emplace(field.name, field);
+ }
+
+ if(field.is_single_integer() || field.is_single_float()) {
+ spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
+ sort_index.emplace(field.name, doc_to_score);
+ sort_schema.emplace(field.name, field);
+ }
+ }
+
+ num_documents = 0;
+}
+
+Collection::~Collection() {
+ for(auto & name_tree: search_index) {
+ art_tree_destroy(name_tree.second);
+ delete name_tree.second;
+ name_tree.second = nullptr;
+ }
+
+ search_index.clear();
+
+ for(auto & name_map: sort_index) {
+ delete name_map.second;
+ name_map.second = nullptr;
+ }
+
+ sort_index.clear();
+}
+
+uint32_t Collection::get_next_seq_id() {
+ store->increment(get_next_seq_id_key(name), 1);
+ return next_seq_id++;
+}
+
+void Collection::set_next_seq_id(uint32_t seq_id) {
+ next_seq_id = seq_id;
+}
+
+void Collection::increment_next_seq_id_field() {
+ next_seq_id++;
+}
+
+Option<std::string> Collection::add(const std::string & json_str) {
+ nlohmann::json document;
+ try {
+ document = nlohmann::json::parse(json_str);
+ } catch(...) {
+ return Option<std::string>(400, "Bad JSON.");
+ }
+
+ uint32_t seq_id = get_next_seq_id();
+ std::string seq_id_str = std::to_string(seq_id);
+
+ if(document.count("id") == 0) {
+ document["id"] = seq_id_str;
+ } else if(!document["id"].is_string()) {
+ return Option<std::string>(400, "Document's `id` field should be a string.");
+ }
+
+ std::string doc_id = document["id"];
+
+ const Option<uint32_t> & index_memory_op = index_in_memory(document, seq_id);
+
+ if(!index_memory_op.ok()) {
+ return Option<std::string>(index_memory_op.code(), index_memory_op.error());
+ }
+
+ store->insert(get_doc_id_key(document["id"]), seq_id_str);
+ store->insert(get_seq_id_key(seq_id), document.dump());
+
+ return Option<std::string>(doc_id);
+}
+
+Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &document, uint32_t seq_id) {
+ if(!token_ranking_field.empty() && document.count(token_ranking_field) == 0) {
+ return Option<>(400, "Field `" + token_ranking_field + "` has been declared as a token ranking field, "
+ "but is not found in the document.");
+ }
+
+ if(!token_ranking_field.empty() && !document[token_ranking_field].is_number_integer() &&
+ !document[token_ranking_field].is_number_float()) {
+ return Option<>(400, "Token ranking field `" + token_ranking_field + "` must be a number.");
+ }
+
+ if(!token_ranking_field.empty() && document[token_ranking_field].is_number_integer() &&
+ document[token_ranking_field].get<int64_t>() > std::numeric_limits<int32_t>::max()) {
+ return Option<>(400, "Token ranking field `" + token_ranking_field + "` exceeds maximum value of int32.");
+ }
+
+ if(!token_ranking_field.empty() && document[token_ranking_field].is_number_float() &&
+ document[token_ranking_field].get<float>() > std::numeric_limits<float>::max()) {
+ return Option<>(400, "Token ranking field `" + token_ranking_field + "` exceeds maximum value of a float.");
+ }
+
+ for(const std::pair<std::string, field> & field_pair: search_schema) {
+ const std::string & field_name = field_pair.first;
+
+ if(document.count(field_name) == 0) {
+ return Option<>(400, "Field `" + field_name + "` has been declared in the schema, "
+ "but is not found in the document.");
+ }
+
+ art_tree *t = search_index.at(field_name);
+
+ if(field_pair.second.type == field_types::STRING) {
+ if(!document[field_name].is_string()) {
+ return Option<>(400, "Field `" + field_name + "` must be a string.");
+ }
+ } else if(field_pair.second.type == field_types::INT32) {
+ if(!document[field_name].is_number_integer()) {
+ return Option<>(400, "Field `" + field_name + "` must be an int32.");
+ }
+
+ if(document[field_name].get<int64_t>() > INT32_MAX) {
+ return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
+ }
+ } else if(field_pair.second.type == field_types::INT64) {
+ if(!document[field_name].is_number_integer()) {
+ return Option<>(400, "Field `" + field_name + "` must be an int64.");
+ }
+ } else if(field_pair.second.type == field_types::FLOAT) {
+ if(!document[field_name].is_number()) { // allows integer to be passed to a float field
+ return Option<>(400, "Field `" + field_name + "` must be a float.");
+ }
+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Field `" + field_name + "` must be a string array.");
+ }
+ if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
+ return Option<>(400, "Field `" + field_name + "` must be a string array.");
+ }
+ } else if(field_pair.second.type == field_types::INT32_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
+ return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
+ }
+ } else if(field_pair.second.type == field_types::INT64_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
+ return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
+ }
+ } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Field `" + field_name + "` must be a float array.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_number_float()) {
+ return Option<>(400, "Field `" + field_name + "` must be a float array.");
+ }
+ }
+ }
+
+ for(const std::pair<std::string, field> & field_pair: facet_schema) {
+ const std::string & field_name = field_pair.first;
+
+ if(document.count(field_name) == 0) {
+ return Option<>(400, "Field `" + field_name + "` has been declared as a facet field in the schema, "
+ "but is not found in the document.");
+ }
+
+ facet_value & fvalue = facet_index.at(field_name);
+ if(field_pair.second.type == field_types::STRING) {
+ if(!document[field_name].is_string()) {
+ return Option<>(400, "Facet field `" + field_name + "` must be a string.");
+ }
+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Facet field `" + field_name + "` must be a string array.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
+ return Option<>(400, "Facet field `" + field_name + "` must be a string array.");
+ }
+ } else {
+ return Option<>(400, "Facet field `" + field_name + "` must be a string or a string[].");
+ }
+ }
+
+ return Option<>(200);
+}
+
+Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id) {
+ Option<uint32_t> validation_op = validate_index_in_memory(document, seq_id);
+
+ if(!validation_op.ok()) {
+ return validation_op;
+ }
+
+ int32_t points = 0;
+
+ if(!token_ranking_field.empty()) {
+ if(document[token_ranking_field].is_number_float()) {
+ // serialize float to an integer and reverse the inverted range
+ float n = document[token_ranking_field];
+ memcpy(&points, &n, sizeof(int32_t));
+ points ^= ((points >> (std::numeric_limits<int32_t>::digits - 1)) | INT32_MIN);
+ points = -1 * (INT32_MAX - points);
+ } else {
+ points = document[token_ranking_field];
+ }
+ }
+
+ for(const std::pair<std::string, field> & field_pair: search_schema) {
+ const std::string & field_name = field_pair.first;
+ art_tree *t = search_index.at(field_name);
+
+ if(field_pair.second.type == field_types::STRING) {
+ const std::string & text = document[field_name];
+ index_string_field(text, points, t, seq_id, field_pair.second.is_facet());
+ } else if(field_pair.second.type == field_types::INT32) {
+ uint32_t value = document[field_name];
+ index_int32_field(value, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::INT64) {
+ uint64_t value = document[field_name];
+ index_int64_field(value, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::FLOAT) {
+ float value = document[field_name];
+ index_float_field(value, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ std::vector<std::string> strings = document[field_name];
+ index_string_array_field(strings, points, t, seq_id, field_pair.second.is_facet());
+ } else if(field_pair.second.type == field_types::INT32_ARRAY) {
+ std::vector<int32_t> values = document[field_name];
+ index_int32_array_field(values, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::INT64_ARRAY) {
+ std::vector<int64_t> values = document[field_name];
+ index_int64_array_field(values, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
+ std::vector<float> values = document[field_name];
+ index_float_array_field(values, points, t, seq_id);
+ }
+
+ // add numerical values automatically into sort index
+ if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||
+ field_pair.second.type == field_types::FLOAT) {
+ spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);
+
+ if(document[field_pair.first].is_number_integer()) {
+ doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());
+ } else if(document[field_pair.first].is_number_float()) {
+ doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());
+ }
+ }
+ }
+
+ for(const std::pair<std::string, field> & field_pair: facet_schema) {
+ const std::string & field_name = field_pair.first;
+ facet_value & fvalue = facet_index.at(field_name);
+ if(field_pair.second.type == field_types::STRING) {
+ const std::string & value = document[field_name];
+ fvalue.index_values(seq_id, { value });
+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ const std::vector<std::string> & values = document[field_name];
+ fvalue.index_values(seq_id, values);
+ }
+ }
+
+ num_documents += 1;
+ return Option<>(200);
+}
+
+void Collection::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+
+ encode_int32(value, key);
+
+ uint32_t num_hits = 0;
+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
+ }
+
+ num_hits += 1;
+
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = score;
+ art_doc.offsets_len = 0;
+ art_doc.offsets = nullptr;
+
+ art_insert(t, key, KEY_LEN, &art_doc, num_hits);
+}
+
+void Collection::index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+
+ encode_int64(value, key);
+
+ uint32_t num_hits = 0;
+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
+ }
+
+ num_hits += 1;
+
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = score;
+ art_doc.offsets_len = 0;
+ art_doc.offsets = nullptr;
+
+ art_insert(t, key, KEY_LEN, &art_doc, num_hits);
+}
+
+void Collection::index_float_field(const float value, uint32_t score, art_tree *t, uint32_t seq_id) const {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+
+ encode_float(value, key);
+
+ uint32_t num_hits = 0;
+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
+ }
+
+ num_hits += 1;
+
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = score;
+ art_doc.offsets_len = 0;
+ art_doc.offsets = nullptr;
+
+ art_insert(t, key, KEY_LEN, &art_doc, num_hits);
+}
+
+
+void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t,
+ uint32_t seq_id, const bool verbatim) const {
+ std::vector<std::string> tokens;
+ std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
+
+ if(verbatim) {
+ tokens.push_back(text);
+ token_to_offsets[text].push_back(0);
+ } else {
+ StringUtils::split(text, tokens, " ");
+ for(uint32_t i=0; i<tokens.size(); i++) {
+ auto & token = tokens[i];
+ transform(token.begin(), token.end(), token.begin(), tolower);
+ token_to_offsets[token].push_back(i);
+ }
+ }
+
+ for(auto & kv: token_to_offsets) {
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = score;
+ art_doc.offsets_len = (uint32_t) kv.second.size();
+ art_doc.offsets = new uint32_t[kv.second.size()];
+
+ uint32_t num_hits = 0;
+
+ const unsigned char *key = (const unsigned char *) kv.first.c_str();
+ int key_len = (int) kv.first.length() + 1; // for the terminating \0 char
+
+ art_leaf* leaf = (art_leaf *) art_search(t, key, key_len);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
+ }
+
+ num_hits += 1;
+
+ for(auto i=0; i<kv.second.size(); i++) {
+ art_doc.offsets[i] = kv.second[i];
+ }
+
+ art_insert(t, key, key_len, &art_doc, num_hits);
+ delete [] art_doc.offsets;
+ art_doc.offsets = nullptr;
+ }
+}
+
+void Collection::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
+ uint32_t seq_id, const bool verbatim) const {
+ for(const std::string & str: strings) {
+ index_string_field(str, score, t, seq_id, verbatim);
+ }
+}
+
+void Collection::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,
+ uint32_t seq_id) const {
+ for(const int32_t value: values) {
+ index_int32_field(value, score, t, seq_id);
+ }
+}
+
+void Collection::index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t,
+ uint32_t seq_id) const {
+ for(const int64_t value: values) {
+ index_int64_field(value, score, t, seq_id);
+ }
+}
+
+void Collection::index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t,
+ uint32_t seq_id) const {
+ for(const float value: values) {
+ index_float_field(value, score, t, seq_id);
+ }
+}
+
+void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {
+ for(auto & a_facet: facets) {
+ // assumed that facet fields have already been validated upstream
+ const field & facet_field = facet_schema.at(a_facet.field_name);
+ const facet_value & fvalue = facet_index.at(facet_field.name);
+
+ for(auto i = 0; i < results_size; i++) {
+ uint32_t doc_seq_id = result_ids[i];
+ if(fvalue.doc_values.count(doc_seq_id) != 0) {
+ // for every result document, get the values associated and increment counter
+ const std::vector<uint32_t> & value_indices = fvalue.doc_values.at(doc_seq_id);
+ for(auto j = 0; j < value_indices.size(); j++) {
+ const std::string & facet_value = fvalue.index_value.at(value_indices.at(j));
+ a_facet.result_map[facet_value] += 1;
+ }
+ }
+ }
+ }
+}
+
+void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
+ const std::vector<sort_by> & sort_fields, int & candidate_rank,
+ std::vector<std::vector<art_leaf*>> & token_to_candidates,
+ std::vector<std::vector<art_leaf*>> & searched_queries, Topster<100> & topster,
+ size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ const size_t & max_results, const bool prefix) {
+ const size_t combination_limit = 10;
+ const size_t prefix_candidates_limit = 5;
+
+ auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
+ long long int N = std::accumulate(token_to_candidates.begin(), token_to_candidates.end(), 1LL, product);
+
+ for(long long n=0; n<N && n<combination_limit; ++n) {
+ // every element in `query_suggestion` contains a token and its associated hits
+ std::vector<art_leaf *> query_suggestion = next_suggestion(token_to_candidates, n);
+
+ /*for(auto i=0; i < query_suggestion.size(); i++) {
+ std::cout << "i: " << i << " - " << query_suggestion[i]->key << std::endl;
+ }*/
+
+ // initialize results with the starting element (for further intersection)
+ size_t result_size = query_suggestion[0]->values->ids.getLength();
+ if(result_size == 0) {
+ continue;
+ }
+
+ uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
+ candidate_rank += 1;
+
+ // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
+ for(auto i=1; i < query_suggestion.size(); i++) {
+ uint32_t* out = nullptr;
+ uint32_t* ids = query_suggestion[i]->values->ids.uncompress();
+ result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);
+ delete[] ids;
+ delete[] result_ids;
+ result_ids = out;
+ }
+
+ if(filter_ids != nullptr) {
+ // intersect once again with filter ids
+ uint32_t* filtered_result_ids = nullptr;
+ size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
+ result_size, &filtered_result_ids);
+
+ uint32_t* new_all_result_ids;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
+ filtered_results_size, &new_all_result_ids);
+ delete [] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+
+ do_facets(facets, filtered_result_ids, filtered_results_size);
+
+ // go through each matching document id and calculate match score
+ score_results(sort_fields, searched_queries.size(), candidate_rank, topster, query_suggestion,
+ filtered_result_ids, filtered_results_size);
+
+ delete[] filtered_result_ids;
+ delete[] result_ids;
+ } else {
+ do_facets(facets, result_ids, result_size);
+
+ uint32_t* new_all_result_ids;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
+ result_size, &new_all_result_ids);
+ delete [] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+
+ score_results(sort_fields, searched_queries.size(), candidate_rank, topster, query_suggestion,
+ result_ids, result_size);
+ delete[] result_ids;
+ }
+
+ total_results += topster.size;
+ searched_queries.push_back(query_suggestion);
+
+ if(!prefix && total_results >= max_results) {
+ break;
+ }
+
+ if(prefix && candidate_rank >= prefix_candidates_limit) {
+ break;
+ }
+ }
+}
+
+size_t Collection::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,
+ uint32_t **results_out) {
+ uint32_t *results = nullptr;
+ size_t results_length = 0;
+
+ uint32_t *prev_results = nullptr;
+ size_t prev_results_length = 0;
+
+ for(const std::pair<uint32_t*, size_t> & result_array_pair: result_array_pairs) {
+ results_length = ArrayUtils::or_scalar(prev_results, prev_results_length, result_array_pair.first,
+ result_array_pair.second, &results);
+ delete [] prev_results;
+ prev_results = results;
+ prev_results_length = results_length;
+ }
+
+ *results_out = results;
+ return results_length;
+}
+
+Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str) {
+ // parse the filter string
+ std::vector<std::string> filter_blocks;
+ StringUtils::split(simple_filter_str, filter_blocks, "&&");
+
+ std::vector<filter> filters;
+
+ for(const std::string & filter_block: filter_blocks) {
+ // split into [field_name, value]
+ std::vector<std::string> expression_parts;
+ StringUtils::split(filter_block, expression_parts, ":");
+ if(expression_parts.size() != 2) {
+ return Option<>(400, "Could not parse the filter query.");
+ }
+
+ const std::string & field_name = expression_parts[0];
+ if(search_schema.count(field_name) == 0) {
+ return Option<>(400, "Could not find a filter field named `" + field_name + "` in the schema.");
+ }
+
+ field _field = search_schema.at(field_name);
+ std::string & raw_value = expression_parts[1];
+ filter f;
+
+ if(_field.is_integer() || _field.is_float()) {
+ // could be a single value or a list
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+
+ for(const std::string & filter_value: filter_values) {
+ if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
+ return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");
+ }
+
+ if(_field.is_float() && !StringUtils::is_float(filter_value)) {
+ return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");
+ }
+ }
+
+ f = {field_name, filter_values, EQUALS};
+ } else {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
+ if(!op_comparator.ok()) {
+ return Option<>(400, "Error with field `" + _field.name + "`: " + op_comparator.error());
+ }
+
+ // extract numerical value
+ std::string filter_value;
+ if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {
+ filter_value = raw_value.substr(1);
+ } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {
+ filter_value = raw_value.substr(2);
+ } else {
+ // EQUALS
+ filter_value = raw_value;
+ }
+
+ filter_value = StringUtils::trim(filter_value);
+
+ if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
+ return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");
+ }
+
+ if(_field.is_float() && !StringUtils::is_float(filter_value)) {
+ return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");
+ }
+
+ f = {field_name, {filter_value}, op_comparator.get()};
+ }
+ } else if(_field.is_string()) {
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ f = {field_name, filter_values, EQUALS};
+ } else {
+ f = {field_name, {raw_value}, EQUALS};
+ }
+ } else {
+ return Option<>(400, "Error with field `" + _field.name + "`: Unidentified field type.");
+ }
+
+ filters.push_back(f);
+ }
+
+ uint32_t* filter_ids = nullptr;
+ uint32_t filter_ids_length = 0;
+
+ // process the filters first
+ for(const filter & a_filter: filters) {
+ if(search_index.count(a_filter.field_name) != 0) {
+ art_tree* t = search_index.at(a_filter.field_name);
+ field f = search_schema.at(a_filter.field_name);
+ std::vector<std::pair<uint32_t*, size_t>> filter_result_array_pairs;
+
+ if(f.is_integer()) {
+ std::vector<const art_leaf*> leaves;
+
+ for(const std::string & filter_value: a_filter.values) {
+ if(f.type == field_types::INT32 || f.type == field_types::INT32_ARRAY) {
+ int32_t value = (int32_t) std::stoi(filter_value);
+ art_int32_search(t, value, a_filter.compare_operator, leaves);
+ } else {
+ int64_t value = (int64_t) std::stoi(filter_value);
+ art_int64_search(t, value, a_filter.compare_operator, leaves);
+ }
+
+ for(const art_leaf* leaf: leaves) {
+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),
+ leaf->values->ids.getLength()));
+ }
+ }
+ } else if(f.is_float()) {
+ std::vector<const art_leaf*> leaves;
+
+ for(const std::string & filter_value: a_filter.values) {
+ float value = (float) std::atof(filter_value.c_str());
+ art_float_search(t, value, a_filter.compare_operator, leaves);
+ for(const art_leaf* leaf: leaves) {
+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),
+ leaf->values->ids.getLength()));
+ }
+ }
+ } else if(f.is_string()) {
+ for(const std::string & filter_value: a_filter.values) {
+ // we have to tokenize the string, standardize it and then do an exact match
+ std::vector<std::string> str_tokens;
+ StringUtils::split(filter_value, str_tokens, " ");
+
+ uint32_t* filtered_ids = nullptr;
+ size_t filtered_size = 0;
+
+ for(auto i = 0; i < str_tokens.size(); i++) {
+ std::string & str_token = str_tokens[i];
+ StringUtils::normalize(str_token);
+ art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
+ str_token.length()+1);
+ if(leaf == nullptr) {
+ continue;
+ }
+
+ if(i == 0) {
+ filtered_ids = leaf->values->ids.uncompress();
+ filtered_size = leaf->values->ids.getLength();
+ } else {
+ // do AND for an exact match
+ uint32_t* out = nullptr;
+ uint32_t* leaf_ids = leaf->values->ids.uncompress();
+ filtered_size = ArrayUtils::and_scalar(filtered_ids, filtered_size, leaf_ids,
+ leaf->values->ids.getLength(), &out);
+ delete[] leaf_ids;
+ delete[] filtered_ids;
+ filtered_ids = out;
+ }
+ }
+
+ filter_result_array_pairs.push_back(std::make_pair(filtered_ids, filtered_size));
+ }
+ }
+
+ uint32_t* result_ids = nullptr;
+ size_t result_ids_length = union_of_ids(filter_result_array_pairs, &result_ids);
+
+ if(filter_ids == nullptr) {
+ filter_ids = result_ids;
+ filter_ids_length = result_ids_length;
+ } else {
+ uint32_t* filtered_results = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
+ result_ids_length, &filtered_results);
+ delete [] result_ids;
+ delete [] filter_ids;
+ filter_ids = filtered_results;
+ }
+
+ for(std::pair<uint32_t*, size_t> & filter_result_array_pair: filter_result_array_pairs) {
+ delete[] filter_result_array_pair.first;
+ }
+ }
+ }
+
+ *filter_ids_out = filter_ids;
+ return Option<>(filter_ids_length);
+}
+
+Option<nlohmann::json> Collection::search(std::string query, const std::vector<std::string> search_fields,
+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
+ const std::vector<sort_by> & sort_fields, const int num_typos,
+ const size_t per_page, const size_t page,
+ const token_ordering token_order, const bool prefix) {
+ nlohmann::json result = nlohmann::json::object();
+ std::vector<facet> facets;
+
+ // validate search fields
+ for(const std::string & field_name: search_fields) {
+ if(search_schema.count(field_name) == 0) {
+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";
+ return Option<nlohmann::json>(400, error);
+ }
+
+ field search_field = search_schema.at(field_name);
+ if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {
+ std::string error = "Field `" + field_name + "` should be a string or a string array.";
+ return Option<nlohmann::json>(400, error);
+ }
+
+ if(search_field.facet) {
+ std::string error = "Field `" + field_name + "` is a faceted field - it cannot be used as a query field.";
+ return Option<nlohmann::json>(400, error);
+ }
+ }
+
+ // validate facet fields
+ for(const std::string & field_name: facet_fields) {
+ if(facet_schema.count(field_name) == 0) {
+ std::string error = "Could not find a facet field named `" + field_name + "` in the schema.";
+ return Option<nlohmann::json>(400, error);
+ }
+ facets.push_back(facet(field_name));
+ }
+
+ // validate sort fields and standardize
+
+ std::vector<sort_by> sort_fields_std;
+
+ for(const sort_by & _sort_field: sort_fields) {
+ if(sort_index.count(_sort_field.name) == 0) {
+ std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema for sorting.";
+ return Option<nlohmann::json>(400, error);
+ }
+
+ std::string sort_order = _sort_field.order;
+ StringUtils::toupper(sort_order);
+
+ if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {
+ std::string error = "Order for field` " + _sort_field.name + "` should be either ASC or DESC.";
+ return Option<nlohmann::json>(400, error);
+ }
+
+ sort_fields_std.push_back({_sort_field.name, sort_order});
+ }
+
+ // check for valid pagination
+ if(page < 1) {
+ std::string message = "Page must be an integer of value greater than 0.";
+ return Option<nlohmann::json>(422, message);
+ }
+
+ if((page * per_page) > MAX_RESULTS) {
+ std::string message = "Only the first " + std::to_string(MAX_RESULTS) + " results are available.";
+ return Option<nlohmann::json>(422, message);
+ }
+
+ const size_t num_results = (page * per_page);
+
+ // process the filters
+ uint32_t* filter_ids = nullptr;
+ Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, simple_filter_query);
+ if(!op_filter_ids_length.ok()) {
+ return Option<nlohmann::json>(op_filter_ids_length.code(), op_filter_ids_length.error());
+ }
+
+ const uint32_t filter_ids_length = op_filter_ids_length.get();
+
+ // Order of `fields` are used to sort results
+ auto begin = std::chrono::high_resolution_clock::now();
+ std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
+ uint32_t* all_result_ids = nullptr;
+ size_t all_result_ids_len = 0;
+
+ // all search queries that were used for generating the results
+ std::vector<std::vector<art_leaf*>> searched_queries;
+ int searched_queries_index = 0;
+
+ for(int i = 0; i < search_fields.size(); i++) {
+ Topster<100> topster;
+ const std::string & field = search_fields[i];
+ // proceed to query search only when no filters are provided or when filtering produces results
+ if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
+ search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,
+ searched_queries, searched_queries_index, topster, &all_result_ids, all_result_ids_len, token_order, prefix);
+ topster.sort();
+ }
+
+ // order of fields specified matter: matching docs from earlier fields are more important
+ for(auto t = 0; t < topster.size && t < num_results; t++) {
+ field_order_kvs.push_back(std::make_pair(search_fields.size() - i, topster.getKV(t)));
+ }
+ }
+
+ delete [] filter_ids;
+ delete [] all_result_ids;
+
+ // All fields are sorted descending
+ std::sort(field_order_kvs.begin(), field_order_kvs.end(),
+ [](const std::pair<int, Topster<100>::KV> & a, const std::pair<int, Topster<100>::KV> & b) {
+ return std::tie(a.second.match_score, a.second.primary_attr, a.second.secondary_attr, a.first, a.second.key) >
+ std::tie(b.second.match_score, b.second.primary_attr, b.second.secondary_attr, b.first, b.second.key);
+ });
+
+ result["hits"] = nlohmann::json::array();
+ result["found"] = all_result_ids_len;
+
+ const int start_result_index = (page - 1) * per_page;
+ const int kvsize = field_order_kvs.size();
+
+ if(start_result_index > (kvsize - 1)) {
+ return Option<nlohmann::json>(result);
+ }
+
+ const int end_result_index = std::min(int(page * per_page), kvsize) - 1;
+
+ for(size_t field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {
+ const auto & field_order_kv = field_order_kvs[field_order_kv_index];
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);
+
+ std::string value;
+ store->get(seq_id_key, value);
+
+ nlohmann::json document;
+ try {
+ document = nlohmann::json::parse(value);
+ } catch(...) {
+ return Option<nlohmann::json>(500, "Error while parsing stored document.");
+ }
+
+ // highlight query words in the result
+ const std::string & field_name = search_fields[search_fields.size() - field_order_kv.first];
+ field search_field = search_schema.at(field_name);
+
+ // only string fields are supported for now
+ if(search_field.type == field_types::STRING) {
+ std::vector<std::string> tokens;
+ StringUtils::split(document[field_name], tokens, " ");
+
+ // positions in the document of each token in the query
+ std::vector<std::vector<uint16_t>> token_positions;
+
+ for (const art_leaf *token_leaf : searched_queries[field_order_kv.second.query_index]) {
+ std::vector<uint16_t> positions;
+ int doc_index = token_leaf->values->ids.indexOf(field_order_kv.second.key);
+ if(doc_index == token_leaf->values->ids.getLength()) {
+ continue;
+ }
+
+ uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
+ uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
+ token_leaf->values->offsets.getLength() :
+ token_leaf->values->offset_index.at(doc_index+1);
+
+ while(start_offset < end_offset) {
+ positions.push_back((uint16_t) token_leaf->values->offsets.at(start_offset));
+ start_offset++;
+ }
+
+ token_positions.push_back(positions);
+ }
+
+ MatchScore mscore = MatchScore::match_score(field_order_kv.second.key, token_positions);
+
+ // unpack `mscore.offset_diffs` into `token_indices`
+ std::vector<size_t> token_indices;
+ char num_tokens_found = mscore.offset_diffs[0];
+ for(size_t i = 1; i <= num_tokens_found; i++) {
+ if(mscore.offset_diffs[i] != std::numeric_limits<int8_t>::max()) {
+ size_t token_index = (size_t)(mscore.start_offset + mscore.offset_diffs[i]);
+ token_indices.push_back(token_index);
+ }
+ }
+
+ auto minmax = std::minmax_element(token_indices.begin(), token_indices.end());
+
+ // For longer strings, pick surrounding tokens within N tokens of min_index and max_index for the snippet
+ const size_t start_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? 0 :
+ std::max(0, (int)(*(minmax.first)-5));
+
+ const size_t end_index = (tokens.size() <= SNIPPET_STR_ABOVE_LEN) ? tokens.size() :
+ std::min((int)tokens.size(), (int)(*(minmax.second)+5));
+
+ for(const size_t token_index: token_indices) {
+ tokens[token_index] = "<mark>" + tokens[token_index] + "</mark>";
+ }
+
+ std::stringstream snippet_stream;
+ for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {
+ if(snippet_index != start_index) {
+ snippet_stream << " ";
+ }
+
+ snippet_stream << tokens[snippet_index];
+ }
+
+ document["_highlight"] = nlohmann::json::object();
+ document["_highlight"][field_name] = snippet_stream.str();
+ }
+
+ result["hits"].push_back(document);
+ }
+
+ result["facet_counts"] = nlohmann::json::array();
+
+ // populate facets
+ for(const facet & a_facet: facets) {
+ nlohmann::json facet_result = nlohmann::json::object();
+ facet_result["field_name"] = a_facet.field_name;
+ facet_result["counts"] = nlohmann::json::array();
+
+ // keep only top 10 facets
+ std::vector<std::pair<std::string, size_t>> value_to_count;
+ for (auto itr = a_facet.result_map.begin(); itr != a_facet.result_map.end(); ++itr) {
+ value_to_count.push_back(*itr);
+ }
+
+ std::sort(value_to_count.begin(), value_to_count.end(),
+ [=](std::pair<std::string, size_t>& a, std::pair<std::string, size_t>& b) {
+ return a.second > b.second;
+ });
+
+ for(auto i = 0; i < std::min((size_t)10, value_to_count.size()); i++) {
+ auto & kv = value_to_count[i];
+ nlohmann::json facet_value_count = nlohmann::json::object();
+ facet_value_count["value"] = kv.first;
+ facet_value_count["count"] = kv.second;
+ facet_result["counts"].push_back(facet_value_count);
+ }
+
+ result["facet_counts"].push_back(facet_result);
+ }
+
+ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ //!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
+ //!store->print_memory_usage();
+ return result;
+}
+
+/*
+ 1. Split the query into tokens
+ 2. Outer loop will generate bounded cartesian product with costs for each token
+ 3. Inner loop will iterate on each token with associated cost
+ 4. Cartesian product of the results of the token searches will be used to form search phrases
+ (cartesian product adapted from: http://stackoverflow.com/a/31169617/131050)
+ 4. Intersect the lists to find docs that match each phrase
+ 5. Sort the docs based on some ranking criteria
+*/
+void Collection::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
+ std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,
+ const size_t num_results, std::vector<std::vector<art_leaf*>> & searched_queries,
+ int & searched_queries_index, Topster<100> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,
+ const token_ordering token_order, const bool prefix) {
+ std::vector<std::string> tokens;
+ StringUtils::split(query, tokens, " ");
+
+ const int max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
+ const size_t max_results = std::min(num_results, (size_t) Collection::MAX_RESULTS);
+
+ size_t total_results = topster.size;
+
+ // To prevent us from doing ART search repeatedly as we iterate through possible corrections
+ spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
+
+ // Used to drop the least occurring token(s) for partial searches
+ spp::sparse_hash_map<std::string, uint32_t> token_to_count;
+
+ std::vector<std::vector<int>> token_to_costs;
+
+ for(size_t token_index = 0; token_index < tokens.size(); token_index++) {
+ std::vector<int> all_costs;
+ const size_t token_len = tokens[token_index].length();
+
+ // This ensures that we don't end up doing a cost of 1 for a single char etc.
+ int bounded_cost = max_cost;
+ if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {
+ bounded_cost = token_len - 1;
+ }
+
+ for(int cost = 0; cost <= bounded_cost; cost++) {
+ all_costs.push_back(cost);
+ }
+
+ token_to_costs.push_back(all_costs);
+ StringUtils::normalize(tokens[token_index]);
+ }
+
+ // stores candidates for each token, i.e. i-th index would have all possible tokens with a cost of "c"
+ std::vector<std::vector<art_leaf*>> token_to_candidates;
+
+ const size_t combination_limit = 10;
+ auto product = []( long long a, std::vector<int>& b ) { return a*b.size(); };
+ int candidate_rank = 0;
+ long long n = 0;
+ long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
+
+ while(n < N && n < combination_limit) {
+ // Outerloop generates combinations of [cost to max_cost] for each token
+ // For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.
+ std::vector<uint32_t> costs(token_to_costs.size());
+ ldiv_t q { n, 0 };
+ for(long long i = (token_to_costs.size() - 1); 0 <= i ; --i ) {
+ q = ldiv(q.quot, token_to_costs[i].size());
+ costs[i] = token_to_costs[i][q.rem];
+ }
+
+ token_to_candidates.clear();
+ int token_index = 0;
+
+ while(token_index < tokens.size()) {
+ // For each token, look up the generated cost for this iteration and search using that cost
+ std::string token = tokens[token_index];
+ const std::string token_cost_hash = token + std::to_string(costs[token_index]);
+
+ std::vector<art_leaf*> leaves;
+ /*std::cout << "\nSearching for: " << token << " - cost: " << costs[token_index] << ", candidate_rank: "
+ << candidate_rank << std::endl;*/
+
+ if(token_cost_cache.count(token_cost_hash) != 0) {
+ leaves = token_cost_cache[token_cost_hash];
+ } else {
+ // prefix should apply only for last token
+ const bool prefix_search = prefix && ((token_index == tokens.size()-1) ? true : false);
+ const int token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
+ const int max_candidates = prefix_search ? 5 : 3;
+
+ art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
+ costs[token_index], costs[token_index], max_candidates, token_order, prefix_search, leaves);
+
+ if(!leaves.empty()) {
+ token_cost_cache.emplace(token_cost_hash, leaves);
+ }
+ }
+
+ if(!leaves.empty()) {
+ //!log_leaves(costs[token_index], token, leaves);
+ token_to_candidates.push_back(leaves);
+ token_to_count[token] = std::max(token_to_count[token], leaves.at(0)->values->ids.getLength());
+ } else {
+ // No result at `cost = costs[token_index]`. Remove costs until `cost` for token and re-do combinations
+ auto it = std::find(token_to_costs[token_index].begin(), token_to_costs[token_index].end(), costs[token_index]);
+ if(it != token_to_costs[token_index].end()) {
+ token_to_costs[token_index].erase(it);
+
+ // no more costs left for this token, clean up
+ if(token_to_costs[token_index].empty()) {
+ token_to_costs.erase(token_to_costs.begin()+token_index);
+ tokens.erase(tokens.begin()+token_index);
+ token_index--;
+ }
+ }
+
+ // To continue outerloop on new cost combination
+ n = -1;
+ N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
+ break;
+ }
+
+ token_index++;
+ }
+
+ if(token_to_candidates.size() != 0 && token_to_candidates.size() == tokens.size()) {
+ // If all tokens were found, go ahead and search for candidates with what we have so far
+ search_candidates(filter_ids, filter_ids_length, facets, sort_fields, candidate_rank, token_to_candidates,
+ searched_queries, topster, total_results, all_result_ids, all_result_ids_len,
+ max_results, prefix);
+
+ if (!prefix && total_results >= max_results) {
+ // If we don't find enough results, we continue outerloop (looking at tokens with greater cost)
+ break;
+ }
+
+ // only allow upto 10 prefix candidate tokens
+ if(prefix && candidate_rank > 10) {
+ break;
+ }
+ }
+
+ n++;
+ }
+
+ // When there are not enough overall results and atleast one token has results
+ if(topster.size < max_results && token_to_count.size() > 1) {
+ // Drop token with least hits and try searching again
+ std::string truncated_query;
+
+ std::vector<std::pair<std::string, uint32_t>> token_count_pairs;
+ for (auto itr = token_to_count.begin(); itr != token_to_count.end(); ++itr) {
+ token_count_pairs.push_back(*itr);
+ }
+
+ std::sort(token_count_pairs.begin(), token_count_pairs.end(), [=]
+ (const std::pair<std::string, uint32_t>& a, const std::pair<std::string, uint32_t>& b) {
+ return a.second > b.second;
+ }
+ );
+
+ for(uint32_t i = 0; i < token_count_pairs.size()-1; i++) {
+ // iterate till last but one
+ truncated_query += " " + token_count_pairs.at(i).first;
+ }
+
+ return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos,
+ num_results, searched_queries, candidate_rank, topster, all_result_ids, all_result_ids_len,
+ token_order, prefix);
+ }
+}
+
+void Collection::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {
+ printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);
+ for(auto i=0; i < leaves.size(); i++) {
+ printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);
+ printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score);
+ /*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {
+ printf("id: %d\n", leaves[i]->values->ids.at(j));
+ }*/
+ }
+}
+
+void Collection::score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
+ Topster<100> & topster, const std::vector<art_leaf *> &query_suggestion,
+ const uint32_t *result_ids, const size_t result_size) const {
+
+ const int max_candidate_rank = 250;
+ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
+
+ for (art_leaf *token_leaf : query_suggestion) {
+ uint32_t *indices = new uint32_t[result_size];
+ token_leaf->values->ids.indexOf(result_ids, result_size, indices);
+ leaf_to_indices.emplace(token_leaf, indices);
+ }
+
+ spp::sparse_hash_map<uint32_t, number_t> * primary_rank_scores = nullptr;
+ spp::sparse_hash_map<uint32_t, number_t> * secondary_rank_scores = nullptr;
+
+ // Used for asc/desc ordering. NOTE: Topster keeps biggest keys (i.e. it's desc in nature)
+ number_t primary_rank_factor;
+ number_t secondary_rank_factor;
+
+ if(sort_fields.size() > 0) {
+ // assumed that rank field exists in the index - checked earlier in the chain
+ primary_rank_scores = sort_index.at(sort_fields[0].name);
+
+ // initialize primary_rank_factor
+ field sort_field = sort_schema.at(sort_fields[0].name);
+ if(sort_field.is_single_integer()) {
+ primary_rank_factor = ((int64_t) 1);
+ } else {
+ primary_rank_factor = ((float) 1);
+ }
+
+ if(sort_fields[0].order == sort_field_const::asc) {
+ primary_rank_factor = -primary_rank_factor;
+ }
+ }
+
+ if(sort_fields.size() > 1) {
+ secondary_rank_scores = sort_index.at(sort_fields[1].name);
+
+ // initialize secondary_rank_factor
+ field sort_field = sort_schema.at(sort_fields[1].name);
+ if(sort_field.is_single_integer()) {
+ secondary_rank_factor = ((int64_t) 1);
+ } else {
+ secondary_rank_factor = ((float) 1);
+ }
+
+ if(sort_fields[1].order == sort_field_const::asc) {
+ secondary_rank_factor = -secondary_rank_factor;
+ }
+ }
+
+ for(auto i=0; i<result_size; i++) {
+ uint32_t seq_id = result_ids[i];
+ MatchScore mscore;
+
+ if(query_suggestion.size() == 1) {
+ // short circuit to speed up single token searches (use dummy offsets for now)
+ char offset_diffs[16];
+ std::fill_n(offset_diffs, 16, 0);
+ mscore = MatchScore(1, 0, 0, offset_diffs);
+ } else {
+ std::vector<std::vector<uint16_t>> token_positions;
+ populate_token_positions(query_suggestion, leaf_to_indices, i, token_positions);
+ mscore = MatchScore::match_score(seq_id, token_positions);
+ }
+
+ int candidate_rank_score = max_candidate_rank - candidate_rank;
+
+ // Construct a single match_score from individual components (for multi-field sort)
+ const uint64_t match_score = ((uint64_t)(mscore.words_present) << 16) +
+ (candidate_rank_score << 8) +
+ (MAX_SEARCH_TOKENS - mscore.distance);
+
+ const int64_t default_score = 0;
+ const number_t & primary_rank_score = (primary_rank_scores && primary_rank_scores->count(seq_id) > 0) ?
+ primary_rank_scores->at(seq_id) : default_score;
+ const number_t & secondary_rank_score = (secondary_rank_scores && secondary_rank_scores->count(seq_id) > 0) ?
+ secondary_rank_scores->at(seq_id) : default_score;
+
+ const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;
+ const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;
+ topster.add(seq_id, query_index, match_score, primary_rank_value, secondary_rank_value);
+
+ /*std::cout << "candidate_rank: " << candidate_rank << ", candidate_rank_score: " << candidate_rank_score
+ << ", words_present: " << mscore.words_present << ", match_score: " << match_score
+ << ", primary_rank_score: " << primary_rank_score.intval << ", seq_id: " << seq_id << std::endl;*/
+ }
+
+ for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
+ delete [] it->second;
+ it->second = nullptr;
+ }
+}
+
+void Collection::populate_token_positions(const std::vector<art_leaf *> &query_suggestion,
+ spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
+ size_t result_index, std::vector<std::vector<uint16_t>> &token_positions) const {
+ // for each token in the query, find the positions that it appears in this document
+ for (const art_leaf *token_leaf : query_suggestion) {
+ std::vector<uint16_t> positions;
+ int doc_index = leaf_to_indices.at(token_leaf)[result_index];
+ if(doc_index == token_leaf->values->ids.getLength()) {
+ continue;
+ }
+
+ uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);
+ uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?
+ token_leaf->values->offsets.getLength() :
+ token_leaf->values->offset_index.at(doc_index+1);
+
+ while(start_offset < end_offset) {
+ positions.push_back((uint16_t) token_leaf->values->offsets.at(start_offset));
+ start_offset++;
+ }
+
+ token_positions.push_back(positions);
+ }
+}
+
+inline std::vector<art_leaf *> Collection::next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
+ long long int n) {
+ std::vector<art_leaf*> query_suggestion(token_leaves.size());
+
+ // generate the next combination from `token_leaves` and store it in `query_suggestion`
+ ldiv_t q { n, 0 };
+ for(long long i = 0 ; i < token_leaves.size(); i++) {
+ q = ldiv(q.quot, token_leaves[i].size());
+ query_suggestion[i] = token_leaves[i][q.rem];
+ }
+
+ // sort ascending based on matched documents for each token for faster intersection
+ sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {
+ return left->values->ids.getLength() < right->values->ids.getLength();
+ });
+
+ return query_suggestion;
+}
+
+void Collection::remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
+ const uint32_t indices_length) {
+ uint32_t *curr_array = offset_index.uncompress();
+ uint32_t *new_array = new uint32_t[offset_index.getLength()];
+
+ new_array[0] = 0;
+ uint32_t new_index = 0;
+ uint32_t curr_index = 0;
+ uint32_t indices_counter = 0;
+ uint32_t shift_value = 0;
+
+ while(curr_index < offset_index.getLength()) {
+ if(indices_counter < indices_length && curr_index >= indices_sorted[indices_counter]) {
+ // skip copying
+ if(curr_index == indices_sorted[indices_counter]) {
+ curr_index++;
+ const uint32_t diff = curr_index == offset_index.getLength() ?
+ 0 : (offset_index.at(curr_index) - offset_index.at(curr_index-1));
+
+ shift_value += diff;
+ }
+ indices_counter++;
+ } else {
+ new_array[new_index++] = curr_array[curr_index++] - shift_value;
+ }
+ }
+
+ offset_index.load(new_array, new_index);
+
+ delete[] curr_array;
+ delete[] new_array;
+}
+
+Option<nlohmann::json> Collection::get(const std::string & id) {
+ std::string seq_id_str;
+ StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
+
+ if(status == StoreStatus::NOT_FOUND) {
+ return Option<nlohmann::json>(404, "Could not find a document with id: " + id);
+ }
+
+ uint32_t seq_id = (uint32_t) std::stol(seq_id_str);
+
+ std::string parsed_document;
+ store->get(get_seq_id_key(seq_id), parsed_document);
+
+ nlohmann::json document;
+ try {
+ document = nlohmann::json::parse(parsed_document);
+ } catch(...) {
+ return Option<nlohmann::json>(500, "Error while parsing stored document.");
+ }
+
+ return Option<nlohmann::json>(document);
+}
+
+Option<std::string> Collection::remove(const std::string & id, const bool remove_from_store) {
+ std::string seq_id_str;
+ StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
+
+ if(status == StoreStatus::NOT_FOUND) {
+ return Option<std::string>(404, "Could not find a document with id: " + id);
+ }
+
+ uint32_t seq_id = (uint32_t) std::stol(seq_id_str);
+
+ std::string parsed_document;
+ store->get(get_seq_id_key(seq_id), parsed_document);
+
+ nlohmann::json document;
+ try {
+ document = nlohmann::json::parse(parsed_document);
+ } catch(...) {
+ return Option<std::string>(500, "Error while parsing stored document.");
+ }
+
+ for(auto & name_field: search_schema) {
+ // Go through all the field names and find the keys+values so that they can be removed from in-memory index
+ std::vector<std::string> tokens;
+ if(name_field.second.type == field_types::STRING) {
+ StringUtils::split(document[name_field.first], tokens, " ");
+ } else if(name_field.second.type == field_types::STRING_ARRAY) {
+ tokens = document[name_field.first].get<std::vector<std::string>>();
+ } else if(name_field.second.type == field_types::INT32) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ int32_t value = document[name_field.first].get<int32_t>();
+ encode_int32(value, key);
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ } else if(name_field.second.type == field_types::INT32_ARRAY) {
+ std::vector<int32_t> values = document[name_field.first].get<std::vector<int32_t>>();
+ for(const int32_t value: values) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ encode_int32(value, key);
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ }
+ } else if(name_field.second.type == field_types::INT64) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ int64_t value = document[name_field.first].get<int64_t>();
+ encode_int64(value, key);
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ } else if(name_field.second.type == field_types::INT64_ARRAY) {
+ std::vector<int64_t> values = document[name_field.first].get<std::vector<int64_t>>();
+ for(const int64_t value: values) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ encode_int64(value, key);
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ }
+ } else if(name_field.second.type == field_types::FLOAT) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ int64_t value = document[name_field.first].get<int64_t>();
+ encode_float(value, key);
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ } else if(name_field.second.type == field_types::FLOAT_ARRAY) {
+ std::vector<float> values = document[name_field.first].get<std::vector<float>>();
+ for(const float value: values) {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+ encode_float(value, key);
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ }
+ }
+
+ for(auto & token: tokens) {
+ const unsigned char *key;
+ int key_len;
+
+ if(name_field.second.type == field_types::STRING_ARRAY || name_field.second.type == field_types::STRING) {
+ StringUtils::normalize(token);
+ key = (const unsigned char *) token.c_str();
+ key_len = (int) (token.length() + 1);
+ } else {
+ key = (const unsigned char *) token.c_str();
+ key_len = (int) (token.length());
+ }
+
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(name_field.first), key, key_len);
+ if(leaf != NULL) {
+ uint32_t seq_id_values[1] = {seq_id};
+ uint32_t doc_index = leaf->values->ids.indexOf(seq_id);
+
+ if(doc_index == leaf->values->ids.getLength()) {
+ // not found - happens when 2 tokens repeat in a field, e.g "is it or is is not?"
+ continue;
+ }
+
+ uint32_t start_offset = leaf->values->offset_index.at(doc_index);
+ uint32_t end_offset = (doc_index == leaf->values->ids.getLength() - 1) ?
+ leaf->values->offsets.getLength() :
+ leaf->values->offset_index.at(doc_index+1);
+
+ uint32_t doc_indices[1] = {doc_index};
+ remove_and_shift_offset_index(leaf->values->offset_index, doc_indices, 1);
+
+ leaf->values->offsets.remove_index(start_offset, end_offset);
+ leaf->values->ids.remove_values(seq_id_values, 1);
+
+ /*len = leaf->values->offset_index.getLength();
+ for(auto i=0; i<len; i++) {
+ std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;
+ }
+ std::cout << "----" << std::endl;*/
+
+ if(leaf->values->ids.getLength() == 0) {
+ art_values* values = (art_values*) art_delete(search_index.at(name_field.first), key, key_len);
+ delete values;
+ values = nullptr;
+ }
+ }
+ }
+ }
+
+ // remove facets if any
+ for(auto & field_facet_value: facet_index) {
+ field_facet_value.second.doc_values.erase(seq_id);
+ }
+
+ // remove sort index if any
+ for(auto & field_doc_value_map: sort_index) {
+ field_doc_value_map.second->erase(seq_id);
+ }
+
+ if(remove_from_store) {
+ store->remove(get_doc_id_key(id));
+ store->remove(get_seq_id_key(seq_id));
+ }
+
+ num_documents -= 1;
+
+ return Option<std::string>(id);
+}
+
+std::string Collection::get_next_seq_id_key(const std::string & collection_name) {
+ return std::string(COLLECTION_NEXT_SEQ_PREFIX) + "_" + collection_name;
+}
+
+std::string Collection::get_seq_id_key(uint32_t seq_id) {
+ // We can't simply do std::to_string() because we want to preserve the byte order.
+ // & 0xFF masks all but the lowest eight bits.
+ unsigned char bytes[4];
+ bytes[0] = (unsigned char) ((seq_id >> 24) & 0xFF);
+ bytes[1] = (unsigned char) ((seq_id >> 16) & 0xFF);
+ bytes[2] = (unsigned char) ((seq_id >> 8) & 0xFF);
+ bytes[3] = (unsigned char) ((seq_id & 0xFF));
+
+ return get_seq_id_collection_prefix() + "_" + std::string(bytes, bytes+4);
+}
+
+uint32_t Collection::deserialize_seq_id_key(std::string serialized_seq_id) {
+ uint32_t seq_id = ((serialized_seq_id[0] & 0xFF) << 24) | ((serialized_seq_id[1] & 0xFF) << 16) |
+ ((serialized_seq_id[2] & 0xFF) << 8) | (serialized_seq_id[3] & 0xFF);
+ return seq_id;
+}
+
+std::string Collection::get_doc_id_key(const std::string & doc_id) {
+ return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_" + doc_id;
+}
+
+std::string Collection::get_name() {
+ return name;
+}
+
+size_t Collection::get_num_documents() {
+ return num_documents;
+}
+
+uint32_t Collection::get_collection_id() {
+ return collection_id;
+}
+
+uint32_t Collection::doc_id_to_seq_id(std::string doc_id) {
+ std::string seq_id_str;
+ store->get(get_doc_id_key(doc_id), seq_id_str);
+ uint32_t seq_id = (uint32_t) std::stoi(seq_id_str);
+ return seq_id;
+}
+
+std::vector<std::string> Collection::get_facet_fields() {
+ std::vector<std::string> facet_fields_copy;
+ for(auto it = facet_schema.begin(); it != facet_schema.end(); ++it) {
+ facet_fields_copy.push_back(it->first);
+ }
+
+ return facet_fields_copy;
+}
+
+std::vector<field> Collection::get_sort_fields() {
+ std::vector<field> sort_fields_copy;
+ for(auto it = sort_schema.begin(); it != sort_schema.end(); ++it) {
+ sort_fields_copy.push_back(it->second);
+ }
+
+ return sort_fields_copy;
+}
+
+spp::sparse_hash_map<std::string, field> Collection::get_schema() {
+ return search_schema;
+};
+
+std::string Collection::get_meta_key(const std::string & collection_name) {
+ return std::string(COLLECTION_META_PREFIX) + "_" + collection_name;
+}
+
+std::string Collection::get_seq_id_collection_prefix() {
+ return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);
+}
+
+std::string Collection::get_token_ranking_field() {
+ return token_ranking_field;
+}
\ No newline at end of file