Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Basic string sorting support.
Kishore Nallan4de5b141eede
diff --git a/include/adi_tree.h b/include/adi_tree.hnew file mode 100644index 00000000..dd20bf87--- /dev/null+++ b/include/adi_tree.h@@ -0,0 +1,37 @@+#pragma once+#include <string>+#include "sparsepp.h"++struct adi_node_t;++class adi_tree_t {++private:+ static constexpr size_t NOT_FOUND = UINT64_MAX;++ spp::sparse_hash_map<uint32_t, std::string> id_keys;+ adi_node_t* root = nullptr;++ static void add_node(adi_node_t* node, const std::string& key, size_t key_index);++ static bool rank_aggregate(adi_node_t* node, const std::string& key, size_t key_index, size_t& rank);++ static adi_node_t* get_node(adi_node_t* node, const std::string& key, const size_t key_index,+ std::vector<adi_node_t*>& path);++ void remove_node(adi_node_t* node, const std::string& key, const size_t key_index);++public:++ adi_tree_t();++ ~adi_tree_t();++ void index(uint32_t id, const std::string& key);++ size_t rank(uint32_t id);++ void remove(uint32_t id);++ const adi_node_t* get_root();+};diff --git a/include/field.h b/include/field.hindex 0ad27629..ad23f2b9 100644--- a/include/field.h+++ b/include/field.h@@ -37,6 +37,7 @@ namespace fields {static const std::string facet = "facet";static const std::string optional = "optional";static const std::string index = "index";+ static const std::string sort = "sort";static const std::string locale = "locale";}@@ -46,13 +47,18 @@ struct field {bool facet;bool optional;bool index;-std::string locale;+ bool sort;field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,- bool index = true, std::string locale = "") :+ bool index = true, std::string locale = "", int sort = -1) :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale) {+ if(sort != -1) {+ this->sort = bool(sort);+ } else {+ this->sort = is_num_sort_field();+ }}bool is_auto() const {@@ -132,8 +138,24 @@ struct field {type == field_types::FLOAT || type == field_types::BOOL);}+ bool is_num_sort_field() const {+ return (has_numerical_index() || is_geopoint());+ }++ bool is_sort_field() const {+ return is_num_sort_field() || (type == field_types::STRING);+ }++ bool is_num_sortable() const {+ return sort && is_num_sort_field();+ }++ bool is_str_sortable() const {+ return sort && type == field_types::STRING;+ }+bool is_sortable() const {- return is_single_integer() || is_single_float() || is_single_bool() || is_geopoint();+ return is_num_sortable() || is_str_sortable();}bool has_valid_type() const {@@ -209,6 +231,7 @@ struct field {field_val[fields::type] = field.type;field_val[fields::facet] = field.facet;field_val[fields::optional] = field.optional;+ field_val[fields::sort] = field.sort;field_val[fields::locale] = field.locale;@@ -252,6 +275,10 @@ struct field {return Option<bool>(400, "Field `" + field.name + "` cannot be a facet since ""it's marked as non-indexable.");}++ if(!field.is_sort_field() && field.sort) {+ return Option<bool>(400, "Field `" + field.name + "` cannot be a sortabale field.");+ }}if(!default_sorting_field.empty() && !found_default_sorting_field) {@@ -299,6 +326,11 @@ struct field {field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));}+ if(field_json.count(fields::sort) != 0 && !field_json.at(fields::sort).is_boolean()) {+ return Option<bool>(400, std::string("The `sort` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }+if(field_json.count(fields::locale) != 0){if(!field_json.at(fields::locale).is_string()) {return Option<bool>(400, std::string("The `locale` property of the field `") +@@ -329,6 +361,10 @@ struct field {field_json[fields::locale] = "";}+ if(field_json.count(fields::sort) == 0) {+ field_json[fields::sort] = false;+ }+if(field_json[fields::optional] == false) {return Option<bool>(400, "Field `.*` must be an optional field.");}@@ -342,7 +378,8 @@ struct field {}field fallback_field(field_json["name"], field_json["type"], field_json["facet"],- field_json["optional"], field_json[fields::index], field_json[fields::locale]);+ field_json["optional"], field_json[fields::index], field_json[fields::locale],+ field_json[fields::sort]);if(fallback_field.has_valid_type()) {fallback_field_type = fallback_field.type;@@ -367,6 +404,15 @@ struct field {field_json[fields::locale] = "";}+ if(field_json.count(fields::sort) == 0) {+ if(field_json["type"] == field_types::INT32 || field_json["type"] == field_types::INT64 ||+ field_json["type"] == field_types::BOOL || field_json["type"] == field_types::GEOPOINT) {+ field_json[fields::sort] = true;+ } else {+ field_json[fields::sort] = false;+ }+ }+if(field_json.count(fields::optional) == 0) {// dynamic fields are always optionalbool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);@@ -375,7 +421,8 @@ struct field {fields.emplace_back(field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],- field_json[fields::optional], field_json[fields::index], field_json[fields::locale])+ field_json[fields::optional], field_json[fields::index], field_json[fields::locale],+ field_json[fields::sort]));}diff --git a/include/index.h b/include/index.hindex 291c67df..ef85c1f5 100644--- a/include/index.h+++ b/include/index.h@@ -21,6 +21,7 @@#include "match_score.h"#include "posting_list.h"#include "threadpool.h"+#include "adi_tree.h"static constexpr size_t ARRAY_FACET_DIM = 4;using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;@@ -418,6 +419,9 @@ private:// sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;+ // str_sort_field => adi_tree_t+ spp::sparse_hash_map<std::string, adi_tree_t*> str_sort_index;+// geo_array_field => (seq_id => values) used for exact filtering of geo array recordsspp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;@@ -435,6 +439,7 @@ private:static spp::sparse_hash_map<uint32_t, int64_t> text_match_sentinel_value;static spp::sparse_hash_map<uint32_t, int64_t> seq_id_sentinel_value;static spp::sparse_hash_map<uint32_t, int64_t> geo_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t> str_sentinel_value;// Internal utility functions@@ -530,11 +535,6 @@ private:std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,std::vector<uint64_t>& facet_hashes);- void index_strings_field(const int64_t score, art_tree *t,- uint32_t seq_id, bool is_facet, const field & a_field,- const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- const std::vector<uint64_t>& facet_hashes);-static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,const field& a_field,const std::vector<char>& symbols_to_index,diff --git a/src/adi_tree.cpp b/src/adi_tree.cppnew file mode 100644index 00000000..ade8cbcd--- /dev/null+++ b/src/adi_tree.cpp@@ -0,0 +1,286 @@+#include <cstdint>+#include <vector>+#include "adi_tree.h"+#include "logger.h"+#include <set>++//std::set<adi_node_t*> nodes;++struct adi_node_t {+ uint16_t scions;+ uint16_t num_children;+ char* chars;+ adi_node_t** children;++ adi_node_t(): scions(0), num_children(0), chars(nullptr), children(nullptr) {}++ ~adi_node_t() {+ //LOG(INFO) << "~adi_node: " << this;+ //nodes.erase(this);++ delete [] chars;+ chars = nullptr;++ for(size_t i = 0; i < num_children; i++) {+ delete children[i];+ }+ delete [] children;+ children = nullptr;+ num_children = 0;+ }+};++adi_tree_t::adi_tree_t() {+ root = new adi_node_t();+}++void adi_tree_t::add_node(adi_node_t* node, const std::string& key, const size_t key_index) {+ char c = (key_index == key.size()) ? '\0' : key[key_index];++ // find the slot for `c`+ size_t slot = 0;+ bool found_char = false;++ while(slot < node->num_children) {+ if(node->chars[slot] == c) {+ found_char = true;+ break;+ }++ if(node->chars[slot] > c) {+ break;+ }++ slot++;+ }++ if(!found_char) {+ adi_node_t** new_children = new adi_node_t*[node->num_children+1];+ char* new_chars = new char[node->num_children+1];++ for(size_t i = 0; i < slot; i++) {+ new_children[i] = node->children[i];+ new_chars[i] = node->chars[i];+ }++ new_children[slot] = new adi_node_t();++ /*LOG(INFO) << "new node: " << new_children[slot] << ", slot: " << slot+ << ", parent node: " << node << ", key: " << key+ << ", char: " << int(c) << ", node->num_children: " << node->num_children;*/++ //nodes.insert(new_children[slot]);+ new_chars[slot] = c;++ size_t new_index = slot+1;++ for(size_t i = slot; i < node->num_children; i++) {+ new_children[new_index] = node->children[i];+ new_chars[new_index] = node->chars[i];+ new_index++;+ }++ delete [] node->chars;+ delete [] node->children;++ node->chars = new_chars;+ node->children = new_children;++ node->num_children++;+ }++ node->scions++;++ if(c != '\0') {+ add_node(node->children[slot], key, key_index+1);+ } else {+ node->children[slot]->scions++;+ }+}++void adi_tree_t::index(const uint32_t id, const std::string& key) {+ const auto& id_keys_it = id_keys.find(id);+ if(id_keys_it != id_keys.end()) {+ return ;+ }++ if(key.empty()) {+ return;+ }++ if(root == nullptr) {+ root = new adi_node_t();+ }++ add_node(root, key, 0);+ id_keys.emplace(id, key);+}++bool adi_tree_t::rank_aggregate(adi_node_t* node, const std::string& key, const size_t key_index, size_t& rank) {+ char c = (key_index == key.size()) ? '\0' : key[key_index];++ for(size_t i = 0; i < node->num_children; i++) {+ if(node->chars[i] == '\0' && c == '\0') {+ rank += 1;+ return true;+ }++ if(node->chars[i] == '\0') {+ rank += 1;+ } else if(node->chars[i] != c) {+ rank += node->children[i]->scions;+ } else {+ return rank_aggregate(node->children[i], key, key_index+1, rank);+ }+ }++ return false;+}++size_t adi_tree_t::rank(uint32_t id) {+ const auto& id_keys_it = id_keys.find(id);++ if(id_keys_it == id_keys.end()) {+ return NOT_FOUND;+ }++ const std::string& key = id_keys_it->second;+ size_t rank = 0;+ bool found = rank_aggregate(root, key, 0, rank);++ return found ? rank : NOT_FOUND;+}++adi_node_t* adi_tree_t::get_node(adi_node_t* node, const std::string& key, const size_t key_index,+ std::vector<adi_node_t*>& path) {+ if(key_index == key.size()) {+ // still push the null node+ if(node->num_children >= 1 && node->chars[0] == '\0') {+ path.push_back(node);+ path.push_back(node->children[0]);+ return node;+ }++ return nullptr;+ }++ for(size_t i = 0; i < node->num_children; i++) {+ if(node->chars[i] == key[key_index]) {+ path.push_back(node);+ return get_node(node->children[i], key, key_index+1, path);+ }+ }++ return nullptr;+}++// assumes that node already exists+void adi_tree_t::remove_node(adi_node_t* node, const std::string& key, const size_t key_index) {+ char c = (key_index == key.size()) ? '\0' : key[key_index];++ for(size_t i = 0; i < node->num_children; i++) {+ if(node->chars[i] == c) {+ if(node->children[i]->scions > 1) {+ // skip to next character in trie as this character is shared by other entries+ node->scions--;+ if(c != '\0') {+ remove_node(node->children[i], key, key_index+1);+ } else {+ node->children[i]->scions--;+ }+ } else {+ if(node->num_children == 1) {+ // solo child, we will have to delete the node itself++ if(c != '\0') {+ remove_node(node->children[i], key, key_index+1);+ node->children[i] = nullptr;+ }++ if(root == node) {+ delete root;+ root = nullptr;+ } else {+ delete node;+ }+ } else {+ // delete the char from node++ adi_node_t** new_children = new adi_node_t*[node->num_children-1];+ char* new_chars = new char[node->num_children-1];++ for(size_t j = 0; j < node->num_children; j++) {+ if(j == i) {+ continue;+ }+ size_t index = (j < i) ? j : j-1;+ new_children[index] = node->children[j];+ new_chars[index] = node->chars[j];+ }++ if(c != '\0') {+ remove_node(node->children[i], key, key_index+1);+ } else {+ delete node->children[i];+ }++ delete [] node->chars;+ delete [] node->children;++ node->children = new_children;+ node->chars = new_chars;++ node->scions--;+ node->num_children--;+ }+ }++ break;+ }+ }+}++void adi_tree_t::remove(uint32_t id) {+ const auto& id_keys_it = id_keys.find(id);++ if(id_keys_it == id_keys.end()) {+ return ;+ }++ const std::string& key = id_keys_it->second;+ std::vector<adi_node_t*> path;+ auto leaf_node = get_node(root, key, 0, path);++ //LOG(INFO) << "Removing key: " << key;++ if(leaf_node != nullptr) {+ remove_node(root, key, 0);+ }++ id_keys.erase(id);+}++adi_tree_t::~adi_tree_t() {+ std::vector<uint32_t> ids;++ //LOG(INFO) << "ROOT: " << root;++ for(auto& id_key: id_keys) {+ ids.push_back(id_key.first);+ }++ for(auto id: ids) {+ remove(id);+ }++ //LOG(INFO) << "tree destructor, deleting root: " << root;+ delete root;++ //LOG(INFO) << "nodes.size: " << nodes.size();+ //auto missing_node = *nodes.begin();+ //LOG(INFO) << "missing node: " << missing_node;+}++const adi_node_t* adi_tree_t::get_root() {+ return root;+}diff --git a/src/collection.cpp b/src/collection.cppindex a1f893da..893a4ed6 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2439,12 +2439,15 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constnew_field.type = test_field_type;}+ if (new_field.is_num_sort_field()) {+ // only numerical fields are added to sort index in dynamic type detection+ new_field.sort = true;+ sort_schema.emplace(new_field.name, new_field);+ }+search_schema.emplace(new_field.name, new_field);fields.emplace_back(new_field);new_fields.emplace_back(new_field);- if (new_field.is_sortable()) {- sort_schema.emplace(new_field.name, new_field);- }if(new_field.is_facet()) {facet_schema.emplace(new_field.name, new_field);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex f19c4477..0b63116f 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -37,8 +37,18 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::locale] = "";}- fields.push_back({field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],- field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale]});+ field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],+ field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],+ true);++ // value of `sort` depends on field type+ if(field_obj.count(fields::sort) == 0) {+ f.sort = f.is_num_sort_field();+ } else {+ f.sort = field_obj[fields::sort];+ }++ fields.push_back(f);}std::string default_sorting_field = collection_meta[Collection::COLLECTION_DEFAULT_SORTING_FIELD_KEY].get<std::string>();diff --git a/src/index.cpp b/src/index.cppindex 1f47aead..ee0f395f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -36,6 +36,7 @@spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;Index::Index(const std::string& name, const uint32_t collection_id, const Store* store, ThreadPool* thread_pool,const std::unordered_map<std::string, field> & search_schema,@@ -74,7 +75,10 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*}for(const auto & pair: sort_schema) {- if(pair.second.type != field_types::GEOPOINT_ARRAY) {+ if(pair.second.type == field_types::STRING) {+ adi_tree_t* tree = new adi_tree_t();+ str_sort_index.emplace(pair.first, tree);+ } else if(pair.second.type != field_types::GEOPOINT_ARRAY) {spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();sort_index.emplace(pair.first, doc_to_score);}@@ -135,6 +139,13 @@ Index::~Index() {sort_index.clear();+ for(auto& name_tree: str_sort_index) {+ delete name_tree.second;+ name_tree.second = nullptr;+ }++ str_sort_index.clear();+for(auto& field_name_facet_map_array: facet_index_v3) {for(auto& facet_map: field_name_facet_map_array.second) {delete facet_map;@@ -774,10 +785,8 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>});}- // add numerical values automatically into sort index- if(afield.type == field_types::INT32 || afield.type == field_types::INT64 ||- afield.type == field_types::FLOAT || afield.type == field_types::BOOL ||- afield.type == field_types::GEOPOINT) {+ // add numerical values automatically into sort index if sorting is enabled+ if(afield.is_num_sortable() && afield.type != field_types::GEOPOINT_ARRAY) {spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(afield.name);bool is_integer = afield.is_integer();@@ -811,6 +820,25 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}}}+ } else if(afield.is_str_sortable()) {+ adi_tree_t* str_tree = str_sort_index.at(afield.name);++ for(const auto& record: iter_batch) {+ if(!record.indexed.ok()) {+ continue;+ }++ const auto& document = record.doc;+ const auto seq_id = record.seq_id;++ if (document.count(afield.name) == 0 || !afield.index) {+ continue;+ }++ std::string raw_str = document[afield.name].get<std::string>();+ StringUtils::tolowercase(raw_str);+ str_tree->index(seq_id, raw_str);+ }}}@@ -876,32 +904,6 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,}}-void Index::index_strings_field(const int64_t score, art_tree *t,- uint32_t seq_id, bool is_facet, const field & a_field,- const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- const std::vector<uint64_t>& facet_hashes) {-- // requires unique lock from caller-- if(token_to_offsets.empty()) {- return;- }-- insert_doc(score, t, seq_id, token_to_offsets);-- if(is_facet) {- facet_hash_values_t fhashvalues;- fhashvalues.length = facet_hashes.size();- fhashvalues.hashes = new uint64_t[facet_hashes.size()];-- for(size_t i = 0; i < facet_hashes.size(); i++) {- fhashvalues.hashes[i] = facet_hashes[i];- }-- facet_index_v3[a_field.name][seq_id % ARRAY_FACET_DIM]->emplace(seq_id, std::move(fhashvalues));- }-}-void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,const field& a_field,const std::vector<char>& symbols_to_index,@@ -2810,6 +2812,8 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopointif (sort_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {geopoint_indices.push_back(i);field_values[i] = nullptr; // GEOPOINT_ARRAY uses a multi-valued index+ } else if(sort_schema.at(sort_fields_std[i].name).type == field_types::STRING) {+ field_values[i] = &str_sentinel_value;} else {field_values[i] = sort_index.at(sort_fields_std[i].name);@@ -3212,6 +3216,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16scores[0] = seq_id;} else if(field_values[0] == &geo_sentinel_value) {scores[0] = geopoint_distances[0];+ } else if(field_values[0] == &str_sentinel_value) {+ scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);} else {auto it = field_values[0]->find(seq_id);scores[0] = (it == field_values[0]->end()) ? default_score : it->second;@@ -3230,6 +3236,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16scores[1] = seq_id;} else if(field_values[1] == &geo_sentinel_value) {scores[1] = geopoint_distances[1];+ } else if(field_values[0] == &str_sentinel_value) {+ scores[1] = str_sort_index.at(sort_fields[1].name)->rank(seq_id);} else {auto it = field_values[1]->find(seq_id);scores[1] = (it == field_values[1]->end()) ? default_score : it->second;@@ -3248,6 +3256,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16scores[2] = seq_id;} else if(field_values[2] == &geo_sentinel_value) {scores[2] = geopoint_distances[2];+ } else if(field_values[0] == &str_sentinel_value) {+ scores[2] = str_sort_index.at(sort_fields[2].name)->rank(seq_id);} else {auto it = field_values[2]->find(seq_id);scores[2] = (it == field_values[2]->end()) ? default_score : it->second;@@ -3464,6 +3474,10 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docif(sort_index.count(field_name) != 0) {sort_index[field_name]->erase(seq_id);}++ if(str_sort_index.count(field_name) != 0) {+ str_sort_index[field_name]->remove(seq_id);+ }}if(!is_update) {@@ -3507,7 +3521,17 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {for(const auto & new_field: new_fields) {search_schema.emplace(new_field.name, new_field);- sort_schema.emplace(new_field.name, new_field);++ if(new_field.is_sortable()) {+ sort_schema.emplace(new_field.name, new_field);++ if(new_field.is_num_sortable()) {+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();+ sort_index.emplace(new_field.name, doc_to_score);+ } else if(new_field.is_str_sortable()) {+ str_sort_index.emplace(new_field.name, new adi_tree_t);+ }+ }if(search_index.count(new_field.name) == 0) {if(new_field.is_string() || field_types::is_string_or_array(new_field.type)) {@@ -3544,11 +3568,6 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {search_index.emplace(new_field.faceted_name(), ft);}}-- if(sort_index.count(new_field.name) == 0) {- spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();- sort_index.emplace(new_field.name, doc_to_score);- }}}diff --git a/test/adi_tree_test.cpp b/test/adi_tree_test.cppnew file mode 100644index 00000000..7615cf4e--- /dev/null+++ b/test/adi_tree_test.cpp@@ -0,0 +1,107 @@+#include "adi_tree.h"+#include <gtest/gtest.h>+#include "logger.h"+#include <fstream>++class ADITreeTest : public ::testing::Test {+protected:++ virtual void SetUp() {++ }++ virtual void TearDown() {++ }+};++TEST_F(ADITreeTest, BasicOps) {+ adi_tree_t tree;++ // operations on fresh tree+ ASSERT_EQ(UINT64_MAX, tree.rank(100));+ tree.remove(100);++ tree.index(100, "f");+ ASSERT_EQ(1, tree.rank(100));++ tree.index(101, "e");+ ASSERT_EQ(2, tree.rank(100));+ ASSERT_EQ(1, tree.rank(101));++ tree.remove(101);+ ASSERT_EQ(1, tree.rank(100));++ tree.remove(100);+ ASSERT_EQ(UINT64_MAX, tree.rank(100));+ ASSERT_EQ(UINT64_MAX, tree.rank(101));+}++TEST_F(ADITreeTest, OverlappedString) {+ adi_tree_t tree;+ tree.index(1, "t");+ tree.index(2, "to");++ ASSERT_EQ(2, tree.rank(2));+ ASSERT_EQ(1, tree.rank(1));++ tree.remove(1);+ tree.remove(2);++ ASSERT_EQ(UINT64_MAX, tree.rank(2));+ ASSERT_EQ(UINT64_MAX, tree.rank(1));+}++TEST_F(ADITreeTest, OrderInsertedStrings) {+ std::vector<std::pair<uint32_t, std::string>> records = {+ {1, "alpha"}, {2, "beta"},+ {3, "foo"}, {4, "ant"}, {5, "foobar"},+ {6, "buzz"}+ };++ adi_tree_t tree;+ for(auto& record: records) {+ tree.index(record.first, record.second);+ }++ std::sort(records.begin(), records.end(),+ [](const std::pair<uint32_t, std::string>& a, const std::pair<uint32_t, std::string>& b) -> bool {+ return a.second < b.second;+ });++ // alpha, ant, beta, buzz, foo, foobar+ ASSERT_EQ(1, tree.rank(1));+ ASSERT_EQ(3, tree.rank(2));+ ASSERT_EQ(5, tree.rank(3));+ ASSERT_EQ(2, tree.rank(4));+ ASSERT_EQ(6, tree.rank(5));+ ASSERT_EQ(4, tree.rank(6));++ // remove "foo"+ tree.remove(3);+ ASSERT_EQ(5, tree.rank(5));++ // remove "foobar"+ tree.remove(5);+ ASSERT_EQ(4, tree.rank(6));++ // remove "alpha"+ tree.remove(1);+ ASSERT_EQ(1, tree.rank(4));+ ASSERT_EQ(2, tree.rank(2));+ ASSERT_EQ(3, tree.rank(6));+}++TEST_F(ADITreeTest, InsertDuplicateAndDelete) {+ adi_tree_t tree;+ tree.index(100, "map");+ tree.index(101, "map");++ tree.remove(100);+ tree.remove(101);++ ASSERT_EQ(UINT64_MAX, tree.rank(100));+ ASSERT_EQ(UINT64_MAX, tree.rank(101));++ ASSERT_EQ(nullptr, tree.get_root());+}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 505bf364..01f5d4d6 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -25,9 +25,9 @@ protected:collectionManager.load(8, 1000);search_fields = {- field("title", field_types::STRING, false, false, true, "en"),- field("starring", field_types::STRING, false),- field("cast", field_types::STRING_ARRAY, true, true),+ field("title", field_types::STRING, false, false, true, "en", false),+ field("starring", field_types::STRING, false, false, true, "", false),+ field("cast", field_types::STRING_ARRAY, true, true, true, "", false),field(".*_year", field_types::INT32, true, true),field("location", field_types::GEOPOINT, false, true, true),field("points", field_types::INT32, false)@@ -90,12 +90,12 @@ TEST_F(CollectionManagerTest, CollectionCreation) {// we already call `collection1->get_next_seq_id` above, which is side-effectingASSERT_EQ(1, StringUtils::deserialize_uint32_t(next_seq_id));ASSERT_EQ("{\"created_at\":12345,\"default_sorting_field\":\"points\",\"fallback_field_type\":\"\","- "\"fields\":[{\"facet\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"- "{\"facet\":false,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"- "{\"facet\":true,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"- "{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},"- "{\"facet\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},"- "{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,"+ "\"fields\":[{\"facet\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"+ "{\"facet\":false,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"+ "{\"facet\":true,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"sort\":false,\"type\":\"string[]\"},"+ "{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"sort\":true,\"type\":\"int32\"},"+ "{\"facet\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"sort\":true,\"type\":\"geopoint\"},"+ "{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"sort\":true,\"type\":\"int32\"}],\"id\":0,""\"name\":\"collection1\",\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",collection_meta_json);ASSERT_EQ("1", next_collection_id);diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex c69905b9..aed04566 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -1014,3 +1014,93 @@ TEST_F(CollectionSortingTest, GeoPointArraySorting) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionSortingTest, SortByTitle) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", true),+ field("artist", field_types::STRING, true),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 2, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"aaa", "ABCD"},+ {"a", "ABCD"},+ {"abcd", "ABCD"},+ {"abdde", "ABCD"},+ {"b", "ABCD"},+ {"bab", "ABCD"},+ {"baa", "ABCD"},+ {"bcma", "ABCD"},+ {"cdma", "ABCD"},+ {"cc", "ABCD"},+ {"c", "ABCD"},+ {"cxya", "ABCD"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ std::vector<sort_by> sort_fields = {+ sort_by("title", "ASC")+ };++ std::vector<std::string> expected_order = {+ "a",+ "aaa",+ "abcd",+ "abdde",+ "b",+ "baa",+ "bab",+ "bcma",+ "c",+ "cc",+ "cdma",+ "cxya"+ };++ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 20, 1, FREQUENCY, {true}, 10).get();++ ASSERT_EQ(12, results["found"].get<size_t>());++ for(size_t i = 0; i < results["hits"].size(); i++) {+ ASSERT_EQ(expected_order[i], results["hits"][i]["document"]["title"].get<std::string>());+ }++ // descending order+ sort_fields = {+ sort_by("title", "DESC")+ };++ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 20, 1, FREQUENCY, {true}, 10).get();++ ASSERT_EQ(12, results["found"].get<size_t>());++ for(size_t i = 0; i < results["hits"].size(); i++) {+ ASSERT_EQ(expected_order[expected_order.size() - i - 1], results["hits"][i]["document"]["title"].get<std::string>());+ }++ // when querying for a string field with sort disabled+ sort_fields = {+ sort_by("artist", "DESC")+ };++ auto res_op = coll1->search("*", {}, "", {}, sort_fields, {0}, 20, 1, FREQUENCY, {true}, 10);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `artist` in the schema for sorting.", res_op.error());++ collectionManager.drop_collection("coll1");+}