Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Basic string sorting support.

Kishore Nallan4de5b141eede
diff --git a/include/adi_tree.h b/include/adi_tree.h
new file mode 100644
index 00000000..dd20bf87
--- /dev/null
+++ b/include/adi_tree.h
@@ -0,0 +1,37 @@
+#pragma once
+#include <string>
+#include "sparsepp.h"
+
+struct adi_node_t;
+
+class adi_tree_t {
+
+private:
+ static constexpr size_t NOT_FOUND = UINT64_MAX;
+
+ spp::sparse_hash_map<uint32_t, std::string> id_keys;
+ adi_node_t* root = nullptr;
+
+ static void add_node(adi_node_t* node, const std::string& key, size_t key_index);
+
+ static bool rank_aggregate(adi_node_t* node, const std::string& key, size_t key_index, size_t& rank);
+
+ static adi_node_t* get_node(adi_node_t* node, const std::string& key, const size_t key_index,
+ std::vector<adi_node_t*>& path);
+
+ void remove_node(adi_node_t* node, const std::string& key, const size_t key_index);
+
+public:
+
+ adi_tree_t();
+
+ ~adi_tree_t();
+
+ void index(uint32_t id, const std::string& key);
+
+ size_t rank(uint32_t id);
+
+ void remove(uint32_t id);
+
+ const adi_node_t* get_root();
+};
diff --git a/include/field.h b/include/field.h
index 0ad27629..ad23f2b9 100644
--- a/include/field.h
+++ b/include/field.h
@@ -37,6 +37,7 @@ namespace fields {
static const std::string facet = "facet";
static const std::string optional = "optional";
static const std::string index = "index";
+ static const std::string sort = "sort";
static const std::string locale = "locale";
}
@@ -46,13 +47,18 @@ struct field {
bool facet;
bool optional;
bool index;
-
std::string locale;
+ bool sort;
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
- bool index = true, std::string locale = "") :
+ bool index = true, std::string locale = "", int sort = -1) :
name(name), type(type), facet(facet), optional(optional), index(index), locale(locale) {
+ if(sort != -1) {
+ this->sort = bool(sort);
+ } else {
+ this->sort = is_num_sort_field();
+ }
}
bool is_auto() const {
@@ -132,8 +138,24 @@ struct field {
type == field_types::FLOAT || type == field_types::BOOL);
}
+ bool is_num_sort_field() const {
+ return (has_numerical_index() || is_geopoint());
+ }
+
+ bool is_sort_field() const {
+ return is_num_sort_field() || (type == field_types::STRING);
+ }
+
+ bool is_num_sortable() const {
+ return sort && is_num_sort_field();
+ }
+
+ bool is_str_sortable() const {
+ return sort && type == field_types::STRING;
+ }
+
bool is_sortable() const {
- return is_single_integer() || is_single_float() || is_single_bool() || is_geopoint();
+ return is_num_sortable() || is_str_sortable();
}
bool has_valid_type() const {
@@ -209,6 +231,7 @@ struct field {
field_val[fields::type] = field.type;
field_val[fields::facet] = field.facet;
field_val[fields::optional] = field.optional;
+ field_val[fields::sort] = field.sort;
field_val[fields::locale] = field.locale;
@@ -252,6 +275,10 @@ struct field {
return Option<bool>(400, "Field `" + field.name + "` cannot be a facet since "
"it's marked as non-indexable.");
}
+
+ if(!field.is_sort_field() && field.sort) {
+ return Option<bool>(400, "Field `" + field.name + "` cannot be a sortabale field.");
+ }
}
if(!default_sorting_field.empty() && !found_default_sorting_field) {
@@ -299,6 +326,11 @@ struct field {
field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
}
+ if(field_json.count(fields::sort) != 0 && !field_json.at(fields::sort).is_boolean()) {
+ return Option<bool>(400, std::string("The `sort` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
if(field_json.count(fields::locale) != 0){
if(!field_json.at(fields::locale).is_string()) {
return Option<bool>(400, std::string("The `locale` property of the field `") +
@@ -329,6 +361,10 @@ struct field {
field_json[fields::locale] = "";
}
+ if(field_json.count(fields::sort) == 0) {
+ field_json[fields::sort] = false;
+ }
+
if(field_json[fields::optional] == false) {
return Option<bool>(400, "Field `.*` must be an optional field.");
}
@@ -342,7 +378,8 @@ struct field {
}
field fallback_field(field_json["name"], field_json["type"], field_json["facet"],
- field_json["optional"], field_json[fields::index], field_json[fields::locale]);
+ field_json["optional"], field_json[fields::index], field_json[fields::locale],
+ field_json[fields::sort]);
if(fallback_field.has_valid_type()) {
fallback_field_type = fallback_field.type;
@@ -367,6 +404,15 @@ struct field {
field_json[fields::locale] = "";
}
+ if(field_json.count(fields::sort) == 0) {
+ if(field_json["type"] == field_types::INT32 || field_json["type"] == field_types::INT64 ||
+ field_json["type"] == field_types::BOOL || field_json["type"] == field_types::GEOPOINT) {
+ field_json[fields::sort] = true;
+ } else {
+ field_json[fields::sort] = false;
+ }
+ }
+
if(field_json.count(fields::optional) == 0) {
// dynamic fields are always optional
bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);
@@ -375,7 +421,8 @@ struct field {
fields.emplace_back(
field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
- field_json[fields::optional], field_json[fields::index], field_json[fields::locale])
+ field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
+ field_json[fields::sort])
);
}
diff --git a/include/index.h b/include/index.h
index 291c67df..ef85c1f5 100644
--- a/include/index.h
+++ b/include/index.h
@@ -21,6 +21,7 @@
#include "match_score.h"
#include "posting_list.h"
#include "threadpool.h"
+#include "adi_tree.h"
static constexpr size_t ARRAY_FACET_DIM = 4;
using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;
@@ -418,6 +419,9 @@ private:
// sort_field => (seq_id => value)
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;
+ // str_sort_field => adi_tree_t
+ spp::sparse_hash_map<std::string, adi_tree_t*> str_sort_index;
+
// geo_array_field => (seq_id => values) used for exact filtering of geo array records
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;
@@ -435,6 +439,7 @@ private:
static spp::sparse_hash_map<uint32_t, int64_t> text_match_sentinel_value;
static spp::sparse_hash_map<uint32_t, int64_t> seq_id_sentinel_value;
static spp::sparse_hash_map<uint32_t, int64_t> geo_sentinel_value;
+ static spp::sparse_hash_map<uint32_t, int64_t> str_sentinel_value;
// Internal utility functions
@@ -530,11 +535,6 @@ private:
std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
std::vector<uint64_t>& facet_hashes);
- void index_strings_field(const int64_t score, art_tree *t,
- uint32_t seq_id, bool is_facet, const field & a_field,
- const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- const std::vector<uint64_t>& facet_hashes);
-
static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
const field& a_field,
const std::vector<char>& symbols_to_index,
diff --git a/src/adi_tree.cpp b/src/adi_tree.cpp
new file mode 100644
index 00000000..ade8cbcd
--- /dev/null
+++ b/src/adi_tree.cpp
@@ -0,0 +1,286 @@
+#include <cstdint>
+#include <vector>
+#include "adi_tree.h"
+#include "logger.h"
+#include <set>
+
+//std::set<adi_node_t*> nodes;
+
+struct adi_node_t {
+ uint16_t scions;
+ uint16_t num_children;
+ char* chars;
+ adi_node_t** children;
+
+ adi_node_t(): scions(0), num_children(0), chars(nullptr), children(nullptr) {}
+
+ ~adi_node_t() {
+ //LOG(INFO) << "~adi_node: " << this;
+ //nodes.erase(this);
+
+ delete [] chars;
+ chars = nullptr;
+
+ for(size_t i = 0; i < num_children; i++) {
+ delete children[i];
+ }
+ delete [] children;
+ children = nullptr;
+ num_children = 0;
+ }
+};
+
+adi_tree_t::adi_tree_t() {
+ root = new adi_node_t();
+}
+
+void adi_tree_t::add_node(adi_node_t* node, const std::string& key, const size_t key_index) {
+ char c = (key_index == key.size()) ? '\0' : key[key_index];
+
+ // find the slot for `c`
+ size_t slot = 0;
+ bool found_char = false;
+
+ while(slot < node->num_children) {
+ if(node->chars[slot] == c) {
+ found_char = true;
+ break;
+ }
+
+ if(node->chars[slot] > c) {
+ break;
+ }
+
+ slot++;
+ }
+
+ if(!found_char) {
+ adi_node_t** new_children = new adi_node_t*[node->num_children+1];
+ char* new_chars = new char[node->num_children+1];
+
+ for(size_t i = 0; i < slot; i++) {
+ new_children[i] = node->children[i];
+ new_chars[i] = node->chars[i];
+ }
+
+ new_children[slot] = new adi_node_t();
+
+ /*LOG(INFO) << "new node: " << new_children[slot] << ", slot: " << slot
+ << ", parent node: " << node << ", key: " << key
+ << ", char: " << int(c) << ", node->num_children: " << node->num_children;*/
+
+ //nodes.insert(new_children[slot]);
+ new_chars[slot] = c;
+
+ size_t new_index = slot+1;
+
+ for(size_t i = slot; i < node->num_children; i++) {
+ new_children[new_index] = node->children[i];
+ new_chars[new_index] = node->chars[i];
+ new_index++;
+ }
+
+ delete [] node->chars;
+ delete [] node->children;
+
+ node->chars = new_chars;
+ node->children = new_children;
+
+ node->num_children++;
+ }
+
+ node->scions++;
+
+ if(c != '\0') {
+ add_node(node->children[slot], key, key_index+1);
+ } else {
+ node->children[slot]->scions++;
+ }
+}
+
+void adi_tree_t::index(const uint32_t id, const std::string& key) {
+ const auto& id_keys_it = id_keys.find(id);
+ if(id_keys_it != id_keys.end()) {
+ return ;
+ }
+
+ if(key.empty()) {
+ return;
+ }
+
+ if(root == nullptr) {
+ root = new adi_node_t();
+ }
+
+ add_node(root, key, 0);
+ id_keys.emplace(id, key);
+}
+
+bool adi_tree_t::rank_aggregate(adi_node_t* node, const std::string& key, const size_t key_index, size_t& rank) {
+ char c = (key_index == key.size()) ? '\0' : key[key_index];
+
+ for(size_t i = 0; i < node->num_children; i++) {
+ if(node->chars[i] == '\0' && c == '\0') {
+ rank += 1;
+ return true;
+ }
+
+ if(node->chars[i] == '\0') {
+ rank += 1;
+ } else if(node->chars[i] != c) {
+ rank += node->children[i]->scions;
+ } else {
+ return rank_aggregate(node->children[i], key, key_index+1, rank);
+ }
+ }
+
+ return false;
+}
+
+size_t adi_tree_t::rank(uint32_t id) {
+ const auto& id_keys_it = id_keys.find(id);
+
+ if(id_keys_it == id_keys.end()) {
+ return NOT_FOUND;
+ }
+
+ const std::string& key = id_keys_it->second;
+ size_t rank = 0;
+ bool found = rank_aggregate(root, key, 0, rank);
+
+ return found ? rank : NOT_FOUND;
+}
+
+adi_node_t* adi_tree_t::get_node(adi_node_t* node, const std::string& key, const size_t key_index,
+ std::vector<adi_node_t*>& path) {
+ if(key_index == key.size()) {
+ // still push the null node
+ if(node->num_children >= 1 && node->chars[0] == '\0') {
+ path.push_back(node);
+ path.push_back(node->children[0]);
+ return node;
+ }
+
+ return nullptr;
+ }
+
+ for(size_t i = 0; i < node->num_children; i++) {
+ if(node->chars[i] == key[key_index]) {
+ path.push_back(node);
+ return get_node(node->children[i], key, key_index+1, path);
+ }
+ }
+
+ return nullptr;
+}
+
+// assumes that node already exists
+void adi_tree_t::remove_node(adi_node_t* node, const std::string& key, const size_t key_index) {
+ char c = (key_index == key.size()) ? '\0' : key[key_index];
+
+ for(size_t i = 0; i < node->num_children; i++) {
+ if(node->chars[i] == c) {
+ if(node->children[i]->scions > 1) {
+ // skip to next character in trie as this character is shared by other entries
+ node->scions--;
+ if(c != '\0') {
+ remove_node(node->children[i], key, key_index+1);
+ } else {
+ node->children[i]->scions--;
+ }
+ } else {
+ if(node->num_children == 1) {
+ // solo child, we will have to delete the node itself
+
+ if(c != '\0') {
+ remove_node(node->children[i], key, key_index+1);
+ node->children[i] = nullptr;
+ }
+
+ if(root == node) {
+ delete root;
+ root = nullptr;
+ } else {
+ delete node;
+ }
+ } else {
+ // delete the char from node
+
+ adi_node_t** new_children = new adi_node_t*[node->num_children-1];
+ char* new_chars = new char[node->num_children-1];
+
+ for(size_t j = 0; j < node->num_children; j++) {
+ if(j == i) {
+ continue;
+ }
+ size_t index = (j < i) ? j : j-1;
+ new_children[index] = node->children[j];
+ new_chars[index] = node->chars[j];
+ }
+
+ if(c != '\0') {
+ remove_node(node->children[i], key, key_index+1);
+ } else {
+ delete node->children[i];
+ }
+
+ delete [] node->chars;
+ delete [] node->children;
+
+ node->children = new_children;
+ node->chars = new_chars;
+
+ node->scions--;
+ node->num_children--;
+ }
+ }
+
+ break;
+ }
+ }
+}
+
+void adi_tree_t::remove(uint32_t id) {
+ const auto& id_keys_it = id_keys.find(id);
+
+ if(id_keys_it == id_keys.end()) {
+ return ;
+ }
+
+ const std::string& key = id_keys_it->second;
+ std::vector<adi_node_t*> path;
+ auto leaf_node = get_node(root, key, 0, path);
+
+ //LOG(INFO) << "Removing key: " << key;
+
+ if(leaf_node != nullptr) {
+ remove_node(root, key, 0);
+ }
+
+ id_keys.erase(id);
+}
+
+adi_tree_t::~adi_tree_t() {
+ std::vector<uint32_t> ids;
+
+ //LOG(INFO) << "ROOT: " << root;
+
+ for(auto& id_key: id_keys) {
+ ids.push_back(id_key.first);
+ }
+
+ for(auto id: ids) {
+ remove(id);
+ }
+
+ //LOG(INFO) << "tree destructor, deleting root: " << root;
+ delete root;
+
+ //LOG(INFO) << "nodes.size: " << nodes.size();
+ //auto missing_node = *nodes.begin();
+ //LOG(INFO) << "missing node: " << missing_node;
+}
+
+const adi_node_t* adi_tree_t::get_root() {
+ return root;
+}
diff --git a/src/collection.cpp b/src/collection.cpp
index a1f893da..893a4ed6 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2439,12 +2439,15 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
new_field.type = test_field_type;
}
+ if (new_field.is_num_sort_field()) {
+ // only numerical fields are added to sort index in dynamic type detection
+ new_field.sort = true;
+ sort_schema.emplace(new_field.name, new_field);
+ }
+
search_schema.emplace(new_field.name, new_field);
fields.emplace_back(new_field);
new_fields.emplace_back(new_field);
- if (new_field.is_sortable()) {
- sort_schema.emplace(new_field.name, new_field);
- }
if(new_field.is_facet()) {
facet_schema.emplace(new_field.name, new_field);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index f19c4477..0b63116f 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -37,8 +37,18 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
field_obj[fields::locale] = "";
}
- fields.push_back({field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],
- field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale]});
+ field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],
+ field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],
+ true);
+
+ // value of `sort` depends on field type
+ if(field_obj.count(fields::sort) == 0) {
+ f.sort = f.is_num_sort_field();
+ } else {
+ f.sort = field_obj[fields::sort];
+ }
+
+ fields.push_back(f);
}
std::string default_sorting_field = collection_meta[Collection::COLLECTION_DEFAULT_SORTING_FIELD_KEY].get<std::string>();
diff --git a/src/index.cpp b/src/index.cpp
index 1f47aead..ee0f395f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -36,6 +36,7 @@
spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;
+spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;
Index::Index(const std::string& name, const uint32_t collection_id, const Store* store, ThreadPool* thread_pool,
const std::unordered_map<std::string, field> & search_schema,
@@ -74,7 +75,10 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*
}
for(const auto & pair: sort_schema) {
- if(pair.second.type != field_types::GEOPOINT_ARRAY) {
+ if(pair.second.type == field_types::STRING) {
+ adi_tree_t* tree = new adi_tree_t();
+ str_sort_index.emplace(pair.first, tree);
+ } else if(pair.second.type != field_types::GEOPOINT_ARRAY) {
spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
sort_index.emplace(pair.first, doc_to_score);
}
@@ -135,6 +139,13 @@ Index::~Index() {
sort_index.clear();
+ for(auto& name_tree: str_sort_index) {
+ delete name_tree.second;
+ name_tree.second = nullptr;
+ }
+
+ str_sort_index.clear();
+
for(auto& field_name_facet_map_array: facet_index_v3) {
for(auto& facet_map: field_name_facet_map_array.second) {
delete facet_map;
@@ -774,10 +785,8 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
});
}
- // add numerical values automatically into sort index
- if(afield.type == field_types::INT32 || afield.type == field_types::INT64 ||
- afield.type == field_types::FLOAT || afield.type == field_types::BOOL ||
- afield.type == field_types::GEOPOINT) {
+ // add numerical values automatically into sort index if sorting is enabled
+ if(afield.is_num_sortable() && afield.type != field_types::GEOPOINT_ARRAY) {
spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(afield.name);
bool is_integer = afield.is_integer();
@@ -811,6 +820,25 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
}
}
}
+ } else if(afield.is_str_sortable()) {
+ adi_tree_t* str_tree = str_sort_index.at(afield.name);
+
+ for(const auto& record: iter_batch) {
+ if(!record.indexed.ok()) {
+ continue;
+ }
+
+ const auto& document = record.doc;
+ const auto seq_id = record.seq_id;
+
+ if (document.count(afield.name) == 0 || !afield.index) {
+ continue;
+ }
+
+ std::string raw_str = document[afield.name].get<std::string>();
+ StringUtils::tolowercase(raw_str);
+ str_tree->index(seq_id, raw_str);
+ }
}
}
@@ -876,32 +904,6 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,
}
}
-void Index::index_strings_field(const int64_t score, art_tree *t,
- uint32_t seq_id, bool is_facet, const field & a_field,
- const std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- const std::vector<uint64_t>& facet_hashes) {
-
- // requires unique lock from caller
-
- if(token_to_offsets.empty()) {
- return;
- }
-
- insert_doc(score, t, seq_id, token_to_offsets);
-
- if(is_facet) {
- facet_hash_values_t fhashvalues;
- fhashvalues.length = facet_hashes.size();
- fhashvalues.hashes = new uint64_t[facet_hashes.size()];
-
- for(size_t i = 0; i < facet_hashes.size(); i++) {
- fhashvalues.hashes[i] = facet_hashes[i];
- }
-
- facet_index_v3[a_field.name][seq_id % ARRAY_FACET_DIM]->emplace(seq_id, std::move(fhashvalues));
- }
-}
-
void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
const field& a_field,
const std::vector<char>& symbols_to_index,
@@ -2810,6 +2812,8 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint
if (sort_schema.at(sort_fields_std[i].name).type == field_types::GEOPOINT_ARRAY) {
geopoint_indices.push_back(i);
field_values[i] = nullptr; // GEOPOINT_ARRAY uses a multi-valued index
+ } else if(sort_schema.at(sort_fields_std[i].name).type == field_types::STRING) {
+ field_values[i] = &str_sentinel_value;
} else {
field_values[i] = sort_index.at(sort_fields_std[i].name);
@@ -3212,6 +3216,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
scores[0] = seq_id;
} else if(field_values[0] == &geo_sentinel_value) {
scores[0] = geopoint_distances[0];
+ } else if(field_values[0] == &str_sentinel_value) {
+ scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);
} else {
auto it = field_values[0]->find(seq_id);
scores[0] = (it == field_values[0]->end()) ? default_score : it->second;
@@ -3230,6 +3236,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
scores[1] = seq_id;
} else if(field_values[1] == &geo_sentinel_value) {
scores[1] = geopoint_distances[1];
+ } else if(field_values[0] == &str_sentinel_value) {
+ scores[1] = str_sort_index.at(sort_fields[1].name)->rank(seq_id);
} else {
auto it = field_values[1]->find(seq_id);
scores[1] = (it == field_values[1]->end()) ? default_score : it->second;
@@ -3248,6 +3256,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
scores[2] = seq_id;
} else if(field_values[2] == &geo_sentinel_value) {
scores[2] = geopoint_distances[2];
+ } else if(field_values[0] == &str_sentinel_value) {
+ scores[2] = str_sort_index.at(sort_fields[2].name)->rank(seq_id);
} else {
auto it = field_values[2]->find(seq_id);
scores[2] = (it == field_values[2]->end()) ? default_score : it->second;
@@ -3464,6 +3474,10 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
if(sort_index.count(field_name) != 0) {
sort_index[field_name]->erase(seq_id);
}
+
+ if(str_sort_index.count(field_name) != 0) {
+ str_sort_index[field_name]->remove(seq_id);
+ }
}
if(!is_update) {
@@ -3507,7 +3521,17 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {
for(const auto & new_field: new_fields) {
search_schema.emplace(new_field.name, new_field);
- sort_schema.emplace(new_field.name, new_field);
+
+ if(new_field.is_sortable()) {
+ sort_schema.emplace(new_field.name, new_field);
+
+ if(new_field.is_num_sortable()) {
+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
+ sort_index.emplace(new_field.name, doc_to_score);
+ } else if(new_field.is_str_sortable()) {
+ str_sort_index.emplace(new_field.name, new adi_tree_t);
+ }
+ }
if(search_index.count(new_field.name) == 0) {
if(new_field.is_string() || field_types::is_string_or_array(new_field.type)) {
@@ -3544,11 +3568,6 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {
search_index.emplace(new_field.faceted_name(), ft);
}
}
-
- if(sort_index.count(new_field.name) == 0) {
- spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
- sort_index.emplace(new_field.name, doc_to_score);
- }
}
}
diff --git a/test/adi_tree_test.cpp b/test/adi_tree_test.cpp
new file mode 100644
index 00000000..7615cf4e
--- /dev/null
+++ b/test/adi_tree_test.cpp
@@ -0,0 +1,107 @@
+#include "adi_tree.h"
+#include <gtest/gtest.h>
+#include "logger.h"
+#include <fstream>
+
+class ADITreeTest : public ::testing::Test {
+protected:
+
+ virtual void SetUp() {
+
+ }
+
+ virtual void TearDown() {
+
+ }
+};
+
+TEST_F(ADITreeTest, BasicOps) {
+ adi_tree_t tree;
+
+ // operations on fresh tree
+ ASSERT_EQ(UINT64_MAX, tree.rank(100));
+ tree.remove(100);
+
+ tree.index(100, "f");
+ ASSERT_EQ(1, tree.rank(100));
+
+ tree.index(101, "e");
+ ASSERT_EQ(2, tree.rank(100));
+ ASSERT_EQ(1, tree.rank(101));
+
+ tree.remove(101);
+ ASSERT_EQ(1, tree.rank(100));
+
+ tree.remove(100);
+ ASSERT_EQ(UINT64_MAX, tree.rank(100));
+ ASSERT_EQ(UINT64_MAX, tree.rank(101));
+}
+
+TEST_F(ADITreeTest, OverlappedString) {
+ adi_tree_t tree;
+ tree.index(1, "t");
+ tree.index(2, "to");
+
+ ASSERT_EQ(2, tree.rank(2));
+ ASSERT_EQ(1, tree.rank(1));
+
+ tree.remove(1);
+ tree.remove(2);
+
+ ASSERT_EQ(UINT64_MAX, tree.rank(2));
+ ASSERT_EQ(UINT64_MAX, tree.rank(1));
+}
+
+TEST_F(ADITreeTest, OrderInsertedStrings) {
+ std::vector<std::pair<uint32_t, std::string>> records = {
+ {1, "alpha"}, {2, "beta"},
+ {3, "foo"}, {4, "ant"}, {5, "foobar"},
+ {6, "buzz"}
+ };
+
+ adi_tree_t tree;
+ for(auto& record: records) {
+ tree.index(record.first, record.second);
+ }
+
+ std::sort(records.begin(), records.end(),
+ [](const std::pair<uint32_t, std::string>& a, const std::pair<uint32_t, std::string>& b) -> bool {
+ return a.second < b.second;
+ });
+
+ // alpha, ant, beta, buzz, foo, foobar
+ ASSERT_EQ(1, tree.rank(1));
+ ASSERT_EQ(3, tree.rank(2));
+ ASSERT_EQ(5, tree.rank(3));
+ ASSERT_EQ(2, tree.rank(4));
+ ASSERT_EQ(6, tree.rank(5));
+ ASSERT_EQ(4, tree.rank(6));
+
+ // remove "foo"
+ tree.remove(3);
+ ASSERT_EQ(5, tree.rank(5));
+
+ // remove "foobar"
+ tree.remove(5);
+ ASSERT_EQ(4, tree.rank(6));
+
+ // remove "alpha"
+ tree.remove(1);
+ ASSERT_EQ(1, tree.rank(4));
+ ASSERT_EQ(2, tree.rank(2));
+ ASSERT_EQ(3, tree.rank(6));
+}
+
+TEST_F(ADITreeTest, InsertDuplicateAndDelete) {
+ adi_tree_t tree;
+ tree.index(100, "map");
+ tree.index(101, "map");
+
+ tree.remove(100);
+ tree.remove(101);
+
+ ASSERT_EQ(UINT64_MAX, tree.rank(100));
+ ASSERT_EQ(UINT64_MAX, tree.rank(101));
+
+ ASSERT_EQ(nullptr, tree.get_root());
+}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 505bf364..01f5d4d6 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -25,9 +25,9 @@ protected:
collectionManager.load(8, 1000);
search_fields = {
- field("title", field_types::STRING, false, false, true, "en"),
- field("starring", field_types::STRING, false),
- field("cast", field_types::STRING_ARRAY, true, true),
+ field("title", field_types::STRING, false, false, true, "en", false),
+ field("starring", field_types::STRING, false, false, true, "", false),
+ field("cast", field_types::STRING_ARRAY, true, true, true, "", false),
field(".*_year", field_types::INT32, true, true),
field("location", field_types::GEOPOINT, false, true, true),
field("points", field_types::INT32, false)
@@ -90,12 +90,12 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
// we already call `collection1->get_next_seq_id` above, which is side-effecting
ASSERT_EQ(1, StringUtils::deserialize_uint32_t(next_seq_id));
ASSERT_EQ("{\"created_at\":12345,\"default_sorting_field\":\"points\",\"fallback_field_type\":\"\","
- "\"fields\":[{\"facet\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"
- "{\"facet\":false,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"
- "{\"facet\":true,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"
- "{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},"
- "{\"facet\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},"
- "{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,"
+ "\"fields\":[{\"facet\":false,\"locale\":\"en\",\"name\":\"title\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"
+ "{\"facet\":false,\"locale\":\"\",\"name\":\"starring\",\"optional\":false,\"sort\":false,\"type\":\"string\"},"
+ "{\"facet\":true,\"locale\":\"\",\"name\":\"cast\",\"optional\":true,\"sort\":false,\"type\":\"string[]\"},"
+ "{\"facet\":true,\"locale\":\"\",\"name\":\".*_year\",\"optional\":true,\"sort\":true,\"type\":\"int32\"},"
+ "{\"facet\":false,\"locale\":\"\",\"name\":\"location\",\"optional\":true,\"sort\":true,\"type\":\"geopoint\"},"
+ "{\"facet\":false,\"locale\":\"\",\"name\":\"points\",\"optional\":false,\"sort\":true,\"type\":\"int32\"}],\"id\":0,"
"\"name\":\"collection1\",\"num_memory_shards\":4,\"symbols_to_index\":[\"+\"],\"token_separators\":[\"-\"]}",
collection_meta_json);
ASSERT_EQ("1", next_collection_id);
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index c69905b9..aed04566 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -1014,3 +1014,93 @@ TEST_F(CollectionSortingTest, GeoPointArraySorting) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionSortingTest, SortByTitle) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", true),
+ field("artist", field_types::STRING, true),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 2, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"aaa", "ABCD"},
+ {"a", "ABCD"},
+ {"abcd", "ABCD"},
+ {"abdde", "ABCD"},
+ {"b", "ABCD"},
+ {"bab", "ABCD"},
+ {"baa", "ABCD"},
+ {"bcma", "ABCD"},
+ {"cdma", "ABCD"},
+ {"cc", "ABCD"},
+ {"c", "ABCD"},
+ {"cxya", "ABCD"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ std::vector<sort_by> sort_fields = {
+ sort_by("title", "ASC")
+ };
+
+ std::vector<std::string> expected_order = {
+ "a",
+ "aaa",
+ "abcd",
+ "abdde",
+ "b",
+ "baa",
+ "bab",
+ "bcma",
+ "c",
+ "cc",
+ "cdma",
+ "cxya"
+ };
+
+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 20, 1, FREQUENCY, {true}, 10).get();
+
+ ASSERT_EQ(12, results["found"].get<size_t>());
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ ASSERT_EQ(expected_order[i], results["hits"][i]["document"]["title"].get<std::string>());
+ }
+
+ // descending order
+ sort_fields = {
+ sort_by("title", "DESC")
+ };
+
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 20, 1, FREQUENCY, {true}, 10).get();
+
+ ASSERT_EQ(12, results["found"].get<size_t>());
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ ASSERT_EQ(expected_order[expected_order.size() - i - 1], results["hits"][i]["document"]["title"].get<std::string>());
+ }
+
+ // when querying for a string field with sort disabled
+ sort_fields = {
+ sort_by("artist", "DESC")
+ };
+
+ auto res_op = coll1->search("*", {}, "", {}, sort_fields, {0}, 20, 1, FREQUENCY, {true}, 10);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Could not find a field named `artist` in the schema for sorting.", res_op.error());
+
+ collectionManager.drop_collection("coll1");
+}