Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Rewrite facet implementation.
Kishore Nallana5cd45e36297
diff --git a/TODO.md b/TODO.mdindex 13c276c5..197da377 100644--- a/TODO.md+++ b/TODO.md@@ -34,6 +34,7 @@- ~~Found count is wrong~~- ~~Filter query in the API~~- Handle store-get() not finding a key+- Facet limit- Fix API response codes- Test for search without any sort_by given- Test for asc/desc upper/lower casingdiff --git a/include/collection.h b/include/collection.hindex 24c39ce2..b43d8941 100644--- a/include/collection.h+++ b/include/collection.h@@ -10,6 +10,33 @@#include <field.h>#include <option.h>+struct facet_value {+ // use string to int mapping for saving memory+ spp::sparse_hash_map<std::string, uint32_t> value_index;+ spp::sparse_hash_map<uint32_t, std::string> index_value;++ spp::sparse_hash_map<uint32_t, std::vector<uint32_t>*> doc_values;++ uint32_t get_value_index(const std::string & value) {+ if(value_index.count(value) != 0) {+ return value_index[value];+ }++ uint32_t new_index = value_index.size();+ value_index.emplace(value, new_index);+ index_value.emplace(new_index, value);+ return new_index;+ }++ void index_values(uint32_t doc_seq_id, const std::vector<std::string> & values) {+ std::vector<uint32_t>* value_vec = new std::vector<uint32_t>(values.size());+ for(auto i = 0; i < values.size(); i++) {+ (*value_vec)[i] = get_value_index(values[i]);+ }+ doc_values.emplace(doc_seq_id, value_vec);+ }+};+class Collection {private:std::string name;@@ -29,7 +56,7 @@ private:spp::sparse_hash_map<std::string, art_tree*> search_index;- spp::sparse_hash_map<std::string, art_tree*> facet_index;+ spp::sparse_hash_map<std::string, facet_value> facet_index;spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;diff --git a/src/collection.cpp b/src/collection.cppindex 7b3e2a21..90e87ffe 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -21,9 +21,8 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con}for(const field& field: facet_fields) {- art_tree *t = new art_tree;- art_tree_init(t);- facet_index.emplace(field.name, t);+ facet_value fvalue;+ facet_index.emplace(field.name, fvalue);facet_schema.emplace(field.name, field);}@@ -41,9 +40,10 @@ Collection::~Collection() {}for(std::pair<std::string, field> name_field: facet_schema) {- art_tree *t = facet_index.at(name_field.first);- art_tree_destroy(t);- t = nullptr;+ facet_value & fvalue = facet_index.at(name_field.first);+ for(auto doc_value: fvalue.doc_values) {+ delete doc_value.second;+ }}for(std::pair<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> name_map: sort_index) {@@ -176,13 +176,13 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin"but is not found in the document.");}- art_tree *t = facet_index.at(field_name);+ facet_value & fvalue = facet_index.at(field_name);if(field_pair.second.type == field_types::STRING) {if(!document[field_name].is_string()) {return Option<>(400, "Facet field `" + field_name + "` must be a STRING.");}- const std::string & text = document[field_name];- index_string_field(text, points, t, seq_id, true);+ const std::string & value = document[field_name];+ fvalue.index_values(seq_id, { value });} else if(field_pair.second.type == field_types::STRING_ARRAY) {if(!document[field_name].is_array()) {return Option<>(400, "Facet field `" + field_name + "` must be a STRING_ARRAY.");@@ -192,8 +192,8 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinreturn Option<>(400, "Facet field `" + field_name + "` must be a STRING_ARRAY.");}- std::vector<std::string> strings = document[field_name];- index_string_array_field(strings, points, t, seq_id, true);+ const std::vector<std::string> & values = document[field_name];+ fvalue.index_values(seq_id, values);}}@@ -330,28 +330,17 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, sifor(auto & a_facet: facets) {// assumed that facet fields have already been validated upstreamconst field & facet_field = facet_schema.at(a_facet.field_name);-- // loop through the field, get all keys and intersect those ids with result ids- if(facet_index.count(facet_field.name) != 0) {- art_tree *t = facet_index.at(facet_field.name);- std::vector<art_leaf *> leaves;-- art_topk_iter(t->root, MAX_SCORE, 10, leaves);-- for(const art_leaf* leaf: leaves) {- const uint32_t* facet_ids = leaf->values->ids.uncompress();- size_t facet_ids_size = leaf->values->ids.getLength();-- uint32_t* facet_results = new uint32_t[std::min(facet_ids_size, results_size)];- const size_t facet_results_size = ArrayUtils::and_scalar(result_ids, results_size,- facet_ids, facet_ids_size,- facet_results);-- const std::string facet_value((const char *)leaf->key, leaf->key_len-1); // drop trailing null- a_facet.result_map.insert(std::pair<std::string, size_t>(facet_value, facet_results_size));-- delete [] facet_ids;- delete [] facet_results;+ const facet_value & fvalue = facet_index.at(facet_field.name);++ for(auto i = 0; i < results_size; i++) {+ uint32_t doc_seq_id = result_ids[i];+ if(fvalue.doc_values.count(doc_seq_id) != 0) {+ // for every result document, get the values associated and increment counter+ std::vector<uint32_t>* value_indices = fvalue.doc_values.at(doc_seq_id);+ for(auto j = 0; j < value_indices->size(); j++) {+ const std::string & facet_value = fvalue.index_value.at(value_indices->at(j));+ a_facet.result_map[facet_value] += 1;+ }}}}