Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Index and search on multi-valued numeric field.
Kishore Nallan4e468fb0b9e3
diff --git a/TODO.md b/TODO.mdindex 83126bd6..7955c222 100644--- a/TODO.md+++ b/TODO.md@@ -20,7 +20,8 @@- ~~Multi field search tests~~- ~~storage key prefix should include collection name~~- Index and search on multi-valued field-- Proper field for ranking tokens+- range search for art_int+- Proper score field for ranking tokens- Support nested fields via "."- ~~Restore records as well on restart (like for meta)~~- ~~drop collection should remove all records from the store~~@@ -43,7 +44,6 @@- Proper logging- clean special chars before indexing- d-ary heap?-- range search for art_int- test for same match score but different primary, secondary attr**API**diff --git a/include/collection.h b/include/collection.hindex 4047d093..00bfc38e 100644--- a/include/collection.h+++ b/include/collection.h@@ -51,12 +51,17 @@ private:void search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,size_t & total_results, size_t & num_found, const size_t & max_results);- void index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const;+ void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const;- void index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;+ void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t, uint32_t seq_id) const;- void index_numeric_field(const std::string &field_name, const std::string & field_type,- art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;+ void index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const;++ void index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const;++ void index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;++ void index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;public:Collection() = delete;diff --git a/include/field.h b/include/field.hindex add92964..515bce34 100644--- a/include/field.h+++ b/include/field.h@@ -6,7 +6,9 @@ namespace field_types {static const std::string STRING = "STRING";static const std::string INT32 = "INT32";static const std::string INT64 = "INT64";- static const std::string STRING_ARR = "STRING_ARR";+ static const std::string STRING_ARRAY = "STRING_ARRAY";+ static const std::string INT32_ARRAY = "INT32_ARRAY";+ static const std::string INT64_ARRAY = "INT64_ARRAY";}namespace fields {diff --git a/src/collection.cpp b/src/collection.cppindex f5578006..06ae6465 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -52,15 +52,22 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_idfor(const std::pair<std::string, field> & field_pair: schema) {const std::string & field_name = field_pair.first;art_tree *t = index_map.at(field_name);+ uint32_t points = document["points"];if(field_pair.second.type == field_types::STRING) {const std::string & text = document[field_name];- uint32_t points = document["points"];index_string_field(text, points, t, seq_id);- } else if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64) {- index_numeric_field(field_name, field_pair.second.type, t, document, seq_id);- } else if(field_pair.second.type == field_types::STRING_ARR) {- index_string_array_field(field_name, t, document, seq_id);+ } else if(field_pair.second.type == field_types::INT32) {+ uint32_t value = document[field_name];+ index_int32_field(value, points, t, seq_id);+ } else if(field_pair.second.type == field_types::INT64) {+ uint64_t value = document[field_name];+ index_int64_field(value, points, t, seq_id);+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {+ std::vector<std::string> strings = document[field_name];+ index_string_array_field(strings, points, t, seq_id);+ } else if(field_pair.second.type == field_types::INT32_ARRAY) {+ std::vector<std::string> strings = document[field_name];}}if(rank_fields.size() > 0 && document.count(rank_fields[0])) {@@ -72,18 +79,35 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id}}-void Collection::index_numeric_field(const std::string &field_name, const std::string & field_type,- art_tree *t, const nlohmann::json &document, uint32_t seq_id) const {- uint32_t value = document[field_name];+void Collection::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {const int KEY_LEN = 8;unsigned char key[KEY_LEN];- if(field_type == field_types::INT32) {- encode_int32(value, key);- } else if(field_type == field_types::INT64) {- encode_int64(value, key);+ encode_int32(value, key);++ uint32_t num_hits = 0;+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);+ if(leaf != NULL) {+ num_hits = leaf->values->ids.getLength();}+ num_hits += 1;++ art_document art_doc;+ art_doc.id = seq_id;+ art_doc.score = score;+ art_doc.offsets_len = 0;+ art_doc.offsets = nullptr;++ art_insert(t, key, KEY_LEN, &art_doc, num_hits);+}++void Collection::index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {+ const int KEY_LEN = 8;+ unsigned char key[KEY_LEN];++ encode_int64(value, key);+uint32_t num_hits = 0;art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);if(leaf != NULL) {@@ -94,14 +118,14 @@ void Collection::index_numeric_field(const std::string &field_name, const std::sart_document art_doc;art_doc.id = seq_id;- art_doc.score = document["points"];+ art_doc.score = score;art_doc.offsets_len = 0;art_doc.offsets = nullptr;art_insert(t, key, KEY_LEN, &art_doc, num_hits);}-void Collection::index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const {+void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const {std::vector<std::string> tokens;StringUtils::tokenize(text, tokens, " ", true);@@ -140,12 +164,24 @@ void Collection::index_string_field(const std::string & text, uint32_t score, ar}}-void Collection::index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,- uint32_t seq_id) const {- uint32_t points = document["points"];- std::vector<std::string> strings = document[field_name];+void Collection::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,+ uint32_t seq_id) const {for(const std::string & str: strings) {- index_string_field(str, points, t, seq_id);+ index_string_field(str, score, t, seq_id);+ }+}++void Collection::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,+ uint32_t seq_id) const {+ for(const int32_t value: values) {+ index_int32_field(value, score, t, seq_id);+ }+}++void Collection::index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t,+ uint32_t seq_id) const {+ for(const int64_t value: values) {+ index_int64_field(value, score, t, seq_id);}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex d5fbdf76..c7fade15 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -305,7 +305,7 @@ TEST_F(CollectionTest, MultipleFields) {std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),- field("cast", field_types::STRING_ARR)};+ field("cast", field_types::STRING_ARRAY)};std::vector<std::string> rank_fields = {"points"};coll_mul_fields = collectionManager.get_collection("coll_mul_fields");