Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Index and search on multi-valued numeric field.

Kishore Nallan4e468fb0b9e3
diff --git a/TODO.md b/TODO.md
index 83126bd6..7955c222 100644
--- a/TODO.md
+++ b/TODO.md
@@ -20,7 +20,8 @@
- ~~Multi field search tests~~
- ~~storage key prefix should include collection name~~
- Index and search on multi-valued field
-- Proper field for ranking tokens
+- range search for art_int
+- Proper score field for ranking tokens
- Support nested fields via "."
- ~~Restore records as well on restart (like for meta)~~
- ~~drop collection should remove all records from the store~~
@@ -43,7 +44,6 @@
- Proper logging
- clean special chars before indexing
- d-ary heap?
-- range search for art_int
- test for same match score but different primary, secondary attr
**API**
diff --git a/include/collection.h b/include/collection.h
index 4047d093..00bfc38e 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -51,12 +51,17 @@ private:
void search_candidates(int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
size_t & total_results, size_t & num_found, const size_t & max_results);
- void index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const;
+ void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const;
- void index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
+ void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t, uint32_t seq_id) const;
- void index_numeric_field(const std::string &field_name, const std::string & field_type,
- art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
+ void index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
public:
Collection() = delete;
diff --git a/include/field.h b/include/field.h
index add92964..515bce34 100644
--- a/include/field.h
+++ b/include/field.h
@@ -6,7 +6,9 @@ namespace field_types {
static const std::string STRING = "STRING";
static const std::string INT32 = "INT32";
static const std::string INT64 = "INT64";
- static const std::string STRING_ARR = "STRING_ARR";
+ static const std::string STRING_ARRAY = "STRING_ARRAY";
+ static const std::string INT32_ARRAY = "INT32_ARRAY";
+ static const std::string INT64_ARRAY = "INT64_ARRAY";
}
namespace fields {
diff --git a/src/collection.cpp b/src/collection.cpp
index f5578006..06ae6465 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -52,15 +52,22 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id
for(const std::pair<std::string, field> & field_pair: schema) {
const std::string & field_name = field_pair.first;
art_tree *t = index_map.at(field_name);
+ uint32_t points = document["points"];
if(field_pair.second.type == field_types::STRING) {
const std::string & text = document[field_name];
- uint32_t points = document["points"];
index_string_field(text, points, t, seq_id);
- } else if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64) {
- index_numeric_field(field_name, field_pair.second.type, t, document, seq_id);
- } else if(field_pair.second.type == field_types::STRING_ARR) {
- index_string_array_field(field_name, t, document, seq_id);
+ } else if(field_pair.second.type == field_types::INT32) {
+ uint32_t value = document[field_name];
+ index_int32_field(value, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::INT64) {
+ uint64_t value = document[field_name];
+ index_int64_field(value, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ std::vector<std::string> strings = document[field_name];
+ index_string_array_field(strings, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::INT32_ARRAY) {
+ std::vector<std::string> strings = document[field_name];
}
}
if(rank_fields.size() > 0 && document.count(rank_fields[0])) {
@@ -72,18 +79,35 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id
}
}
-void Collection::index_numeric_field(const std::string &field_name, const std::string & field_type,
- art_tree *t, const nlohmann::json &document, uint32_t seq_id) const {
- uint32_t value = document[field_name];
+void Collection::index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
const int KEY_LEN = 8;
unsigned char key[KEY_LEN];
- if(field_type == field_types::INT32) {
- encode_int32(value, key);
- } else if(field_type == field_types::INT64) {
- encode_int64(value, key);
+ encode_int32(value, key);
+
+ uint32_t num_hits = 0;
+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
}
+ num_hits += 1;
+
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = score;
+ art_doc.offsets_len = 0;
+ art_doc.offsets = nullptr;
+
+ art_insert(t, key, KEY_LEN, &art_doc, num_hits);
+}
+
+void Collection::index_int64_field(const int64_t value, uint32_t score, art_tree *t, uint32_t seq_id) const {
+ const int KEY_LEN = 8;
+ unsigned char key[KEY_LEN];
+
+ encode_int64(value, key);
+
uint32_t num_hits = 0;
art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
if(leaf != NULL) {
@@ -94,14 +118,14 @@ void Collection::index_numeric_field(const std::string &field_name, const std::s
art_document art_doc;
art_doc.id = seq_id;
- art_doc.score = document["points"];
+ art_doc.score = score;
art_doc.offsets_len = 0;
art_doc.offsets = nullptr;
art_insert(t, key, KEY_LEN, &art_doc, num_hits);
}
-void Collection::index_string_field(const std::string & text, uint32_t score, art_tree *t, uint32_t seq_id) const {
+void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const {
std::vector<std::string> tokens;
StringUtils::tokenize(text, tokens, " ", true);
@@ -140,12 +164,24 @@ void Collection::index_string_field(const std::string & text, uint32_t score, ar
}
}
-void Collection::index_string_array_field(const std::string &field_name, art_tree *t, const nlohmann::json &document,
- uint32_t seq_id) const {
- uint32_t points = document["points"];
- std::vector<std::string> strings = document[field_name];
+void Collection::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
+ uint32_t seq_id) const {
for(const std::string & str: strings) {
- index_string_field(str, points, t, seq_id);
+ index_string_field(str, score, t, seq_id);
+ }
+}
+
+void Collection::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,
+ uint32_t seq_id) const {
+ for(const int32_t value: values) {
+ index_int32_field(value, score, t, seq_id);
+ }
+}
+
+void Collection::index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t,
+ uint32_t seq_id) const {
+ for(const int64_t value: values) {
+ index_int64_field(value, score, t, seq_id);
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index d5fbdf76..c7fade15 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -305,7 +305,7 @@ TEST_F(CollectionTest, MultipleFields) {
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),
- field("cast", field_types::STRING_ARR)};
+ field("cast", field_types::STRING_ARRAY)};
std::vector<std::string> rank_fields = {"points"};
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");