Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix edge case involving filtering of negative values.

Kishore Nallan60a5cff29136
diff --git a/include/num_tree.h b/include/num_tree.h
index 8f42c1cd..39276cf0 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -4,6 +4,7 @@
#include "sparsepp.h"
#include "sorted_array.h"
#include "array_utils.h"
+#include "art.h"
class num_tree_t {
private:
@@ -17,150 +18,15 @@ public:
}
}
- void insert(int64_t value, uint32_t id) {
- if(int64map.count(value) == 0) {
- int64map.emplace(value, new sorted_array);
- }
-
- if(!int64map[value]->contains(id)) {
- int64map[value]->append(id);
- }
- }
-
- void range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len) {
- if(int64map.empty()) {
- return ;
- }
-
- auto it_start = int64map.lower_bound(start); // iter values will be >= start
-
- std::vector<uint32_t> consolidated_ids;
- while(it_start != int64map.end() && it_start->first <= end) {
- for(size_t i = 0; i < it_start->second->getLength(); i++) {
- consolidated_ids.push_back(it_start->second->at(i));
- }
-
- it_start++;
- }
-
- std::sort(consolidated_ids.begin(), consolidated_ids.end());
-
- uint32_t *out = nullptr;
- ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
- *ids, ids_len, &out);
-
- delete [] *ids;
- *ids = out;
- }
-
- size_t get(int64_t value, std::vector<uint32_t>& geo_result_ids) {
- const auto& it = int64map.find(value);
- if(it == int64map.end()) {
- return 0;
- }
-
- uint32_t* ids = it->second->uncompress();
- for(size_t i = 0; i < it->second->getLength(); i++) {
- geo_result_ids.push_back(ids[i]);
- }
-
- delete [] ids;
-
- return it->second->getLength();
- }
-
- void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len) {
- if(int64map.empty()) {
- return ;
- }
-
- if(comparator == EQUALS) {
- const auto& it = int64map.find(value);
- if(it != int64map.end()) {
- uint32_t *out = nullptr;
- uint32_t* val_ids = it->second->uncompress();
- ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),
- *ids, ids_len, &out);
- delete[] *ids;
- *ids = out;
- delete[] val_ids;
- }
- } else if(comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {
- // iter entries will be >= value, or end() if all entries are before value
- auto iter_ge_value = int64map.lower_bound(value);
-
- if(iter_ge_value == int64map.end()) {
- return ;
- }
-
- if(comparator == GREATER_THAN && iter_ge_value->first == value) {
- iter_ge_value++;
- }
-
- std::vector<uint32_t> consolidated_ids;
- while(iter_ge_value != int64map.end()) {
- for(size_t i = 0; i < iter_ge_value->second->getLength(); i++) {
- consolidated_ids.push_back(iter_ge_value->second->at(i));
- }
- iter_ge_value++;
- }
-
- std::sort(consolidated_ids.begin(), consolidated_ids.end());
- consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());
-
- uint32_t *out = nullptr;
- ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
- *ids, ids_len, &out);
-
- delete [] *ids;
- *ids = out;
+ void insert(int64_t value, uint32_t id);
- } else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {
- // iter entries will be >= value, or end() if all entries are before value
- auto iter_ge_value = int64map.lower_bound(value);
+ void range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len);
- std::vector<uint32_t> consolidated_ids;
- auto it = int64map.begin();
+ size_t get(int64_t value, std::vector<uint32_t>& geo_result_ids);
- while(it != iter_ge_value) {
- for(size_t i = 0; i < it->second->getLength(); i++) {
- consolidated_ids.push_back(it->second->at(i));
- }
- it++;
- }
+ void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len);
- // for LESS_THAN_EQUALS, check if last iter entry is equal to value
- if(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {
- for(size_t i = 0; i < it->second->getLength(); i++) {
- consolidated_ids.push_back(it->second->at(i));
- }
- }
-
- std::sort(consolidated_ids.begin(), consolidated_ids.end());
- consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());
-
- uint32_t *out = nullptr;
- ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
- *ids, ids_len, &out);
-
- delete [] *ids;
- *ids = out;
- }
- }
-
- void remove(uint64_t value, uint32_t id) {
- if(int64map.count(value) != 0) {
- sorted_array* arr = int64map[value];
- arr->remove_value(id);
- if(arr->getLength() == 0) {
- delete arr;
- int64map.erase(value);
- }
- }
- }
-
- size_t size() {
- return int64map.size();
- }
+ void remove(uint64_t value, uint32_t id);
+ size_t size();
};
\ No newline at end of file
diff --git a/src/index.cpp b/src/index.cpp
index f3320853..203ace2c 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -181,15 +181,15 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
else if(field_pair.second.type == field_types::INT32) {
auto num_tree = numerical_index.at(field_name);
- uint32_t value = document[field_name];
+ int32_t value = document[field_name].get<int32_t>();
num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::INT64) {
auto num_tree = numerical_index.at(field_name);
- uint64_t value = document[field_name];
+ int64_t value = document[field_name].get<int64_t>();
num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::FLOAT) {
auto num_tree = numerical_index.at(field_name);
- float fvalue = document[field_name];
+ float fvalue = document[field_name].get<float>();
int64_t value = float_to_in64_t(fvalue);
num_tree->insert(value, seq_id);
} else if(field_pair.second.type == field_types::BOOL) {
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
new file mode 100644
index 00000000..67bf512b
--- /dev/null
+++ b/src/num_tree.cpp
@@ -0,0 +1,147 @@
+#include "num_tree.h"
+
+void num_tree_t::insert(int64_t value, uint32_t id) {
+ if (int64map.count(value) == 0) {
+ int64map.emplace(value, new sorted_array);
+ }
+
+ if (!int64map[value]->contains(id)) {
+ int64map[value]->append(id);
+ }
+}
+
+void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len) {
+ if(int64map.empty()) {
+ return ;
+ }
+
+ auto it_start = int64map.lower_bound(start); // iter values will be >= start
+
+ std::vector<uint32_t> consolidated_ids;
+ while(it_start != int64map.end() && it_start->first <= end) {
+ for(size_t i = 0; i < it_start->second->getLength(); i++) {
+ consolidated_ids.push_back(it_start->second->at(i));
+ }
+
+ it_start++;
+ }
+
+ std::sort(consolidated_ids.begin(), consolidated_ids.end());
+
+ uint32_t *out = nullptr;
+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
+ *ids, ids_len, &out);
+
+ delete [] *ids;
+ *ids = out;
+}
+
+size_t num_tree_t::get(int64_t value, std::vector<uint32_t>& geo_result_ids) {
+ const auto& it = int64map.find(value);
+ if(it == int64map.end()) {
+ return 0;
+ }
+
+ uint32_t* ids = it->second->uncompress();
+ for(size_t i = 0; i < it->second->getLength(); i++) {
+ geo_result_ids.push_back(ids[i]);
+ }
+
+ delete [] ids;
+
+ return it->second->getLength();
+}
+
+void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len) {
+ if(int64map.empty()) {
+ return ;
+ }
+
+ if(comparator == EQUALS) {
+ const auto& it = int64map.find(value);
+ if(it != int64map.end()) {
+ uint32_t *out = nullptr;
+ uint32_t* val_ids = it->second->uncompress();
+ ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),
+ *ids, ids_len, &out);
+ delete[] *ids;
+ *ids = out;
+ delete[] val_ids;
+ }
+ } else if(comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {
+ // iter entries will be >= value, or end() if all entries are before value
+ auto iter_ge_value = int64map.lower_bound(value);
+
+ if(iter_ge_value == int64map.end()) {
+ return ;
+ }
+
+ if(comparator == GREATER_THAN && iter_ge_value->first == value) {
+ iter_ge_value++;
+ }
+
+ std::vector<uint32_t> consolidated_ids;
+ while(iter_ge_value != int64map.end()) {
+ for(size_t i = 0; i < iter_ge_value->second->getLength(); i++) {
+ consolidated_ids.push_back(iter_ge_value->second->at(i));
+ }
+ iter_ge_value++;
+ }
+
+ std::sort(consolidated_ids.begin(), consolidated_ids.end());
+ consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());
+
+ uint32_t *out = nullptr;
+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
+ *ids, ids_len, &out);
+
+ delete [] *ids;
+ *ids = out;
+
+ } else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {
+ // iter entries will be >= value, or end() if all entries are before value
+ auto iter_ge_value = int64map.lower_bound(value);
+
+ std::vector<uint32_t> consolidated_ids;
+ auto it = int64map.begin();
+
+ while(it != iter_ge_value) {
+ for(size_t i = 0; i < it->second->getLength(); i++) {
+ consolidated_ids.push_back(it->second->at(i));
+ }
+ it++;
+ }
+
+ // for LESS_THAN_EQUALS, check if last iter entry is equal to value
+ if(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {
+ for(size_t i = 0; i < it->second->getLength(); i++) {
+ consolidated_ids.push_back(it->second->at(i));
+ }
+ }
+
+ std::sort(consolidated_ids.begin(), consolidated_ids.end());
+ consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());
+
+ uint32_t *out = nullptr;
+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
+ *ids, ids_len, &out);
+
+ delete [] *ids;
+ *ids = out;
+ }
+}
+
+void num_tree_t::remove(uint64_t value, uint32_t id) {
+ if(int64map.count(value) != 0) {
+ sorted_array* arr = int64map[value];
+ arr->remove_value(id);
+ if(arr->getLength() == 0) {
+ delete arr;
+ int64map.erase(value);
+ }
+ }
+}
+
+size_t num_tree_t::size() {
+ return int64map.size();
+}
\ No newline at end of file
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index ccbc56f1..f27e2e7e 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -814,6 +814,62 @@ TEST_F(CollectionFilteringTest, FilterOnFloatFields) {
collectionManager.drop_collection("coll_array_fields");
}
+TEST_F(CollectionFilteringTest, FilterOnNegativeNumericalFields) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("int32_field", field_types::INT32, false),
+ field("int64_field", field_types::INT64, false),
+ field("float_field", field_types::FLOAT, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "int32_field").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Title 1", "-100", "5000000", "-10.45124"},
+ {"Title 2", "100", "-1000000", "0.45124"},
+ {"Title 3", "-200", "3000000", "-0.45124"},
+ {"Title 4", "150", "10000000", "1.45124"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["int32_field"] = std::stoi(records[i][1]);
+ doc["int64_field"] = std::stoll(records[i][2]);
+ doc["float_field"] = std::stof(records[i][3]);
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("*", {}, "int32_field:<0", {}, {}, 0, 10, 1, FREQUENCY, true, 10).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+
+ results = coll1->search("*", {}, "int64_field:<0", {}, {}, 0, 10, 1, FREQUENCY, true, 10).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ results = coll1->search("*", {}, "float_field:<0", {}, {sort_by("float_field", "desc")}, 0, 10, 1, FREQUENCY,
+ true, 10).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionFilteringTest, ComparatorsOnMultiValuedNumericalField) {
Collection *coll_array_fields;
diff --git a/test/num_tree_test.cpp b/test/num_tree_test.cpp
new file mode 100644
index 00000000..b34afb17
--- /dev/null
+++ b/test/num_tree_test.cpp
@@ -0,0 +1,44 @@
+#include <gtest/gtest.h>
+#include <art.h>
+#include "num_tree.h"
+
+TEST(NumTreeTest, Searches) {
+ num_tree_t tree;
+ tree.insert(-1200, 0);
+ tree.insert(-1750, 1);
+ tree.insert(0, 2);
+ tree.insert(100, 3);
+ tree.insert(2000, 4);
+
+ tree.insert(-1200, 5);
+ tree.insert(100, 6);
+
+ uint32_t* ids;
+ size_t ids_len;
+
+ tree.search(NUM_COMPARATOR::EQUALS, -1750, &ids, ids_len);
+ ASSERT_EQ(1, ids_len);
+ ASSERT_EQ(1, ids[0]);
+ delete [] ids;
+ ids = nullptr;
+
+ tree.search(NUM_COMPARATOR::GREATER_THAN_EQUALS, -1200, &ids, ids_len);
+ ASSERT_EQ(6, ids_len);
+ delete [] ids;
+ ids = nullptr;
+
+ tree.search(NUM_COMPARATOR::GREATER_THAN, -1200, &ids, ids_len);
+ ASSERT_EQ(4, ids_len);
+ delete [] ids;
+ ids = nullptr;
+
+ tree.search(NUM_COMPARATOR::LESS_THAN_EQUALS, 100, &ids, ids_len);
+ ASSERT_EQ(6, ids_len);
+ delete [] ids;
+ ids = nullptr;
+
+ tree.search(NUM_COMPARATOR::LESS_THAN, 100, &ids, ids_len);
+ ASSERT_EQ(4, ids_len);
+ delete [] ids;
+ ids = nullptr;
+}