Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix edge case involving filtering of negative values.
Kishore Nallan60a5cff29136
diff --git a/include/num_tree.h b/include/num_tree.hindex 8f42c1cd..39276cf0 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -4,6 +4,7 @@#include "sparsepp.h"#include "sorted_array.h"#include "array_utils.h"+#include "art.h"class num_tree_t {private:@@ -17,150 +18,15 @@ public:}}- void insert(int64_t value, uint32_t id) {- if(int64map.count(value) == 0) {- int64map.emplace(value, new sorted_array);- }-- if(!int64map[value]->contains(id)) {- int64map[value]->append(id);- }- }-- void range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len) {- if(int64map.empty()) {- return ;- }-- auto it_start = int64map.lower_bound(start); // iter values will be >= start-- std::vector<uint32_t> consolidated_ids;- while(it_start != int64map.end() && it_start->first <= end) {- for(size_t i = 0; i < it_start->second->getLength(); i++) {- consolidated_ids.push_back(it_start->second->at(i));- }-- it_start++;- }-- std::sort(consolidated_ids.begin(), consolidated_ids.end());-- uint32_t *out = nullptr;- ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),- *ids, ids_len, &out);-- delete [] *ids;- *ids = out;- }-- size_t get(int64_t value, std::vector<uint32_t>& geo_result_ids) {- const auto& it = int64map.find(value);- if(it == int64map.end()) {- return 0;- }-- uint32_t* ids = it->second->uncompress();- for(size_t i = 0; i < it->second->getLength(); i++) {- geo_result_ids.push_back(ids[i]);- }-- delete [] ids;-- return it->second->getLength();- }-- void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len) {- if(int64map.empty()) {- return ;- }-- if(comparator == EQUALS) {- const auto& it = int64map.find(value);- if(it != int64map.end()) {- uint32_t *out = nullptr;- uint32_t* val_ids = it->second->uncompress();- ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),- *ids, ids_len, &out);- delete[] *ids;- *ids = out;- delete[] val_ids;- }- } else if(comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {- // iter entries will be >= value, or end() if all entries are before value- auto iter_ge_value = int64map.lower_bound(value);-- if(iter_ge_value == int64map.end()) {- return ;- }-- if(comparator == GREATER_THAN && iter_ge_value->first == value) {- iter_ge_value++;- }-- std::vector<uint32_t> consolidated_ids;- while(iter_ge_value != int64map.end()) {- for(size_t i = 0; i < iter_ge_value->second->getLength(); i++) {- consolidated_ids.push_back(iter_ge_value->second->at(i));- }- iter_ge_value++;- }-- std::sort(consolidated_ids.begin(), consolidated_ids.end());- consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());-- uint32_t *out = nullptr;- ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),- *ids, ids_len, &out);-- delete [] *ids;- *ids = out;+ void insert(int64_t value, uint32_t id);- } else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {- // iter entries will be >= value, or end() if all entries are before value- auto iter_ge_value = int64map.lower_bound(value);+ void range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len);- std::vector<uint32_t> consolidated_ids;- auto it = int64map.begin();+ size_t get(int64_t value, std::vector<uint32_t>& geo_result_ids);- while(it != iter_ge_value) {- for(size_t i = 0; i < it->second->getLength(); i++) {- consolidated_ids.push_back(it->second->at(i));- }- it++;- }+ void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len);- // for LESS_THAN_EQUALS, check if last iter entry is equal to value- if(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {- for(size_t i = 0; i < it->second->getLength(); i++) {- consolidated_ids.push_back(it->second->at(i));- }- }-- std::sort(consolidated_ids.begin(), consolidated_ids.end());- consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());-- uint32_t *out = nullptr;- ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),- *ids, ids_len, &out);-- delete [] *ids;- *ids = out;- }- }-- void remove(uint64_t value, uint32_t id) {- if(int64map.count(value) != 0) {- sorted_array* arr = int64map[value];- arr->remove_value(id);- if(arr->getLength() == 0) {- delete arr;- int64map.erase(value);- }- }- }-- size_t size() {- return int64map.size();- }+ void remove(uint64_t value, uint32_t id);+ size_t size();};\ No newline at end of filediff --git a/src/index.cpp b/src/index.cppindex f3320853..203ace2c 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -181,15 +181,15 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_telse if(field_pair.second.type == field_types::INT32) {auto num_tree = numerical_index.at(field_name);- uint32_t value = document[field_name];+ int32_t value = document[field_name].get<int32_t>();num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::INT64) {auto num_tree = numerical_index.at(field_name);- uint64_t value = document[field_name];+ int64_t value = document[field_name].get<int64_t>();num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::FLOAT) {auto num_tree = numerical_index.at(field_name);- float fvalue = document[field_name];+ float fvalue = document[field_name].get<float>();int64_t value = float_to_in64_t(fvalue);num_tree->insert(value, seq_id);} else if(field_pair.second.type == field_types::BOOL) {diff --git a/src/num_tree.cpp b/src/num_tree.cppnew file mode 100644index 00000000..67bf512b--- /dev/null+++ b/src/num_tree.cpp@@ -0,0 +1,147 @@+#include "num_tree.h"++void num_tree_t::insert(int64_t value, uint32_t id) {+ if (int64map.count(value) == 0) {+ int64map.emplace(value, new sorted_array);+ }++ if (!int64map[value]->contains(id)) {+ int64map[value]->append(id);+ }+}++void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len) {+ if(int64map.empty()) {+ return ;+ }++ auto it_start = int64map.lower_bound(start); // iter values will be >= start++ std::vector<uint32_t> consolidated_ids;+ while(it_start != int64map.end() && it_start->first <= end) {+ for(size_t i = 0; i < it_start->second->getLength(); i++) {+ consolidated_ids.push_back(it_start->second->at(i));+ }++ it_start++;+ }++ std::sort(consolidated_ids.begin(), consolidated_ids.end());++ uint32_t *out = nullptr;+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),+ *ids, ids_len, &out);++ delete [] *ids;+ *ids = out;+}++size_t num_tree_t::get(int64_t value, std::vector<uint32_t>& geo_result_ids) {+ const auto& it = int64map.find(value);+ if(it == int64map.end()) {+ return 0;+ }++ uint32_t* ids = it->second->uncompress();+ for(size_t i = 0; i < it->second->getLength(); i++) {+ geo_result_ids.push_back(ids[i]);+ }++ delete [] ids;++ return it->second->getLength();+}++void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len) {+ if(int64map.empty()) {+ return ;+ }++ if(comparator == EQUALS) {+ const auto& it = int64map.find(value);+ if(it != int64map.end()) {+ uint32_t *out = nullptr;+ uint32_t* val_ids = it->second->uncompress();+ ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),+ *ids, ids_len, &out);+ delete[] *ids;+ *ids = out;+ delete[] val_ids;+ }+ } else if(comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {+ // iter entries will be >= value, or end() if all entries are before value+ auto iter_ge_value = int64map.lower_bound(value);++ if(iter_ge_value == int64map.end()) {+ return ;+ }++ if(comparator == GREATER_THAN && iter_ge_value->first == value) {+ iter_ge_value++;+ }++ std::vector<uint32_t> consolidated_ids;+ while(iter_ge_value != int64map.end()) {+ for(size_t i = 0; i < iter_ge_value->second->getLength(); i++) {+ consolidated_ids.push_back(iter_ge_value->second->at(i));+ }+ iter_ge_value++;+ }++ std::sort(consolidated_ids.begin(), consolidated_ids.end());+ consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());++ uint32_t *out = nullptr;+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),+ *ids, ids_len, &out);++ delete [] *ids;+ *ids = out;++ } else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {+ // iter entries will be >= value, or end() if all entries are before value+ auto iter_ge_value = int64map.lower_bound(value);++ std::vector<uint32_t> consolidated_ids;+ auto it = int64map.begin();++ while(it != iter_ge_value) {+ for(size_t i = 0; i < it->second->getLength(); i++) {+ consolidated_ids.push_back(it->second->at(i));+ }+ it++;+ }++ // for LESS_THAN_EQUALS, check if last iter entry is equal to value+ if(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {+ for(size_t i = 0; i < it->second->getLength(); i++) {+ consolidated_ids.push_back(it->second->at(i));+ }+ }++ std::sort(consolidated_ids.begin(), consolidated_ids.end());+ consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());++ uint32_t *out = nullptr;+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),+ *ids, ids_len, &out);++ delete [] *ids;+ *ids = out;+ }+}++void num_tree_t::remove(uint64_t value, uint32_t id) {+ if(int64map.count(value) != 0) {+ sorted_array* arr = int64map[value];+ arr->remove_value(id);+ if(arr->getLength() == 0) {+ delete arr;+ int64map.erase(value);+ }+ }+}++size_t num_tree_t::size() {+ return int64map.size();+}\ No newline at end of filediff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex ccbc56f1..f27e2e7e 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -814,6 +814,62 @@ TEST_F(CollectionFilteringTest, FilterOnFloatFields) {collectionManager.drop_collection("coll_array_fields");}+TEST_F(CollectionFilteringTest, FilterOnNegativeNumericalFields) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("int32_field", field_types::INT32, false),+ field("int64_field", field_types::INT64, false),+ field("float_field", field_types::FLOAT, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "int32_field").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Title 1", "-100", "5000000", "-10.45124"},+ {"Title 2", "100", "-1000000", "0.45124"},+ {"Title 3", "-200", "3000000", "-0.45124"},+ {"Title 4", "150", "10000000", "1.45124"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["int32_field"] = std::stoi(records[i][1]);+ doc["int64_field"] = std::stoll(records[i][2]);+ doc["float_field"] = std::stof(records[i][3]);++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("*", {}, "int32_field:<0", {}, {}, 0, 10, 1, FREQUENCY, true, 10).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());++ results = coll1->search("*", {}, "int64_field:<0", {}, {}, 0, 10, 1, FREQUENCY, true, 10).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ results = coll1->search("*", {}, "float_field:<0", {}, {sort_by("float_field", "desc")}, 0, 10, 1, FREQUENCY,+ true, 10).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionFilteringTest, ComparatorsOnMultiValuedNumericalField) {Collection *coll_array_fields;diff --git a/test/num_tree_test.cpp b/test/num_tree_test.cppnew file mode 100644index 00000000..b34afb17--- /dev/null+++ b/test/num_tree_test.cpp@@ -0,0 +1,44 @@+#include <gtest/gtest.h>+#include <art.h>+#include "num_tree.h"++TEST(NumTreeTest, Searches) {+ num_tree_t tree;+ tree.insert(-1200, 0);+ tree.insert(-1750, 1);+ tree.insert(0, 2);+ tree.insert(100, 3);+ tree.insert(2000, 4);++ tree.insert(-1200, 5);+ tree.insert(100, 6);++ uint32_t* ids;+ size_t ids_len;++ tree.search(NUM_COMPARATOR::EQUALS, -1750, &ids, ids_len);+ ASSERT_EQ(1, ids_len);+ ASSERT_EQ(1, ids[0]);+ delete [] ids;+ ids = nullptr;++ tree.search(NUM_COMPARATOR::GREATER_THAN_EQUALS, -1200, &ids, ids_len);+ ASSERT_EQ(6, ids_len);+ delete [] ids;+ ids = nullptr;++ tree.search(NUM_COMPARATOR::GREATER_THAN, -1200, &ids, ids_len);+ ASSERT_EQ(4, ids_len);+ delete [] ids;+ ids = nullptr;++ tree.search(NUM_COMPARATOR::LESS_THAN_EQUALS, 100, &ids, ids_len);+ ASSERT_EQ(6, ids_len);+ delete [] ids;+ ids = nullptr;++ tree.search(NUM_COMPARATOR::LESS_THAN, 100, &ids, ids_len);+ ASSERT_EQ(4, ids_len);+ delete [] ids;+ ids = nullptr;+}