Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Speed up numerical filtering
Jason Boscoceaee0fb8054
diff --git a/include/num_tree.h b/include/num_tree.hindex 6f99f822..bdcb7e91 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -52,16 +52,23 @@ public:it++;}+ std::vector<uint32_t> consolidated_ids;while(it != int64map.end()) {- uint32_t *out = nullptr;- uint32_t *val_ids = it->second->uncompress();- ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),- *ids, ids_len, &out);- delete [] val_ids;- delete [] *ids;- *ids = out;+ for(size_t i = 0; i < it->second->getLength(); i++) {+ consolidated_ids.push_back(it->second->at(i));+ }it++;}++ std::sort(consolidated_ids.begin(), consolidated_ids.end());++ uint32_t *out = nullptr;+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),+ *ids, ids_len, &out);++ delete [] *ids;+ *ids = out;+} else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {auto max_iter = int64map.lower_bound(value); // iter values will be >= value@@ -83,23 +90,29 @@ public:}}- auto iter = int64map.begin();+ std::vector<uint32_t> consolidated_ids;+ auto it = int64map.begin();while(true) {- uint32_t* out = nullptr;- uint32_t* val_ids = iter->second->uncompress();- ids_len = ArrayUtils::or_scalar(val_ids, iter->second->getLength(),- *ids, ids_len, &out);- delete[] val_ids;- delete[] *ids;- *ids = out;+ for(size_t i = 0; i < it->second->getLength(); i++) {+ consolidated_ids.push_back(it->second->at(i));+ }- if(iter == max_iter) {+ if(it == max_iter) {break;}- iter++;+ it++;}++ std::sort(consolidated_ids.begin(), consolidated_ids.end());++ uint32_t *out = nullptr;+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),+ *ids, ids_len, &out);++ delete [] *ids;+ *ids = out;}}diff --git a/src/index.cpp b/src/index.cppindex 29bfd318..5684965e 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1692,8 +1692,8 @@ void Index::search_field(const uint8_t & field_id,const bool prefix_search = prefix && (token_index == search_tokens.size()-1);const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;- // If this is a prefix search, look for more candidates and do a union of those document IDs- const int max_candidates = prefix_search ? 10 : 3;+ // need less candidates for filtered searches since we already only pick tokens with results+ const int max_candidates = (filter_ids_length == 0) ? 10 : 3;art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,filter_ids, filter_ids_length, leaves);