Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Speed up numerical filtering

Jason Boscoceaee0fb8054
diff --git a/include/num_tree.h b/include/num_tree.h
index 6f99f822..bdcb7e91 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -52,16 +52,23 @@ public:
it++;
}
+ std::vector<uint32_t> consolidated_ids;
while(it != int64map.end()) {
- uint32_t *out = nullptr;
- uint32_t *val_ids = it->second->uncompress();
- ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),
- *ids, ids_len, &out);
- delete [] val_ids;
- delete [] *ids;
- *ids = out;
+ for(size_t i = 0; i < it->second->getLength(); i++) {
+ consolidated_ids.push_back(it->second->at(i));
+ }
it++;
}
+
+ std::sort(consolidated_ids.begin(), consolidated_ids.end());
+
+ uint32_t *out = nullptr;
+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
+ *ids, ids_len, &out);
+
+ delete [] *ids;
+ *ids = out;
+
} else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {
auto max_iter = int64map.lower_bound(value); // iter values will be >= value
@@ -83,23 +90,29 @@ public:
}
}
- auto iter = int64map.begin();
+ std::vector<uint32_t> consolidated_ids;
+ auto it = int64map.begin();
while(true) {
- uint32_t* out = nullptr;
- uint32_t* val_ids = iter->second->uncompress();
- ids_len = ArrayUtils::or_scalar(val_ids, iter->second->getLength(),
- *ids, ids_len, &out);
- delete[] val_ids;
- delete[] *ids;
- *ids = out;
+ for(size_t i = 0; i < it->second->getLength(); i++) {
+ consolidated_ids.push_back(it->second->at(i));
+ }
- if(iter == max_iter) {
+ if(it == max_iter) {
break;
}
- iter++;
+ it++;
}
+
+ std::sort(consolidated_ids.begin(), consolidated_ids.end());
+
+ uint32_t *out = nullptr;
+ ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),
+ *ids, ids_len, &out);
+
+ delete [] *ids;
+ *ids = out;
}
}
diff --git a/src/index.cpp b/src/index.cpp
index 29bfd318..5684965e 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1692,8 +1692,8 @@ void Index::search_field(const uint8_t & field_id,
const bool prefix_search = prefix && (token_index == search_tokens.size()-1);
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
- // If this is a prefix search, look for more candidates and do a union of those document IDs
- const int max_candidates = prefix_search ? 10 : 3;
+ // need less candidates for filtered searches since we already only pick tokens with results
+ const int max_candidates = (filter_ids_length == 0) ? 10 : 3;
art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,
filter_ids, filter_ids_length, leaves);