Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.26-filter' into v0.26-facets

Kishore Nallanb85da1b2f5b3
diff --cc include/collection.h
index c56d1964,12b6a7ab..571d0b1c
--- a/include/collection.h
+++ b/include/collection.h
@@@ -463,8 -464,7 +464,8 @@@ public
const size_t facet_sample_percent = 100,
const size_t facet_sample_threshold = 0,
const size_t page_offset = UINT32_MAX,
- facet_index_type_t facet_index_type = HASH
- ) const;
++ facet_index_type_t facet_index_type = HASH,
+ const size_t vector_query_hits = 250) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --cc include/index.h
index 4a1e760f,1d7e8c23..36c2e24a
--- a/include/index.h
+++ b/include/index.h
@@@ -30,8 -30,7 +30,8 @@@
#include "vector_query_ops.h"
#include "hnswlib/hnswlib.h"
#include "filter.h"
+#include "facet_index.h"
- #include "numeric_range_trie_test.h"
+ #include "numeric_range_trie.h"
static constexpr size_t ARRAY_FACET_DIM = 4;
using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;
@@@ -530,7 -527,7 +534,7 @@@ private
void initialize_facet_indexes(const field& facet_field);
- static Option<bool> batch_embed_fields(std::vector<nlohmann::json*>& documents,
- static void batch_embed_fields(std::vector<index_record*>& documents,
++ static void batch_embed_fields(std::vector<index_record*>& documents,
const tsl::htrie_map<char, field>& embedding_fields,
const tsl::htrie_map<char, field> & search_schema);
diff --cc src/collection.cpp
index 098ea064,c044c01e..ff28887d
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -1077,7 -1108,7 +1107,8 @@@ Option<nlohmann::json> Collection::sear
const size_t facet_sample_percent,
const size_t facet_sample_threshold,
const size_t page_offset,
- facet_index_type_t facet_index_type) const {
++ facet_index_type_t facet_index_type,
+ const size_t vector_query_hits) const {
std::shared_lock lock(mutex);
diff --cc src/index.cpp
index c118ad0a,9c8d1b80..57509543
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -42,7 -43,18 +42,8 @@@ spp::sparse_hash_map<uint32_t, int64_t
spp::sparse_hash_map<uint32_t, int64_t> Index::eval_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;
+ spp::sparse_hash_map<uint32_t, int64_t> Index::vector_distance_sentinel_value;
-struct token_posting_t {
- uint32_t token_id;
- const posting_list_t::iterator_t& posting;
-
- token_posting_t(uint32_t token_id, const posting_list_t::iterator_t& posting):
- token_id(token_id), posting(posting) {
-
- }
-};
-
Index::Index(const std::string& name, const uint32_t collection_id, const Store* store,
SynonymIndex* synonym_index, ThreadPool* thread_pool,
const tsl::htrie_map<char, field> & search_schema,
@@@ -991,32 -939,43 +984,39 @@@ void Index::index_field_in_memory(cons
if(afield.type == field_types::INT32_ARRAY) {
const int32_t value = arr_value;
- num_tree->insert(value, seq_id, afield.is_facet());
-
if (afield.range_index) {
trie->insert(value, seq_id);
+ } else {
+ num_tree->insert(value, seq_id);
}
}
else if(afield.type == field_types::INT64_ARRAY) {
const int64_t value = arr_value;
- num_tree->insert(value, seq_id, afield.is_facet());
-
if (afield.range_index) {
trie->insert(value, seq_id);
+ } else {
+ num_tree->insert(value, seq_id);
}
}
else if(afield.type == field_types::FLOAT_ARRAY) {
const float fvalue = arr_value;
int64_t value = float_to_int64_t(fvalue);
- num_tree->insert(value, seq_id, afield.is_facet());
-
if (afield.range_index) {
trie->insert(value, seq_id);
+ } else {
+ num_tree->insert(value, seq_id);
}
}
else if(afield.type == field_types::BOOL_ARRAY) {
const bool value = record.doc[afield.name][arr_i];
- num_tree->insert(int64_t(value), seq_id, afield.is_facet());
-
+ if (afield.range_index) {
+ trie->insert(value, seq_id);
+ } else {
+ num_tree->insert(value, seq_id);
+ }
}
}
});
@@@ -1921,15 -1645,7 +1742,8 @@@ Option<bool> Index::do_reference_filter
return Option(true);
}
- Option<bool> Index::get_approximate_reference_filter_ids_with_lock(filter_node_t* const filter_tree_root,
- uint32_t& filter_ids_length) const {
- std::shared_lock lock(mutex);
-
- return rearrange_filter_tree(filter_tree_root, filter_ids_length);
- }
-
-Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name) {
+Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,
+ facet_index_type_t facet_index_type) {
return search(search_params->field_query_tokens,
search_params->search_fields,
search_params->match_type,
@@@ -2414,17 -2130,10 +2228,10 @@@ Option<bool> Index::search(std::vector<
const bool filter_curated_hits, const enable_t split_join_tokens,
const vector_query_t& vector_query,
size_t facet_sample_percent, size_t facet_sample_threshold,
- const std::string& collection_name) const {
+ const std::string& collection_name, facet_index_type_t facet_index_type) const {
std::shared_lock lock(mutex);
- uint32_t approx_filter_ids_length = 0;
- auto rearrange_op = rearrange_filter_tree(filter_tree_root, approx_filter_ids_length, collection_name);
- if (!rearrange_op.ok()) {
- return rearrange_op;
- }
-
- auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,
- approx_filter_ids_length);
+ auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root);
std::unique_ptr<filter_result_iterator_t> filter_iterator_guard(filter_result_iterator);
auto filter_init_op = filter_result_iterator->init_status();