Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.26-filter' into v0.26-facets
Kishore Nallanb85da1b2f5b3
diff --cc include/collection.hindex c56d1964,12b6a7ab..571d0b1c--- a/include/collection.h+++ b/include/collection.h@@@ -463,8 -464,7 +464,8 @@@ publicconst size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,const size_t page_offset = UINT32_MAX,- facet_index_type_t facet_index_type = HASH- ) const;++ facet_index_type_t facet_index_type = HASH,+ const size_t vector_query_hits = 250) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --cc include/index.hindex 4a1e760f,1d7e8c23..36c2e24a--- a/include/index.h+++ b/include/index.h@@@ -30,8 -30,7 +30,8 @@@#include "vector_query_ops.h"#include "hnswlib/hnswlib.h"#include "filter.h"+#include "facet_index.h"- #include "numeric_range_trie_test.h"+ #include "numeric_range_trie.h"static constexpr size_t ARRAY_FACET_DIM = 4;using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;@@@ -530,7 -527,7 +534,7 @@@ privatevoid initialize_facet_indexes(const field& facet_field);- static Option<bool> batch_embed_fields(std::vector<nlohmann::json*>& documents,- static void batch_embed_fields(std::vector<index_record*>& documents,++ static void batch_embed_fields(std::vector<index_record*>& documents,const tsl::htrie_map<char, field>& embedding_fields,const tsl::htrie_map<char, field> & search_schema);diff --cc src/collection.cppindex 098ea064,c044c01e..ff28887d--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -1077,7 -1108,7 +1107,8 @@@ Option<nlohmann::json> Collection::searconst size_t facet_sample_percent,const size_t facet_sample_threshold,const size_t page_offset,- facet_index_type_t facet_index_type) const {++ facet_index_type_t facet_index_type,+ const size_t vector_query_hits) const {std::shared_lock lock(mutex);diff --cc src/index.cppindex c118ad0a,9c8d1b80..57509543--- a/src/index.cpp+++ b/src/index.cpp@@@ -42,7 -43,18 +42,8 @@@ spp::sparse_hash_map<uint32_t, int64_tspp::sparse_hash_map<uint32_t, int64_t> Index::eval_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;+ spp::sparse_hash_map<uint32_t, int64_t> Index::vector_distance_sentinel_value;-struct token_posting_t {- uint32_t token_id;- const posting_list_t::iterator_t& posting;-- token_posting_t(uint32_t token_id, const posting_list_t::iterator_t& posting):- token_id(token_id), posting(posting) {-- }-};-Index::Index(const std::string& name, const uint32_t collection_id, const Store* store,SynonymIndex* synonym_index, ThreadPool* thread_pool,const tsl::htrie_map<char, field> & search_schema,@@@ -991,32 -939,43 +984,39 @@@ void Index::index_field_in_memory(consif(afield.type == field_types::INT32_ARRAY) {const int32_t value = arr_value;- num_tree->insert(value, seq_id, afield.is_facet());-if (afield.range_index) {trie->insert(value, seq_id);+ } else {+ num_tree->insert(value, seq_id);}}else if(afield.type == field_types::INT64_ARRAY) {const int64_t value = arr_value;- num_tree->insert(value, seq_id, afield.is_facet());-if (afield.range_index) {trie->insert(value, seq_id);+ } else {+ num_tree->insert(value, seq_id);}}else if(afield.type == field_types::FLOAT_ARRAY) {const float fvalue = arr_value;int64_t value = float_to_int64_t(fvalue);- num_tree->insert(value, seq_id, afield.is_facet());-if (afield.range_index) {trie->insert(value, seq_id);+ } else {+ num_tree->insert(value, seq_id);}}else if(afield.type == field_types::BOOL_ARRAY) {const bool value = record.doc[afield.name][arr_i];- num_tree->insert(int64_t(value), seq_id, afield.is_facet());-+ if (afield.range_index) {+ trie->insert(value, seq_id);+ } else {+ num_tree->insert(value, seq_id);+ }}}});@@@ -1921,15 -1645,7 +1742,8 @@@ Option<bool> Index::do_reference_filterreturn Option(true);}- Option<bool> Index::get_approximate_reference_filter_ids_with_lock(filter_node_t* const filter_tree_root,- uint32_t& filter_ids_length) const {- std::shared_lock lock(mutex);-- return rearrange_filter_tree(filter_tree_root, filter_ids_length);- }--Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name) {+Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,+ facet_index_type_t facet_index_type) {return search(search_params->field_query_tokens,search_params->search_fields,search_params->match_type,@@@ -2414,17 -2130,10 +2228,10 @@@ Option<bool> Index::search(std::vector<const bool filter_curated_hits, const enable_t split_join_tokens,const vector_query_t& vector_query,size_t facet_sample_percent, size_t facet_sample_threshold,- const std::string& collection_name) const {+ const std::string& collection_name, facet_index_type_t facet_index_type) const {std::shared_lock lock(mutex);- uint32_t approx_filter_ids_length = 0;- auto rearrange_op = rearrange_filter_tree(filter_tree_root, approx_filter_ids_length, collection_name);- if (!rearrange_op.ok()) {- return rearrange_op;- }-- auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,- approx_filter_ids_length);+ auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root);std::unique_ptr<filter_result_iterator_t> filter_iterator_guard(filter_result_iterator);auto filter_init_op = filter_result_iterator->init_status();