Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
adding leftover changes
krunal77d934d38135
diff --git a/include/index.h b/include/index.hindex e60a9e21..95cdc1e7 100644--- a/include/index.h+++ b/include/index.h@@ -304,6 +304,11 @@ struct hnsw_index_t {}};+struct group_by_field_it_t {+ std::string field_name;+ posting_list_t::iterator_t it;+};+class Index {private:mutable std::shared_mutex mutex;@@ -538,7 +543,9 @@ private:static void batch_embed_fields(std::vector<index_record*>& documents,const tsl::htrie_map<char, field>& embedding_fields,const tsl::htrie_map<char, field> & search_schema, const size_t remote_embedding_batch_size = 200);-++ std::vector<group_by_field_it_t> get_group_by_field_iterators(const std::vector<std::string>&) const;+public:// for limiting number of results on multiple candidates / query rewritesenum {TYPO_TOKENS_THRESHOLD = 1};diff --git a/src/index.cpp b/src/index.cppindex 72b4e0f4..c86cb929 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1259,6 +1259,21 @@ int64_t Index::get_doc_val_from_sort_index(sort_index_iterator sort_index_it, uireturn INT64_MAX;}+std::vector<group_by_field_it_t> Index::get_group_by_field_iterators(const std::vector<std::string>& group_by_fields) const {+ std::vector<group_by_field_it_t> group_by_field_it_vec;+ for (const auto &field_name: group_by_fields) {+ if (!facet_index_v4->has_hash_index(field_name)) {+ continue;+ }+ auto facet_index = facet_index_v4->get_facet_hash_index(field_name);+ auto facet_index_it = facet_index->new_iterator();++ group_by_field_it_t group_by_field_it_struct {field_name, std::move(facet_index_it)};+ group_by_field_it_vec.emplace_back(std::move(group_by_field_it_struct));+ }+ return group_by_field_it_vec;+}+void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,bool estimate_facets, size_t facet_sample_percent,const std::vector<facet_info_t>& facet_infos,@@ -1272,18 +1287,9 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,return ;}- size_t total_docs = seq_ids->num_ids();-- std::vector<std::pair<posting_list_t::iterator_t, std::string>> group_by_it_field_vec;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- group_by_it_field_vec.emplace_back(std::make_pair(std::move(facet_index_it), field_name));- }+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);+ size_t total_docs = seq_ids->num_ids();// assumed that facet fields have already been validated upstreamfor(size_t findex=0; findex < facets.size(); findex++) {auto& a_facet = facets[findex];@@ -1384,8 +1390,8 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,uint64_t distinct_id = 0;if(group_limit) {distinct_id = 1;- for(auto& kv : group_by_it_field_vec) {- get_distinct_id(kv.second, kv.first, doc_seq_id, group_missing_values, distinct_id);+ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, doc_seq_id, group_missing_values, distinct_id);}}//LOG(INFO) << "facet_hash_count " << facet_hash_count;@@ -2387,13 +2393,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consuint64_t distinct_id = seq_id;if (group_limit != 0) {distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);+ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}if(excluded_group_ids.count(distinct_id) != 0) {continue;@@ -2516,13 +2518,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consuint64_t distinct_id = seq_id;if (group_limit != 0) {distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);+ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}if(excluded_group_ids.count(distinct_id) != 0) {continue;@@ -2954,14 +2952,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consuint64_t distinct_id = seq_id;if (group_limit != 0) {distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);++ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}+if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -3221,14 +3217,12 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>// if one `id` of a group is present in curated hits, we have to exclude that entire group from resultsfor(auto seq_id: included_ids_vec) {uint64_t distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);++ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}+excluded_group_ids.emplace(distinct_id);}}@@ -3947,14 +3941,12 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokenuint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);++ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}+if(excluded_group_ids.count(distinct_id) != 0) {return;}@@ -4764,14 +4756,12 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::uint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);++ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}+if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -4934,14 +4924,12 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vuint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);++ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}+if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -5301,16 +5289,6 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootsearch_stop_us = parent_search_stop_ms;search_cutoff = parent_search_cutoff;- std::vector<std::pair<posting_list_t::iterator_t, std::string>> group_by_it_field_vec;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- group_by_it_field_vec.emplace_back(std::make_pair(std::move(facet_index_it), field_name));- }-std::vector<uint32_t> filter_indexes;for(size_t i = 0; i < batch_result->count; i++) {@@ -5339,8 +5317,10 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootuint64_t distinct_id = seq_id;if(group_limit != 0) {distinct_id = 1;- for(auto& kv: group_by_it_field_vec) {- get_distinct_id(kv.second, kv.first, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);++ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}if(excluded_group_ids.count(distinct_id) != 0) {@@ -5997,13 +5977,10 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16if(group_limit != 0) {distinct_id = 1;- for(const auto& field_name: group_by_fields) {- if (!facet_index_v4->has_hash_index(field_name)) {- continue;- }- auto facet_index = facet_index_v4->get_facet_hash_index(field_name);- auto facet_index_it = facet_index->new_iterator();- get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);+ auto group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);++ for(auto& kv : group_by_field_it_vec) {+ get_distinct_id(kv.field_name, kv.it, seq_id, group_missing_values, distinct_id);}}