Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Minor cleanup of faceting code.
Kishore Nallan94e84d2c9056
diff --git a/src/index.cpp b/src/index.cppindex 0e6609bd..56200c0b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1350,6 +1350,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const bool use_value_index = facet_infos[findex].use_value_index;auto sort_index_it = sort_index.find(a_facet.field_name);+ auto facet_sort_index_it = sort_index.find(a_facet.sort_field);size_t mod_value = 100 / facet_sample_percent;@@ -1394,11 +1395,9 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}if(should_compute_stats) {- //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;- for(size_t i = 0; i < kv.second.count; ++i) {- compute_facet_stats(a_facet, kv.first, facet_field.type);- }- }+ //LOG(INFO) << "Computing facet stats for facet value" << kv.first;+ compute_facet_stats(a_facet, kv.first, facet_field.type);+ }}if(should_compute_stats) {@@ -1425,11 +1424,11 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const auto& fhash_int64_map = facet_index_v4->get_fhash_int64_map(a_facet.field_name);const auto facet_field_is_array = facet_field.is_array();+ const auto facet_field_is_int64 = facet_field.is_int64();const auto& facet_index = facet_index_v4->get_facet_hash_index(facet_field.name);posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();- std::vector<uint32_t> facet_hashes;- facet_hashes.reserve(1);+ std::vector<uint32_t> facet_hashes(1);if (group_limit != 0) {group_by_field_it_vec = get_group_by_field_iterators(group_by_fields);@@ -1476,18 +1475,20 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,std::set<uint32_t> unique_facet_hashes;for(size_t j = 0; j < facet_hashes.size(); j++) {-const auto& fhash = facet_hashes[j];- if(unique_facet_hashes.count(fhash) == 0) {- unique_facet_hashes.insert(fhash);- } else {- continue;+ // explicitly check for value of facet_hashes to avoid set lookup/insert for non-array faceting+ if(facet_hashes.size() > 1) {+ if(unique_facet_hashes.count(fhash) != 0) {+ continue;+ } else {+ unique_facet_hashes.insert(fhash);+ }}if(should_compute_stats) {int64_t val = fhash;- if(facet_field.is_int64()) {+ if(facet_field_is_int64) {if(fhash_int64_map.find(fhash) != fhash_int64_map.end()) {val = fhash_int64_map.at(fhash);} else {@@ -1526,8 +1527,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);}if(!a_facet.sort_field.empty()) {- sort_index_it = sort_index.find(a_facet.sort_field);- facet_count.sort_field_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);+ facet_count.sort_field_val = get_doc_val_from_sort_index(facet_sort_index_it, doc_seq_id);//LOG(INFO) << "found sort_field val " << facet_count.sort_field;}}@@ -3157,7 +3157,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consbool is_wildcard_no_filter_query = is_wildcard_non_phrase_query && no_filters_provided;if(!facets.empty()) {- const size_t num_threads = 1;+ const size_t num_threads = 1; //std::min(concurrency, all_result_ids_len);const size_t window_size = (num_threads == 0) ? 0 :(all_result_ids_len + num_threads - 1) / num_threads; // rounds up@@ -3215,7 +3215,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conssearch_cutoff = false;auto fq = facet_query;- std::vector<std::pair<std::string, uint32_t>> found_docs;do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields, group_missing_values,batch_result_ids, batch_res_len, max_facet_values,@@ -3234,7 +3233,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscv_process.wait(lock_process, [&](){ return num_processed == num_queued; });search_cutoff = parent_search_cutoff;- for(auto& facet_batch: facet_batches) {+ // use `num_processed` since < `facet_batches.size()` batches could be processed (uneven splitting of records)+ for(size_t batch_index = 0; batch_index < num_processed; batch_index++) {+ auto& facet_batch = facet_batches[batch_index];for(size_t fi = 0; fi < facet_batch.size(); fi++) {auto& this_facet = facet_batch[fi];auto& acc_facet = facets[fi];@@ -3323,7 +3324,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::chrono::high_resolution_clock::now() - beginF).count();LOG(INFO) << "Time for faceting: " << timeMillisF;*/}- std::vector<std::pair<std::string, uint32_t>> found_docs;std::vector<facet_info_t> facet_infos(facets.size());compute_facet_infos(facets, facet_query, facet_query_num_typos,&included_ids_vec[0], included_ids_vec.size(), group_by_fields,diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cppindex f15da086..b2bcd84a 100644--- a/test/collection_optimized_faceting_test.cpp+++ b/test/collection_optimized_faceting_test.cpp@@ -198,8 +198,6 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {ASSERT_EQ(3, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());- LOG(INFO) << results["facet_counts"][0];-ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);@@ -511,6 +509,52 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {collectionManager.drop_collection("coll_array_fields");}+TEST_F(CollectionOptimizedFacetingTest, FacetCountsStringArraySimple) {+ Collection *coll1;++ std::vector<field> fields = {field("tags", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, false),+ field("in_stock", field_types::BOOL, false)};++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();+ }++ nlohmann::json doc;+ doc["id"] = "100";+ doc["tags"] = {"gold", "silver"};+ doc["points"] = 25;+ doc["in_stock"] = true;++ coll1->add(doc.dump());++ std::vector<std::string> facets = {"tags"};++ nlohmann::json results = coll1->search("*", {"tags"}, "", facets, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10,"", 30UL, 4UL,+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());++ ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());++ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());++ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionOptimizedFacetingTest, FacetCountsBool) {Collection *coll1;