Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use enum for selecting facet index type.
Kishore Nallan3c4e93887999
diff --git a/include/collection.h b/include/collection.hindex 834f17af..c56d1964 100644--- a/include/collection.h+++ b/include/collection.h@@ -463,7 +463,7 @@ public:const size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,const size_t page_offset = UINT32_MAX,- bool use_facet_intersection = false+ facet_index_type_t facet_index_type = HASH) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/facet_index.h b/include/facet_index.hindex 6076f5af..575a6964 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -137,5 +137,7 @@ public:bool has_hash_index(const std::string& field_name);+ bool has_value_index(const std::string& field_name);+posting_list_t* get_facet_hash_index(const std::string& field_name);};\ No newline at end of filediff --git a/include/index.h b/include/index.hindex da706763..0ac4c5fd 100644--- a/include/index.h+++ b/include/index.h@@ -188,6 +188,12 @@ struct search_args {};};+enum facet_index_type_t {+ HASH,+ VALUE,+ DETECT,+};+struct offsets_facet_hashes_t {// token to offsetsstd::unordered_map<std::string, std::vector<uint32_t>> offsets;@@ -364,7 +370,7 @@ private:size_t group_limit, const std::vector<std::string>& group_by_fields,const uint32_t* result_ids, size_t results_size,int max_facet_count, bool is_wildcard_query, bool no_filters_provided,- bool use_facet_intersection = false+ facet_index_type_t facet_index_type) const;bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,@@ -621,7 +627,7 @@ public:// Public operationsOption<bool> run_search(search_args* search_params, const std::string& collection_name,- bool use_facet_intersection);+ facet_index_type_t facet_index_type);Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,const text_match_type_t match_type,@@ -646,7 +652,7 @@ public:const size_t max_extra_suffix, const size_t facet_query_num_typos,const bool filter_curated_hits, enable_t split_join_tokens,const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold,- const std::string& collection_name, bool use_facet_intersection = false) const;+ const std::string& collection_name, facet_index_type_t facet_index_type = DETECT) const;void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name);@@ -750,7 +756,7 @@ public:const uint32_t* all_result_ids, const size_t& all_result_ids_len,const std::vector<std::string>& group_by_fields,size_t max_candidates,- std::vector<facet_info_t>& facet_infos, bool use_facet_intersection) const;+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const;void resolve_space_as_typos(std::vector<std::string>& qtokens, const std::string& field_name,std::vector<std::vector<std::string>>& resolved_queries) const;diff --git a/src/collection.cpp b/src/collection.cppindex 92b1cc72..aefc2973 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1078,7 +1078,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const size_t facet_sample_percent,const size_t facet_sample_threshold,const size_t page_offset,- bool use_facet_intersection) const {+ facet_index_type_t facet_index_type) const {std::shared_lock lock(mutex);@@ -1543,7 +1543,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,std::unique_ptr<search_args> search_params_guard(search_params);- auto search_op = index->run_search(search_params, name, use_facet_intersection);+ auto search_op = index->run_search(search_params, name, facet_index_type);// filter_tree_root might be updated in Index::static_filter_query_eval.filter_tree_root_guard.release();diff --git a/src/facet_index.cpp b/src/facet_index.cppindex c7981fb1..0c5d495e 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -270,6 +270,11 @@ bool facet_index_t::has_hash_index(const std::string &field_name) {return facet_index_it != facet_field_map.end() && facet_index_it->second.has_hash_index;}+bool facet_index_t::has_value_index(const std::string &field_name) {+ auto facet_index_it = facet_field_map.find(field_name);+ return facet_index_it != facet_field_map.end() && facet_index_it->second.has_value_index;+}+posting_list_t* facet_index_t::get_facet_hash_index(const std::string &field_name) {auto facet_index_it = facet_field_map.find(field_name);if(facet_index_it != facet_field_map.end()) {diff --git a/src/index.cpp b/src/index.cppindex d90407b5..584a4655 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -14,9 +14,7 @@#include <s2/s2latlng.h>#include <s2/s2region_term_indexer.h>#include <s2/s2cap.h>-#include <s2/s2earth.h>#include <s2/s2loop.h>-#include <s2/s2builder.h>#include <posting.h>#include <thread_local_vars.h>#include <unordered_set>@@ -1203,7 +1201,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const size_t group_limit, const std::vector<std::string>& group_by_fields,const uint32_t* result_ids, size_t results_size,int max_facet_count, bool is_wildcard_query, bool no_filters_provided,- bool use_facet_intersection) const {+ facet_index_type_t facet_index_type) const {if(results_size == 0) {return ;@@ -1222,23 +1220,19 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,size_t mod_value = 100 / facet_sample_percent;auto num_facet_values = facet_index_v4->get_facet_count(facet_field.name);-- bool use_hashes = false;-- if(!use_facet_intersection) {- use_hashes = true;+ if(num_facet_values == 0) {+ continue;}-#ifndef TEST_BUILD- // non-test build should not accidentally set this flag- use_facet_intersection = false;- use_hashes = false;-#endifbool is_wildcard_no_filter_query = is_wildcard_query && no_filters_provided;bool facet_hash_index_exists = facet_index_v4->has_hash_index(facet_field.name);+ bool facet_value_index_exists = facet_index_v4->has_value_index(facet_field.name);- if((num_facet_values && ((!facet_hash_index_exists || is_wildcard_no_filter_query- || num_facet_values > 50000) && group_limit == 0) && !use_hashes) || use_facet_intersection) {+#ifdef TEST_BUILD+ if(facet_index_type == VALUE) {+#else+ if(facet_value_index_exists && (group_limit == 0 || is_wildcard_no_filter_query)) {+#endif// LOG(INFO) << "Using intersection to find facets";a_facet.is_intersected = true;@@ -1882,7 +1876,7 @@ Option<bool> Index::get_approximate_reference_filter_ids_with_lock(filter_node_t}Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,- bool use_facet_intersection) {+ facet_index_type_t facet_index_type) {return search(search_params->field_query_tokens,search_params->search_fields,search_params->match_type,@@ -1918,7 +1912,7 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->facet_sample_percent,search_params->facet_sample_threshold,collection_name,- use_facet_intersection);+ facet_index_type);}void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,@@ -2367,7 +2361,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consconst bool filter_curated_hits, const enable_t split_join_tokens,const vector_query_t& vector_query,size_t facet_sample_percent, size_t facet_sample_threshold,- const std::string& collection_name, bool use_facet_intersection) const {+ const std::string& collection_name, facet_index_type_t facet_index_type) const {std::shared_lock lock(mutex);uint32_t approx_filter_ids_length = 0;@@ -2895,7 +2889,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<facet_info_t> facet_infos(facets.size());compute_facet_infos(facets, facet_query, facet_query_num_typos, all_result_ids, all_result_ids_len,- group_by_fields, max_candidates, facet_infos, use_facet_intersection);+ group_by_fields, max_candidates, facet_infos, facet_index_type);std::vector<std::vector<facet>> facet_batches(num_threads);for(size_t i = 0; i < num_threads; i++) {@@ -2928,7 +2922,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consbatch_result_ids, batch_res_len, &facet_infos, max_facet_values,is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,- &num_processed, &m_process, &cv_process, use_facet_intersection]() {+ &num_processed, &m_process, &cv_process, facet_index_type]() {search_begin_us = parent_search_begin;search_stop_us = parent_search_stop_ms;search_cutoff = parent_search_cutoff;@@ -2939,7 +2933,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfacet_infos, group_limit, group_by_fields,batch_result_ids, batch_res_len, max_facet_values,is_wildcard_query, no_filters_provided,- use_facet_intersection);+ facet_index_type);std::unique_lock<std::mutex> lock(m_process);num_processed++;parent_search_cutoff = parent_search_cutoff || search_cutoff;@@ -3017,11 +3011,11 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<facet_info_t> facet_infos(facets.size());compute_facet_infos(facets, facet_query, facet_query_num_typos,&included_ids_vec[0], included_ids_vec.size(), group_by_fields,- max_candidates, facet_infos, use_facet_intersection);+ max_candidates, facet_infos, facet_index_type);do_facets(facets, facet_query, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields, &included_ids_vec[0],included_ids_vec.size(), max_facet_values, is_wildcard_query, no_filters_provided,- use_facet_intersection);+ facet_index_type);all_result_ids_len += curated_topster->size;@@ -3248,7 +3242,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,// NOTE: when accessing other field ordered properties like prefixes or num_typos we have to index// them by `the_field.orig_index` since the original fields could be reordered on their weights.auto& the_field = the_fields[field_id];- const bool field_prefix = (the_field.orig_index < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;+ const bool field_prefix = (the_field.orig_index < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;@@ -4423,7 +4417,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&const uint32_t* all_result_ids, const size_t& all_result_ids_len,const std::vector<std::string>& group_by_fields,const size_t max_candidates,- std::vector<facet_info_t>& facet_infos, bool use_facet_intersection) const {+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const {if(all_result_ids_len == 0) {return;@@ -4431,9 +4425,6 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(size_t findex=0; findex < facets.size(); findex++) {const auto& a_facet = facets[findex];-- bool has_hash_index = facet_index_v4->has_hash_index(a_facet.field_name);-facet_infos[findex].use_facet_query = false;const field &facet_field = search_schema.at(a_facet.field_name);@@ -4489,7 +4480,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&//LOG(INFO) << "searched_queries.size: " << searched_queries.size();- // NOTE: `field_result_ids` will consists of IDs across ALL queries in searched_queries+ // NOTE: `field_result_ids` will consist of IDs across ALL queries in searched_queriesfor(size_t si = 0; si < searched_queries.size(); si++) {const auto& searched_query = searched_queries[si];@@ -4507,10 +4498,6 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(size_t i = 0; i < field_result_ids_len; i++) {uint32_t seq_id = field_result_ids[i];----bool id_matched = true;for(auto pl: posting_lists) {@@ -4525,15 +4512,13 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&if(!id_matched) {continue;}- //LOG(INFO) << "seq_id matched : " << seq_id;-#ifdef TEST_BUILD- if(!use_facet_intersection) {-#else- //checking if facets is migrated to old index- if(has_hash_index) {-#endif- std::vector<uint32_t> facet_hashes;+ #ifdef TEST_BUILD+ if(facet_index_type == HASH) {+ #else+ if(facet_index_v4->has_hash_index(a_facet.field_name)) {+ #endif+ std::vector<uint32_t> facet_hashes;auto facet_index = facet_index_v4->get_facet_hash_index(a_facet.field_name);posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();facet_index_it.skip_to(seq_id);@@ -4566,7 +4551,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&}}}- } else { //for new index adding search tokens+ } else {+ // value based indexfor(const auto& val : searched_tokens) {facet_infos[findex].hashes[facet_field.name].emplace_back(val);}diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cppindex d5e23a5d..71c3472e 100644--- a/test/collection_optimized_faceting_test.cpp+++ b/test/collection_optimized_faceting_test.cpp@@ -79,7 +79,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {3UL, "<mark>", "</mark>", {}, 4294967295UL, true,false, true, "", false, 6000000UL, 4UL, 7UL, fallback,4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false, "", true,- 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());@@ -110,7 +110,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", "", {}, 3UL, "<mark>", "</mark>", {}, 4294967295UL, true,false, true, "", false, 6000000UL, 4UL, 7UL, fallback, 4UL, {off},32767UL, 32767UL, 2UL, 2UL, false, "", true, 0UL, max_score, 100UL,- 0UL, 4294967295UL, true).get();+ 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());@@ -136,7 +136,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {3UL, "<mark>", "</mark>", {}, 4294967295UL, true,false, true, "", false, 6000000UL, 4UL, 7UL, fallback,4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false, "", true,- 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(2, results["facet_counts"].size());@@ -156,7 +156,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {3UL, "<mark>", "</mark>", {}, 4294967295UL, true,false, true, "", false, 6000000UL, 4UL, 7UL, fallback,4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false, "", true,- 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -170,7 +170,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["facet_counts"][0]["stats"].size());ASSERT_FLOAT_EQ(4.880199885368347, results["facet_counts"][0]["stats"]["avg"].get<double>());@@ -193,7 +193,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(3, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -217,7 +217,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -233,7 +233,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -250,7 +250,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -266,7 +266,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -282,7 +282,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -304,7 +304,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -328,7 +328,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());ASSERT_STREQ("age", results["facet_counts"][0]["field_name"].get<std::string>().c_str());@@ -341,7 +341,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());@@ -352,7 +352,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());@@ -364,7 +364,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -378,7 +378,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(0, results["facet_counts"].size());@@ -390,7 +390,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -404,7 +404,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_STREQ("Facet query must be in the `facet_field: value` format.", res_op.error().c_str());@@ -417,7 +417,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_STREQ("Could not find a facet field named `foobar` in the schema.", res_op.error().c_str());@@ -430,7 +430,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_STREQ("Only prefix matching with a wildcard is allowed.", res_op.error().c_str());@@ -443,7 +443,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_STREQ("Could not find a facet field for `foo*` in the schema.", res_op.error().c_str());@@ -456,7 +456,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_STREQ("The `facet_query` parameter is supplied without a `facet_by` parameter.", res_op.error().c_str());@@ -468,7 +468,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_STREQ("Could not find a facet field named `` in the schema.", res_op.error().c_str());@@ -481,7 +481,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_STREQ("Facet query refers to a facet field `name_facet` that is not part of `facet_by` parameter.", res_op.error().c_str());@@ -533,7 +533,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCountsBool) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());@@ -575,7 +575,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCountsFloatPrecision) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());@@ -620,7 +620,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetStatOnFloatFields) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);auto results = res_op.get();@@ -668,7 +668,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCountOnSimilarStrings) {"", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},4294967295UL, true, false, true, "", false, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(2, results["hits"].size());ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());@@ -716,7 +716,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetByNestedIntField) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1UL, "", "", {}, 3UL,"<mark>", "</mark>", {}, 4294967295UL, true, false, true, "", false, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(2, results["found"].get<size_t>());ASSERT_EQ(1, results["facet_counts"].size());@@ -902,7 +902,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTest) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);@@ -919,7 +919,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTest) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(1, results2["facet_counts"][0]["counts"].size());ASSERT_EQ(1, results2["facet_counts"][0]["counts"][0]["count"].get<std::size_t>());@@ -936,7 +936,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTest) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", true, 6000000UL, 4UL,7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false, "", true, 0UL,- max_score, 100UL, 0UL, 4294967295UL, true);+ max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(results3.ok());ASSERT_EQ("Could not find a facet field named `visitorsz` in the schema.", results3.error());@@ -949,7 +949,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTest) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true).get();+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();ASSERT_EQ(2, results4["facet_counts"][0]["counts"].size());ASSERT_EQ(3, results4["facet_counts"][0]["counts"][0]["count"].get<std::size_t>());@@ -968,7 +968,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTest) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(rop.ok());ASSERT_EQ("Range facet is restricted to only integer and float fields.", rop.error());@@ -983,7 +983,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTest) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_FALSE(rop.ok());ASSERT_EQ("Facet range value is not valid.", rop.error());@@ -1044,7 +1044,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetContinuity) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_STREQ("Ranges in range facet syntax should be continous.", results.error().c_str());@@ -1057,7 +1057,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetContinuity) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_STREQ("Ranges in range facet syntax should be continous.", results2.error().c_str());@@ -1117,7 +1117,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTypo) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_STREQ("Error splitting the facet range values.", results.error().c_str());@@ -1130,7 +1130,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTypo) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_STREQ("Error splitting the facet range values.", results2.error().c_str());@@ -1143,7 +1143,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTypo) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_STREQ("Error splitting the facet range values.", results3.error().c_str());@@ -1156,7 +1156,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTypo) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_STREQ("Facet range value is not valid.", results4.error().c_str());@@ -1169,7 +1169,7 @@ TEST_F(CollectionOptimizedFacetingTest, RangeFacetTypo) {"<mark>", "</mark>", {}, 1000,true, false, true, "", true, 6000000UL,4UL, 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,- "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, true);+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE);ASSERT_STREQ("Facet range value is not valid.", results5.error().c_str());@@ -1208,7 +1208,7 @@ TEST_F(CollectionOptimizedFacetingTest, SampleFacetCounts) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,- 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 10, 0, 4294967295UL, true).get();+ 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 10, 0, 4294967295UL, VALUE).get();ASSERT_EQ(1000, res["found"].get<size_t>());ASSERT_EQ(1, res["facet_counts"].size());@@ -1224,7 +1224,7 @@ TEST_F(CollectionOptimizedFacetingTest, SampleFacetCounts) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,- 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 10, 10000, 4294967295UL, true).get();+ 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 10, 10000, 4294967295UL, VALUE).get();ASSERT_EQ(1000, res["found"].get<size_t>());ASSERT_EQ(1, res["facet_counts"].size());@@ -1246,7 +1246,7 @@ TEST_F(CollectionOptimizedFacetingTest, SampleFacetCounts) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,- 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 200, 0, 4294967295UL, true);+ 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 200, 0, 4294967295UL, VALUE);ASSERT_FALSE(res_op.ok());ASSERT_EQ("Value of `facet_sample_percent` must be less than 100.", res_op.error());@@ -1273,7 +1273,7 @@ TEST_F(CollectionOptimizedFacetingTest, FacetOnArrayFieldWithSpecialChars) {"", {"tags"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,- 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 100, 0, 4294967295UL, true).get();+ 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 100, 0, 4294967295UL, VALUE).get();ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(4, results["facet_counts"][0]["counts"].size());@@ -1326,7 +1326,7 @@ TEST_F(CollectionOptimizedFacetingTest, StringLengthTest) {"", {"tags"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,- 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 100, 0, 4294967295UL, true).get();+ 4, {off}, 3, 3, 2, 2, false, "", true, 0, max_score, 100, 0, 4294967295UL, VALUE).get();ASSERT_EQ(1, results["facet_counts"].size());diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex ff416f7d..25515f83 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -223,26 +223,6 @@ TEST(TokenizerTest, ShouldTokenizeChineseText) {}TEST(TokenizerTest, ShouldTokenizeLocaleText) {- std::string text = R"("王獣を倒すと入手した折れた角。追放された後、この世に存在すべきではないもの。\n獣域ウルブズの中で帝王と呼ばれていても、魔獣たちの系譜では、その兄たちの万分の一にも満たないだろう。\n「黄-金」が無数の獣域ウルブズを捨て紙のように圧縮して偶然にできた異形の魔獣。その角には、黒いウルブズを命じて自分のために空間を溶かす権威が秘めている。")";-- Tokenizer tokenizer(text, true, false, "ja", {}, {});-- LOG(INFO) << "text.size: " << text.size();-- std::string raw_token;- size_t raw_token_index = 0, tok_start = 0, tok_end = 0;-- // based on `highlight_affix_num_tokens`- size_t snippet_start_offset = 0, snippet_end_offset = (text.empty() ? 0 : text.size() - 1);-- // window used to locate the starting offset for snippet on the text- while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {- LOG(INFO) << "tok_start: " << tok_start;- }-- return ;-std::vector<std::string> tokens;tokens.clear();