Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed an issue with facet query containing multiple tokens.
kishorenc67c2169b889b
diff --git a/include/index.h b/include/index.hindex e91e35e4..eef34861 100644--- a/include/index.h+++ b/include/index.h@@ -154,7 +154,7 @@ private:size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY,const bool prefix = false, const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD);- void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,+ void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,Topster & topster, uint32_t** all_result_ids,@@ -164,10 +164,10 @@ private:const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,- const bool verbatim) const;+ int facet_id);void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,- uint32_t seq_id, const bool verbatim) const;+ uint32_t seq_id, int facet_id);void index_int32_field(const int32_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;diff --git a/include/string_utils.h b/include/string_utils.hindex bf517868..ff4c1057 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -7,6 +7,7 @@#include <unicode/translit.h>#include <iconv.h>#include <vector>+#include "wyhash_v5.h"struct StringUtils {UErrorCode status;@@ -218,4 +219,9 @@ struct StringUtils {((serialized_num[2] & 0xFF) << 8) | (serialized_num[3] & 0xFF);return seq_id;}++ static int64_t hash_wy(const void* key, uint64_t len) {+ uint64_t hash = wyhash(key, len, 0, _wyp);+ return hash != 0 ? hash : 1; // reserve 0 for use as a delimiter+ }};\ No newline at end of filediff --git a/src/index.cpp b/src/index.cppindex e2670820..b4519f01 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -8,7 +8,6 @@#include <match_score.h>#include <string_utils.h>#include <art.h>-#include "wyhash_v5.h"#include "logger.h"Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,@@ -73,14 +72,24 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tconst std::string & default_sorting_field) {int32_t points = get_points_from_doc(document, default_sorting_field);+ // initialize facet index since it will be updated as well during search indexing+ std::vector<std::vector<uint64_t>> values(facet_schema.size());+ facet_index_v2.emplace(seq_id, values);++ int facet_id = -1; // used to assign a facet field ID+// assumes that validation has already been donefor(const std::pair<std::string, field> & field_pair: search_schema) {const std::string & field_name = field_pair.first;art_tree *t = search_index.at(field_name);+ if(field_pair.second.is_facet()) {+ facet_id++;+ }+if(field_pair.second.type == field_types::STRING) {const std::string & text = document[field_name];- index_string_field(text, points, t, seq_id, false);+ index_string_field(text, points, t, seq_id, facet_id);} else if(field_pair.second.type == field_types::INT32) {uint32_t value = document[field_name];index_int32_field(value, points, t, seq_id);@@ -95,7 +104,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tindex_bool_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {std::vector<std::string> strings = document[field_name];- index_string_array_field(strings, points, t, seq_id, false);+ index_string_array_field(strings, points, t, seq_id, facet_id);} else if(field_pair.second.type == field_types::INT32_ARRAY) {std::vector<int32_t> values = document[field_name];index_int32_array_field(values, points, t, seq_id);@@ -125,29 +134,6 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t}}- std::vector<std::vector<uint64_t>> values(facet_schema.size());- facet_index_v2.emplace(seq_id, values);-- size_t index_facet = 0;- for(const auto & field_pair: facet_schema) {- const std::string & field_name = field_pair.first;- if(field_pair.second.type == field_types::STRING) {- const std::string & text = document[field_name].get<std::string>();- uint64_t hash = wyhash(text.c_str(), text.size(), 0, _wyp);- facet_index_v2[seq_id][index_facet].push_back(hash);- facet_index_v2[seq_id][index_facet].resize(facet_index_v2[seq_id][index_facet].size());- } else if(field_pair.second.type == field_types::STRING_ARRAY) {- const std::vector<std::string> & texts = document[field_name];- for(auto & text: texts) {- uint64_t hash = wyhash(text.c_str(), text.size(), 0, _wyp);- facet_index_v2[seq_id][index_facet].push_back(hash);- }- facet_index_v2[seq_id][index_facet].resize(facet_index_v2[seq_id][index_facet].size());- }-- index_facet++;- }-num_documents += 1;return Option<>(201);}@@ -436,34 +422,39 @@ void Index::index_float_field(const float value, uint32_t score, art_tree *t, uivoid Index::index_string_field(const std::string & text, const uint32_t score, art_tree *t,- uint32_t seq_id, const bool verbatim) const {+ uint32_t seq_id, int facet_id) {std::vector<std::string> tokens;+ StringUtils::split(text, tokens, " ");+std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- if(verbatim) {- tokens.push_back(text);- token_to_offsets[text].push_back(0);- } else {- StringUtils::split(text, tokens, " ");+ for(uint32_t i=0; i<tokens.size(); i++) {+ auto & token = tokens[i];+ string_utils.unicode_normalize(token);+ token_to_offsets[token].push_back(i);- for(uint32_t i=0; i<tokens.size(); i++) {- auto & token = tokens[i];- string_utils.unicode_normalize(token);- token_to_offsets[token].push_back(i);+ if(facet_id >= 0) {+ // facet field+ uint64_t hash = StringUtils::hash_wy(token.c_str(), token.size());+ facet_index_v2[seq_id][facet_id].push_back(hash);}}insert_doc(score, t, seq_id, token_to_offsets);++ if(facet_id >= 0) {+ facet_index_v2[seq_id][facet_id].resize(facet_index_v2[seq_id][facet_id].size());+ }}void Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,- uint32_t seq_id, const bool verbatim) const {+ uint32_t seq_id, int facet_id) {std::unordered_map<std::string, std::vector<uint32_t>> token_positions;for(size_t array_index = 0; array_index < strings.size(); array_index++) {const std::string & str = strings[array_index];std::vector<std::string> tokens;- std::string delim = verbatim ? "" : " ";+ std::string delim = " ";StringUtils::split(str, tokens, delim);std::set<std::string> token_set; // required to deal with repeating tokens@@ -471,13 +462,19 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c// iterate and append offset positionsfor(size_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];+ string_utils.unicode_normalize(token);+ token_positions[token].push_back(i);+ token_set.insert(token);- if(!verbatim) {- string_utils.unicode_normalize(token);+ if(facet_id >= 0) {+ // facet field+ uint64_t hash = StringUtils::hash_wy(token.c_str(), token.size());+ facet_index_v2[seq_id][facet_id].push_back(hash);}+ }- token_positions[token].push_back(i);- token_set.insert(token);+ if(facet_id >= 0) {+ facet_index_v2[seq_id][facet_id].push_back(0); // as a delimiter}// repeat last element to indicate end of offsets for this array index@@ -491,6 +488,10 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c}}+ if(facet_id >= 0) {+ facet_index_v2[seq_id][facet_id].resize(facet_index_v2[seq_id][facet_id].size());+ }+insert_doc(score, t, seq_id, token_positions);}@@ -525,7 +526,7 @@ void Index::index_float_array_field(const std::vector<float> & values, const uinvoid Index::do_facets(std::vector<facet> & facets, const facet_query_t & facet_query,const uint32_t* result_ids, size_t results_size) {std::map<std::string, size_t> facet_to_index;-+size_t i_facet = 0;for(const auto & facet: facet_schema) {facet_to_index[facet.first] = i_facet;@@ -539,16 +540,22 @@ void Index::do_facets(std::vector<facet> & facets, const facet_query_t & facet_qif(a_facet.field_name == facet_query.field_name && !facet_query.query.empty()) {use_facet_query = true;- int bounded_cost = get_bounded_typo_cost(2, facet_query.query.size());- std::vector<art_leaf*> leaves;- art_fuzzy_search(search_index.at(a_facet.field_name), (const unsigned char *) facet_query.query.c_str(),- facet_query.query.size(),0, bounded_cost, 10000,- token_ordering::MAX_SCORE, true, leaves);-- for(const auto & leaf: leaves) {- // calculate hash without terminating null char- uint64_t hash = wyhash(leaf->key, leaf->key_len-1, 0, _wyp);- facet_filter_set.insert(hash);+ std::vector<std::string> facet_queries;+ StringUtils::split(facet_query.query, facet_queries, " ");++ for(auto & q: facet_queries) {+ StringUtils::trim(q);+ int bounded_cost = get_bounded_typo_cost(2, q.size());+ std::vector<art_leaf*> leaves;+ art_fuzzy_search(search_index.at(a_facet.field_name), (const unsigned char *) q.c_str(),+ q.size(),0, bounded_cost, 10000,+ token_ordering::MAX_SCORE, true, leaves);+ for(const auto & leaf: leaves) {+ // calculate hash without terminating null char+ uint64_t hash = StringUtils::hash_wy(leaf->key, leaf->key_len-1);+ facet_filter_set.insert(hash);+ //printf("%.*s - %llu\n", leaf->key_len, leaf->key, hash);+ }}}@@ -556,16 +563,25 @@ void Index::do_facets(std::vector<facet> & facets, const facet_query_t & facet_quint32_t doc_seq_id = result_ids[i];if(facet_index_v2.count(doc_seq_id) != 0) {+ // FORMAT OF VALUES+ // String: h1 h2 h3+ // String array: h1 h2 h3 0 h1 0 h1 h2 0const std::vector<uint64_t> & values = facet_index_v2[doc_seq_id][facet_to_index[a_facet.field_name]];+ int array_pos = -1;for(size_t j = 0; j < values.size(); j++) {- if(use_facet_query && facet_filter_set.find(values[j]) == facet_filter_set.end()) {- // this particular facet value is not found in facet filter, so ignore- continue;+ if(values[j] == 0 || (values.back() != 0 && j == values.size()-1)) {+ array_pos++;+ size_t value_index = (j == 0 ? j : j - 1);++ if(use_facet_query && facet_filter_set.find(values[value_index]) == facet_filter_set.end()) {+ // this particular facet value is not found in facet filter, so ignore+ continue;+ }+ a_facet.result_map[values[value_index]].count += 1;+ a_facet.result_map[values[value_index]].doc_id = doc_seq_id;+ a_facet.result_map[values[value_index]].array_pos = array_pos;}- a_facet.result_map[values[j]].count += 1;- a_facet.result_map[values[j]].doc_id = doc_seq_id;- a_facet.result_map[values[j]].array_pos = j;}}}@@ -585,12 +601,22 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>// assumed that facet fields have already been validated upstreamfor(const uint32_t doc_seq_id: ids) {if(facet_index_v2.count(doc_seq_id) != 0) {+ // FORMAT OF VALUES+ // String: h1 h2 h3+ // String array: h1 h2 h3 0 h1 0 h1 h2 0const std::vector<uint64_t> & values = facet_index_v2[doc_seq_id][facet_to_index[a_facet.field_name]];+ int array_pos = -1;+for(size_t j = 0; j < values.size(); j++) {- a_facet.result_map[values[j]].count -= 1;- if(a_facet.result_map[values[j]].count == 0) {- a_facet.result_map.erase(values[j]);+ if(values[j] == 0 || (values.back() != 0 && j == values.size()-1)) {+ array_pos++;+ size_t value_index = (j == 0 ? j : j - 1);++ a_facet.result_map[values[value_index]].count -= 1;+ if(a_facet.result_map[values[value_index]].count == 0) {+ a_facet.result_map.erase(values[value_index]);+ }}}}@@ -598,7 +624,7 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>}}-void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,+void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,const std::vector<sort_by> & sort_fields,std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,std::vector<std::vector<art_leaf*>> & searched_queries, Topster & topster,@@ -1136,7 +1162,7 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const stif(token_candidates_vec.size() != 0 && token_candidates_vec.size() == tokens.size()) {// If all tokens were found, go ahead and search for candidates with what we have so far- search_candidates(field_id, filter_ids, filter_ids_length, facets, sort_fields, token_candidates_vec,+ search_candidates(field_id, filter_ids, filter_ids_length, sort_fields, token_candidates_vec,token_order, searched_queries, topster, all_result_ids, all_result_ids_len,Index::SEARCH_LIMIT_NUM);@@ -1346,6 +1372,7 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggesif(end_offset - start_offset < 3) {this_array_size = 1; // can only be a string since array needs atleast 3 positions for storage} else {+ // Could be either an array or a string.// Array offset storage format:// a) last element is array_index b) second and third last elements will be largest offset@@ -1354,6 +1381,7 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggesauto third_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 3);if(second_last_val != third_last_val) {+ // guarantees that this is a stringthis_array_size = 1;} else {this_array_size = last_val + 1;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 146aaed6..92bf3dd2 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -520,6 +520,32 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_EQ("16", results["hits"].at(0)["document"]["id"]);}+TEST_F(CollectionTest, MultiOccurrenceString) {+ Collection *coll_multi_string;++ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("points", field_types::INT32, false)+ };++ coll_multi_string = collectionManager.get_collection("coll_multi_string");+ if (coll_multi_string == nullptr) {+ coll_multi_string = collectionManager.create_collection("coll_multi_string", fields, "points").get();+ }++ nlohmann::json document;+ document["title"] = "The brown fox was the tallest of the lot and the quickest of the trot.";+ document["points"] = 100;++ coll_multi_string->add(document.dump());++ query_fields = {"title"};+ nlohmann::json results = coll_multi_string->search("the", query_fields, "", {}, sort_fields, 0, 10, 1,+ FREQUENCY, false, 0).get();+ ASSERT_EQ(1, results["hits"].size());+ collectionManager.drop_collection("coll_multi_string");+}+TEST_F(CollectionTest, ArrayStringFieldHighlight) {Collection *coll_array_text;@@ -1855,6 +1881,18 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ // facet with facet filter query matching 2 tokens+ results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "tags: fine pltinum").get();++ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+// facet query that does not match any indexed valueresults = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,false, Index::DROP_TOKENS_THRESHOLD,