Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Highlight best matched string in an array.
Kishore Nallanb669a47c2905
diff --git a/TODO.md b/TODO.mdindex b4479eda..ba6b6966 100644--- a/TODO.md+++ b/TODO.md@@ -100,6 +100,8 @@- NOT operator support- Log operations- Parameterize replica's MAX_UPDATES_TO_SEND+- NOT operator support+- 64K token limit- > INT32_MAX validation for float field- highlight of string arrays?- test for token ranking on float fielddiff --git a/include/index.h b/include/index.hindex 79ab0ad2..3346cec9 100644--- a/include/index.h+++ b/include/index.h@@ -84,10 +84,6 @@ private:void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);- void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,- size_t result_index, std::vector<std::vector<uint16_t>> &token_positions) const;-void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,const int num_typos, const size_t num_results,@@ -102,6 +98,9 @@ private:Topster<512> & topster, size_t & total_results, uint32_t** all_result_ids,size_t & all_result_ids_len, const size_t & max_results, const bool prefix);+ void insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,+ const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;+void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,const bool verbatim) const;@@ -147,6 +146,11 @@ public:Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);+ static void populate_token_positions(const std::vector<art_leaf *> &query_suggestion,+ spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,+ size_t result_index,+ std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);+void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint32_t total_cost,Topster<512> &topster, const std::vector<art_leaf *> & query_suggestion,const uint32_t *result_ids, const size_t result_size) const;@@ -162,6 +166,8 @@ public:// strings under this length will be fully highlighted, instead of showing a snippet of relevant portionenum {SNIPPET_STR_ABOVE_LEN = 30};+ enum {ARRAY_SEPARATOR = UINT16_MAX};+// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV storestatic constexpr const char* COLLECTION_META_PREFIX = "$CM";static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";diff --git a/include/match_score.h b/include/match_score.hindex cf901000..29e85329 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -35,7 +35,7 @@ struct Match {uint16_t start_offset;char offset_diffs[16];- Match() {+ Match(): words_present(0), distance(0), start_offset(0) {}@@ -44,6 +44,14 @@ struct Match {memcpy(offset_diffs, offset_diffs_stacked, 16);}+ // Construct a single match score from individual components (for multi-field sort)+ inline uint64_t get_match_score(const uint32_t total_cost) const {+ uint64_t match_score = ((int64_t)(words_present) << 24) |+ ((int64_t)(255 - total_cost) << 16) |+ ((int64_t)(distance));+ return match_score;+ }+static void print_token_offsets(std::vector<std::vector<uint16_t>> &token_offsets) {for(auto offsets: token_offsets) {for(auto offset: offsets) {@@ -54,7 +62,8 @@ struct Match {}static inline void addTopOfHeapToWindow(TokenOffsetHeap &heap, std::queue<TokenOffset> &window,- std::vector<std::vector<uint16_t>> &token_offsets, uint16_t *token_offset) {+ const std::vector<std::vector<uint16_t>> &token_offsets,+ uint16_t *token_offset) {TokenOffset top = heap.top();heap.pop();window.push(top);@@ -90,7 +99,7 @@ struct Match {* We use a priority queue to read the offset vectors in a sorted manner, slide a window of a given size, and* compute the max_match and min_displacement of target tokens across the windows.*/- static Match match(uint32_t doc_id, std::vector<std::vector<uint16_t>> &token_offsets) {+ static Match match(uint32_t doc_id, const std::vector<std::vector<uint16_t>> &token_offsets) {std::priority_queue<TokenOffset, std::vector<TokenOffset>, TokenOffset> heap;for(uint8_t token_id=0; token_id < token_offsets.size(); token_id++) {diff --git a/src/collection.cpp b/src/collection.cppindex 5d87140d..f8630eac 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -545,14 +545,9 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sconst std::string & field_name = search_fields[search_fields.size() - field_order_kv.first];field search_field = search_schema.at(field_name);- // only string fields are supported for now- if(search_field.type == field_types::STRING) {- std::vector<std::string> tokens;- StringUtils::split(document[field_name], tokens, " ");-- // positions in the document of each token in the query- std::vector<std::vector<uint16_t>> token_positions;+ if(search_field.type == field_types::STRING || search_field.type == field_types::STRING_ARRAY) {+ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;for (const art_leaf *token_leaf : searched_queries[field_order_kv.second.query_index]) {std::vector<uint16_t> positions;uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.second.key);@@ -560,20 +555,42 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<scontinue;}- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?- token_leaf->values->offsets.getLength() :- token_leaf->values->offset_index.at(doc_index+1);+ uint32_t *indices = new uint32_t[1];+ indices[0] = doc_index;+ leaf_to_indices.emplace(token_leaf, indices);+ }++ // positions in the field of each token in the query+ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ Index::populate_token_positions(searched_queries[field_order_kv.second.query_index],+ leaf_to_indices, 0, array_token_positions);- while(start_offset < end_offset) {- positions.push_back((uint16_t) token_leaf->values->offsets.at(start_offset));- start_offset++;+ Match match;+ uint64_t match_score = 0;+ size_t matched_array_index = 0;++ for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {+ const std::vector<std::vector<uint16_t>> & token_positions = array_token_positions[array_index];++ if(token_positions.empty()) {+ continue;}- token_positions.push_back(positions);+ const Match & this_match = Match::match(field_order_kv.second.key, token_positions);+ uint64_t this_match_score = this_match.get_match_score(1);+ if(this_match_score > match_score) {+ match_score = this_match_score;+ match = this_match;+ matched_array_index = array_index;+ }}- Match match = Match::match(field_order_kv.second.key, token_positions);+ std::vector<std::string> tokens;+ if(search_field.type == field_types::STRING) {+ StringUtils::split(document[field_name], tokens, " ");+ } else {+ StringUtils::split(document[field_name][matched_array_index], tokens, " ");+ }// unpack `match.offset_diffs` into `token_indices`std::vector<size_t> token_indices;@@ -609,6 +626,11 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<swrapper_doc["highlight"] = nlohmann::json::object();wrapper_doc["highlight"][field_name] = snippet_stream.str();++ for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {+ delete [] it->second;+ it->second = nullptr;+ }}result["hits"].push_back(wrapper_doc);diff --git a/src/index.cpp b/src/index.cppindex 2a73a6fe..2cb1efc5 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -225,6 +225,7 @@ void Index::index_string_field(const std::string & text, const uint32_t score, atoken_to_offsets[text].push_back(0);} else {StringUtils::split(text, tokens, " ");+for(uint32_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];string_utils.unicode_normalize(token);@@ -232,6 +233,11 @@ void Index::index_string_field(const std::string & text, const uint32_t score, a}}+ insert_doc(score, t, seq_id, token_to_offsets);+}++void Index::insert_doc(const uint32_t score, art_tree *t, uint32_t seq_id,+ const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const {for(auto & kv: token_to_offsets) {art_document art_doc;art_doc.id = seq_id;@@ -263,9 +269,33 @@ void Index::index_string_field(const std::string & text, const uint32_t score, avoid Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,uint32_t seq_id, const bool verbatim) const {- for(const std::string & str: strings) {- index_string_field(str, score, t, seq_id, verbatim);+ std::unordered_map<std::string, std::unordered_map<size_t, std::vector<uint32_t>>> token_array_positions;++ for(size_t array_index = 0; array_index < strings.size(); array_index++) {+ const std::string & str = strings[array_index];++ std::vector<std::string> tokens;+ StringUtils::split(str, tokens, " ");++ for(uint32_t i=0; i<tokens.size(); i++) {+ auto & token = tokens[i];+ string_utils.unicode_normalize(token);+ token_array_positions[token][array_index].push_back(i);+ }}++ std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;++ for(const auto & kv: token_array_positions) {+ for(size_t array_index = 0; array_index < strings.size(); array_index++) {+ token_to_offsets[kv.first].insert(token_to_offsets[kv.first].end(),+ token_array_positions[kv.first][array_index].begin(),+ token_array_positions[kv.first][array_index].end());+ token_to_offsets[kv.first].push_back(ARRAY_SEPARATOR);+ }+ }++ insert_doc(score, t, seq_id, token_to_offsets);}void Index::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,@@ -850,9 +880,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &char empty_offset_diffs[16];std::fill_n(empty_offset_diffs, 16, 0);Match single_token_match = Match(1, 0, 0, empty_offset_diffs);- const uint64_t single_token_match_score = ((int64_t)(single_token_match.words_present) << 24) |- ((int64_t)(255 - total_cost) << 16) |- ((int64_t)(single_token_match.distance));+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);for(size_t i=0; i<result_size; i++) {const uint32_t seq_id = result_ids[i];@@ -862,14 +890,28 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &if(query_suggestion.size() == 1) {match_score = single_token_match_score;} else {- std::vector<std::vector<uint16_t>> token_positions;- populate_token_positions(query_suggestion, leaf_to_indices, i, token_positions);- const Match & match = Match::match(seq_id, token_positions);-- // Construct a single match score from individual components (for multi-field sort)- match_score = ((int64_t)(match.words_present) << 24) |- ((int64_t)(255 - total_cost) << 16) |- ((int64_t)(match.distance));+ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);++ for(const std::vector<std::vector<uint16_t>> & token_positions: array_token_positions) {+ if(token_positions.size() == 0) {+ continue;+ }+ const Match & match = Match::match(seq_id, token_positions);+ uint64_t this_match_score = match.get_match_score(total_cost);++ if(this_match_score > match_score) {+ match_score = this_match_score;+ }++ /*std::ostringstream os;+ os << name << ", total_cost: " << (255 - total_cost)+ << ", words_present: " << match.words_present+ << ", match_score: " << match_score+ << ", match.distance: " << match.distance+ << ", seq_id: " << seq_id << std::endl;+ std::cout << os.str();*/+ }}const int64_t default_score = 0;@@ -889,15 +931,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;topster.add(seq_id, query_index, match_score, primary_rank_value, secondary_rank_value);-- /*- std::ostringstream os;- os << name << ", total_cost: " << (255 - total_cost)- << ", words_present: " << match.words_present << ", match_score: " << match_score- << ", match.distance: " << match.distance- << ", seq_id: " << seq_id << std::endl;- LOG(INFO) << os.str();- */}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();@@ -910,28 +943,82 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &}void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggestion,- spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,- size_t result_index, std::vector<std::vector<uint16_t>> &token_positions) const {- // for each token in the query, find the positions that it appears in this document+ spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,+ size_t result_index,+ std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions) {++ // array_token_positions:+ // for every element in a potential array, for every token in query suggestion, get the positions++ // first let's ascertain the size of the array+ size_t array_size = 0;+for (const art_leaf *token_leaf : query_suggestion) {- std::vector<uint16_t> positions;- uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];- if(doc_index == token_leaf->values->ids.getLength()) {- continue;+ uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];+ if(doc_index == token_leaf->values->ids.getLength()) {+ continue;+ }++ uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);+ uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?+ token_leaf->values->offsets.getLength() :+ token_leaf->values->offset_index.at(doc_index+1);++ while(start_offset < end_offset) {+ uint16_t pos = (uint16_t) token_leaf->values->offsets.at(start_offset);+ if(pos == ARRAY_SEPARATOR) {+ array_size++;}+ start_offset++;+ }++ if(array_size == 0) {+ // for plain string fields that don't use an ARRAY_SEPARATOR+ array_size = 1;+ }++ break;+ }- uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);- uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?- token_leaf->values->offsets.getLength() :- token_leaf->values->offset_index.at(doc_index+1);+ // initialize array_token_positions+ array_token_positions = std::vector<std::vector<std::vector<uint16_t>>>(array_size);- while(start_offset < end_offset) {- positions.push_back((uint16_t) token_leaf->values->offsets.at(start_offset));- start_offset++;+ // for each token in the query, find the positions that it appears in the array+ for (const art_leaf *token_leaf : query_suggestion) {+ uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];+ if(doc_index == token_leaf->values->ids.getLength()) {+ continue;+ }++ uint32_t start_offset = token_leaf->values->offset_index.at(doc_index);+ uint32_t end_offset = (doc_index == token_leaf->values->ids.getLength() - 1) ?+ token_leaf->values->offsets.getLength() :+ token_leaf->values->offset_index.at(doc_index+1);++ size_t array_index = 0;+ std::vector<uint16_t> positions;++ while(start_offset < end_offset) {+ uint16_t pos = (uint16_t) token_leaf->values->offsets.at(start_offset);+ start_offset++;++ if(pos == ARRAY_SEPARATOR) {+ if(positions.size() != 0) {+ array_token_positions[array_index].push_back(positions);+ positions.clear();+ }+ array_index++;+ continue;}- token_positions.push_back(positions);+ positions.push_back(pos);}++ if(positions.size() != 0) {+ // for plain string fields that don't use an ARRAY_SEPARATOR+ array_token_positions[array_index].push_back(positions);+ }+ }}inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,diff --git a/test/array_text_documents.jsonl b/test/array_text_documents.jsonlnew file mode 100644index 00000000..8cd27b6a--- /dev/null+++ b/test/array_text_documents.jsonl@@ -0,0 +1,3 @@+{"title": "The Truth About Forever", "tags": ["the truth", "about forever", "truth about"], "points": 100}+{"title": "Plain Truth", "tags": ["plain", "truth", "plain truth"], "points": 40}+{"title": "Temple of the Winds", "tags": ["temple", "of", "temple of"], "points": 87}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex c503916d..219a4c83 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -487,6 +487,82 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_EQ("16", results["hits"].at(0)["document"]["id"]);}+TEST_F(CollectionTest, ArrayStringFieldHighlight) {+ Collection *coll_array_text;++ std::ifstream infile(std::string(ROOT_DIR) + "test/array_text_documents.jsonl");+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("tags", field_types::STRING_ARRAY, false),+ field("points", field_types::INT32, false)+ };++ coll_array_text = collectionManager.get_collection("coll_array_text");+ if (coll_array_text == nullptr) {+ coll_array_text = collectionManager.create_collection("coll_array_text", fields, "points").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_array_text->add(json_line);+ }++ infile.close();++ query_fields = {"tags"};+ std::vector<std::string> facets;++ nlohmann::json results = coll_array_text->search("truth about", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, 0).get();+ ASSERT_EQ(1, results["hits"].size());++ std::vector<std::string> ids = {"0"};++ for (size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ ASSERT_STREQ(results["hits"][0]["highlight"]["tags"].get<std::string>().c_str(), "<mark>truth</mark> <mark>about</mark>");++ results = coll_array_text->search("forever truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, 0).get();+ ASSERT_EQ(1, results["hits"].size());++ ids = {"0"};++ for (size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ ASSERT_STREQ(results["hits"][0]["highlight"]["tags"].get<std::string>().c_str(), "the <mark>truth</mark>");++ results = coll_array_text->search("truth", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, 0).get();+ ASSERT_EQ(2, results["hits"].size());++ ids = {"0", "1"};++ for (size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ results = coll_array_text->search("asdadasd", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ false, 0).get();+ ASSERT_EQ(0, results["hits"].size());++ collectionManager.drop_collection("coll_array_text");+}+TEST_F(CollectionTest, MultipleFields) {Collection *coll_mul_fields;