Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
When multiple fields are searched, the same document should not be returned twice.
Kishore Nallan1d1cd2459be2
diff --git a/include/index.h b/include/index.hindex 3346cec9..291f0a76 100644--- a/include/index.h+++ b/include/index.h@@ -33,7 +33,7 @@ struct search_args {token_ordering token_order;bool prefix;size_t drop_tokens_threshold;- std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;+ std::vector<Topster<512>::KV> field_order_kvs;size_t all_result_ids_len;std::vector<std::vector<art_leaf*>> searched_queries;Option<uint32_t> outcome;@@ -84,7 +84,8 @@ private:void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);- void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,+ void search_field(const uint8_t & field_id, std::string & query,+ const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,std::vector<facet> & facets, const std::vector<sort_by> & sort_fields,const int num_typos, const size_t num_results,std::vector<std::vector<art_leaf*>> & searched_queries,@@ -92,7 +93,7 @@ private:size_t & all_result_ids_len, const token_ordering token_order = FREQUENCY,const bool prefix = false, const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD);- void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,+ void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,const token_ordering token_order, std::vector<std::vector<art_leaf*>> & searched_queries,Topster<512> & topster, size_t & total_results, uint32_t** all_result_ids,@@ -141,7 +142,7 @@ public:std::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page,const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold,- std::vector<std::pair<int, Topster<512>::KV>> & field_order_kv,+ std::vector<Topster<512>::KV> & field_order_kv,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);@@ -151,13 +152,17 @@ public:size_t result_index,std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions);- void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint32_t total_cost,- Topster<512> &topster, const std::vector<art_leaf *> & query_suggestion,+ void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint8_t & field_id,+ const uint32_t total_cost, Topster<512> &topster, const std::vector<art_leaf *> & query_suggestion,const uint32_t *result_ids, const size_t result_size) const;Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id, int32_t points);- static const int SEARCH_LIMIT_NUM = 100; // for limiting number of results on multiple candidates / query rewrites+ // for limiting number of results on multiple candidates / query rewrites+ enum {SEARCH_LIMIT_NUM = 100};++ // for limiting number of fields that can be searched on+ enum {FIELD_LIMIT_NUM = 100};// If the number of results found is less than this threshold, Typesense will attempt to drop the tokens// in the query that have the least individual hits one by one until enough results are found.diff --git a/include/topster.h b/include/topster.hindex 8b51ab14..52191547 100644--- a/include/topster.h+++ b/include/topster.h@@ -14,7 +14,8 @@template <size_t MAX_SIZE=512>struct Topster {struct KV {- uint16_t query_index;+ uint8_t query_index;+ uint8_t field_id;uint64_t key;uint64_t match_score;number_t primary_attr;@@ -35,8 +36,8 @@ struct Topster {b = c;}- void add(const uint64_t &key, const uint16_t &query_index, const uint64_t &match_score, const number_t &primary_attr,- const number_t &secondary_attr) {+ void add(const uint64_t &key, const uint8_t &query_index, const uint8_t &field_id, const uint64_t &match_score,+ const number_t &primary_attr, const number_t &secondary_attr) {if (size >= MAX_SIZE) {if(!is_greater(data[0], match_score, primary_attr, secondary_attr)) {// when incoming value is less than the smallest in the heap, ignore@@ -53,6 +54,7 @@ struct Topster {data[0].key = key;data[0].query_index = query_index;+ data[0].field_id = field_id;data[0].match_score = match_score;data[0].primary_attr = primary_attr;data[0].secondary_attr = secondary_attr;@@ -83,6 +85,7 @@ struct Topster {data[size].key = key;data[size].query_index = query_index;+ data[size].field_id = field_id;data[size].match_score = match_score;data[size].primary_attr = primary_attr;data[size].secondary_attr = secondary_attr;@@ -106,10 +109,11 @@ struct Topster {}static bool is_greater_kv(const struct KV &i, const struct KV &j) {- return std::tie(i.match_score, i.primary_attr, i.secondary_attr) >- std::tie(j.match_score, j.primary_attr, j.secondary_attr);+ return std::tie(i.match_score, i.primary_attr, i.secondary_attr, i.field_id, i.key) >+ std::tie(j.match_score, j.primary_attr, j.secondary_attr, j.field_id, j.key);}+ // topster must be sorted before iterated upon to remove dead array entriesvoid sort() {std::stable_sort(std::begin(data), std::begin(data) + size, is_greater_kv);}diff --git a/src/collection.cpp b/src/collection.cppindex 85ee8d45..00308aaf 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -9,6 +9,7 @@#include <thread>#include <chrono>#include <rocksdb/write_batch.h>+#include "topster.h"#include "logger.h"Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,@@ -416,7 +417,9 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sreturn Option<nlohmann::json>(422, message);}- if((page * per_page) > MAX_RESULTS) {+ const size_t num_results = (page * per_page);++ if(num_results > MAX_RESULTS) {std::string message = "Only the first " + std::to_string(MAX_RESULTS) + " results are available.";return Option<nlohmann::json>(422, message);}@@ -425,7 +428,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// all search queries that were used for generating the resultsstd::vector<std::vector<art_leaf*>> searched_queries;- std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;+ std::vector<Topster<512>::KV> field_order_kvs;size_t total_found = 0;// send data to individual index threads@@ -460,9 +463,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<scontinue;}- // need to remap the search query index before appendingfor(auto & field_order_kv: index->search_params.field_order_kvs) {- field_order_kv.second.query_index += searched_queries.size();+ field_order_kv.query_index += searched_queries.size();field_order_kvs.push_back(field_order_kv);}@@ -495,11 +497,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}// All fields are sorted descending- std::sort(field_order_kvs.begin(), field_order_kvs.end(),- [](const std::pair<int, Topster<512>::KV> & a, const std::pair<int, Topster<512>::KV> & b) {- return std::tie(a.second.match_score, a.second.primary_attr, a.second.secondary_attr, a.first, a.second.key) >- std::tie(b.second.match_score, b.second.primary_attr, b.second.secondary_attr, b.first, b.second.key);- });+ std::sort(field_order_kvs.begin(), field_order_kvs.end(), Topster<>::is_greater_kv);nlohmann::json result = nlohmann::json::object();@@ -513,12 +511,12 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sreturn Option<nlohmann::json>(result);}- const int end_result_index = std::min(int(page * per_page), kvsize) - 1;+ const int end_result_index = std::min(int(num_results), kvsize) - 1;// construct results arrayfor(int field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {const auto & field_order_kv = field_order_kvs[field_order_kv_index];- const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);std::string json_doc_str;StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);@@ -538,19 +536,19 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}wrapper_doc["document"] = document;- //wrapper_doc["match_score"] = field_order_kv.second.match_score;- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.second.key;+ //wrapper_doc["match_score"] = field_order_kv.match_score;+ //wrapper_doc["seq_id"] = (uint32_t) field_order_kv.key;// highlight query words in the result- const std::string & field_name = search_fields[search_fields.size() - field_order_kv.first];+ const std::string & field_name = search_fields[Index::FIELD_LIMIT_NUM - field_order_kv.field_id];field search_field = search_schema.at(field_name);if(search_field.type == field_types::STRING || search_field.type == field_types::STRING_ARRAY) {spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;- for (const art_leaf *token_leaf : searched_queries[field_order_kv.second.query_index]) {+ for (const art_leaf *token_leaf : searched_queries[field_order_kv.query_index]) {std::vector<uint16_t> positions;- uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.second.key);+ uint32_t doc_index = token_leaf->values->ids.indexOf(field_order_kv.key);if(doc_index == token_leaf->values->ids.getLength()) {continue;}@@ -562,7 +560,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// positions in the field of each token in the querystd::vector<std::vector<std::vector<uint16_t>>> array_token_positions;- Index::populate_token_positions(searched_queries[field_order_kv.second.query_index],+ Index::populate_token_positions(searched_queries[field_order_kv.query_index],leaf_to_indices, 0, array_token_positions);Match match;@@ -576,7 +574,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<scontinue;}- const Match & this_match = Match::match(field_order_kv.second.key, token_positions);+ const Match & this_match = Match::match(field_order_kv.key, token_positions);uint64_t this_match_score = this_match.get_match_score(1);if(this_match_score > match_score) {match_score = this_match_score;diff --git a/src/index.cpp b/src/index.cppindex 2cb1efc5..9909bffc 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -346,12 +346,12 @@ void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t}}-void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,- const std::vector<sort_by> & sort_fields,- std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,- std::vector<std::vector<art_leaf*>> & searched_queries, Topster<512> & topster,- size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,- const size_t & max_results, const bool prefix) {+void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,+ const std::vector<sort_by> & sort_fields,+ std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,+ std::vector<std::vector<art_leaf*>> & searched_queries, Topster<512> & topster,+ size_t & total_results, uint32_t** all_result_ids, size_t & all_result_ids_len,+ const size_t & max_results, const bool prefix) {const long long combination_limit = 10;auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };@@ -403,7 +403,7 @@ void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, stdo_facets(facets, filtered_result_ids, filtered_results_size);// go through each matching document id and calculate match score- score_results(sort_fields, searched_queries.size(), total_cost, topster, query_suggestion,+ score_results(sort_fields, searched_queries.size(), field_id, total_cost, topster, query_suggestion,filtered_result_ids, filtered_results_size);delete[] filtered_result_ids;@@ -417,7 +417,7 @@ void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, stdelete [] *all_result_ids;*all_result_ids = new_all_result_ids;- score_results(sort_fields, searched_queries.size(), total_cost, topster, query_suggestion,+ score_results(sort_fields, searched_queries.size(), field_id, total_cost, topster, query_suggestion,result_ids, result_size);delete[] result_ids;}@@ -596,7 +596,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vecstd::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page, const token_ordering token_order,const bool prefix, const size_t drop_tokens_threshold,- std::vector<std::pair<int, Topster<512>::KV>> & field_order_kvs,+ std::vector<Topster<512>::KV> & field_order_kvs,size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {const size_t num_results = (page * per_page);@@ -616,21 +616,27 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec//auto begin = std::chrono::high_resolution_clock::now();uint32_t* all_result_ids = nullptr;- for(size_t i = 0; i < search_fields.size(); i++) {- Topster<512> topster;+ Topster<512> topster;++ const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);+ for(size_t i = 0; i < num_search_fields; i++) {const std::string & field = search_fields[i];// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.size() == 0 || filter_ids_length > 0) {- search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,- searched_queries, topster, &all_result_ids, all_result_ids_len, token_order, prefix,- drop_tokens_threshold);- topster.sort();+ uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);+ search_field(field_id, query, field, filter_ids, filter_ids_length, facets, sort_fields_std,+ num_typos, num_results, searched_queries, topster, &all_result_ids, all_result_ids_len,+ token_order, prefix, drop_tokens_threshold);}+ }- // order of fields specified matter: matching docs from earlier fields are more important- for(uint32_t t = 0; t < topster.size && t < num_results; t++) {- field_order_kvs.push_back(std::make_pair(search_fields.size() - i, topster.getKV(t)));- }+ // must be sorted before iterated upon to remove "empty" array entries+ topster.sort();++ // order of fields specified matter: matching docs from earlier fields are more important+ for(uint32_t t = 0; t < topster.size && t < num_results; t++) {+ const Topster<512>::KV &kv = topster.getKV(t);+ field_order_kvs.push_back(kv);}delete [] filter_ids;@@ -639,7 +645,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();//!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";- outcome = Option<uint32_t>(field_order_kvs.size());+ outcome = Option<uint32_t>(1);}/*@@ -651,11 +657,12 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec4. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/-void Index::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,- std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,- const size_t num_results, std::vector<std::vector<art_leaf*>> & searched_queries,- Topster<512> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,- const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold) {+void Index::search_field(const uint8_t & field_id, std::string & query, const std::string & field,+ uint32_t *filter_ids, size_t filter_ids_length,+ std::vector<facet> & facets, const std::vector<sort_by> & sort_fields, const int num_typos,+ const size_t num_results, std::vector<std::vector<art_leaf*>> & searched_queries,+ Topster<512> &topster, uint32_t** all_result_ids, size_t & all_result_ids_len,+ const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold) {std::vector<std::string> tokens;StringUtils::split(query, tokens, " ");@@ -770,7 +777,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_if(token_candidates_vec.size() != 0 && token_candidates_vec.size() == tokens.size()) {// If all tokens were found, go ahead and search for candidates with what we have so far- search_candidates(filter_ids, filter_ids_length, facets, sort_fields, token_candidates_vec,+ search_candidates(field_id, filter_ids, filter_ids_length, facets, sort_fields, token_candidates_vec,token_order, searched_queries, topster, total_results, all_result_ids, all_result_ids_len,Index::SEARCH_LIMIT_NUM, prefix);@@ -804,7 +811,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_truncated_query += " " + token_count_pairs.at(i).first;}- return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos,+ return search_field(field_id, truncated_query, field, filter_ids, filter_ids_length, facets, sort_fields, num_typos,num_results, searched_queries, topster, all_result_ids, all_result_ids_len,token_order, prefix);}@@ -822,7 +829,7 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect}}-void Index::score_results(const std::vector<sort_by> & sort_fields, const int & query_index,+void Index::score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const uint8_t & field_id,const uint32_t total_cost, Topster<512> & topster,const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, const size_t result_size) const {@@ -930,7 +937,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;- topster.add(seq_id, query_index, match_score, primary_rank_value, secondary_rank_value);+ topster.add(seq_id, query_index, field_id, match_score, primary_rank_value, secondary_rank_value);}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/test/collection_test.cpp b/test/collection_test.cppindex f7b1b8d2..e0387dc0 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -671,6 +671,18 @@ TEST_F(CollectionTest, MultipleFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}+ // when a token exists in multiple fields of the same document, document should be returned only once+ query_fields = {"starring", "title", "cast"};+ results = coll_mul_fields->search("myers", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ids = {"17"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+collectionManager.drop_collection("coll_mul_fields");}diff --git a/test/multi_field_documents.jsonl b/test/multi_field_documents.jsonlindex 71a7bda8..b052c93a 100644--- a/test/multi_field_documents.jsonl+++ b/test/multi_field_documents.jsonl@@ -15,4 +15,4 @@{"title": "Odd Thomas", "starring": "Matthew Page", "points": 69, "cast": ["Anton Yelchin", "Ashley Sommers"] }{"title": "Suffering Man's Charity", "starring": "Henry Thomas", "points": 69, "cast": ["Alan Cumming", "Alison Guh", "David Boreanaz"] }{"title": "The Gospel According to St. Matthew", "starring": "Paola Tedesco", "points": 79, "cast": ["Enrique Irazoqui", "Margherita Caruso"] }-{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Pleasence", "points": 52, "cast": ["Danielle Harris", "Ellie Cornell"] }\ No newline at end of file+{"title": "Halloween 5: The Revenge of Michael Myers", "starring": "Donald Myers", "points": 52, "cast": ["Danielle Harris", "Ellie Cornell"] }\ No newline at end of filediff --git a/test/topster_test.cpp b/test/topster_test.cppindex f09f0417..e4e865d2 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -6,36 +6,37 @@ TEST(TopsterTest, StoreMaxIntValuesWithoutRepetition) {Topster<5> topster;struct {- uint16_t query_index;+ uint8_t query_index;+ uint8_t field_id;uint64_t key;uint64_t match_score;int64_t primary_attr;int64_t secondary_attr;} data[14] = {- {0, 1, 10, 20, 30},- {0, 1, 10, 20, 30},- {0, 2, 4, 20, 30},- {2, 3, 7, 20, 30},- {0, 4, 11, 20, 30},- {1, 5, 9, 20, 30},- {1, 5, 9, 20, 30},- {1, 5, 9, 20, 30},- {0, 6, 6, 20, 30},- {2, 7, 6, 22, 30},- {2, 7, 6, 22, 30},- {1, 8, 9, 20, 30},- {0, 9, 8, 20, 30},- {3, 10, 5, 20, 30},+ {0, 1, 1, 10, 20, 30},+ {0, 1, 1, 10, 20, 30},+ {0, 1, 2, 4, 20, 30},+ {2, 1, 3, 7, 20, 30},+ {0, 1, 4, 11, 20, 30},+ {1, 1, 5, 9, 20, 30},+ {1, 1, 5, 9, 20, 30},+ {1, 1, 5, 9, 20, 30},+ {0, 1, 6, 6, 20, 30},+ {2, 1, 7, 6, 22, 30},+ {2, 1, 7, 6, 22, 30},+ {1, 1, 8, 9, 20, 30},+ {0, 1, 9, 8, 20, 30},+ {3, 1, 10, 5, 20, 30},};for(int i = 0; i < 14; i++) {- topster.add(data[i].key, data[i].query_index, data[i].match_score, data[i].primary_attr,+ topster.add(data[i].key, data[i].query_index, 1, data[i].match_score, data[i].primary_attr,data[i].secondary_attr);}topster.sort();- std::vector<uint64_t> ids = {4, 1, 5, 8, 9};+ std::vector<uint64_t> ids = {4, 1, 8, 5, 9};for(uint32_t i = 0; i < topster.size; i++) {EXPECT_EQ(ids[i], topster.getKeyAt(i));@@ -67,7 +68,7 @@ TEST(TopsterTest, StoreMaxFloatValuesWithoutRepetition) {};for(int i = 0; i < 12; i++) {- topster.add(data[i].key, data[i].query_index, data[i].match_score, data[i].primary_attr,+ topster.add(data[i].key, data[i].query_index, 1, data[i].match_score, data[i].primary_attr,data[i].secondary_attr);}