Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Lay foundations for overriding search results.
Kishore Nallanc1b950135161
diff --git a/TODO.md b/TODO.mdindex 048da518..69298247 100644--- a/TODO.md+++ b/TODO.md@@ -97,6 +97,8 @@- ~~Have a LOG(ERROR) level~~- ~~Handle SIGTERM which is sent when process is killed~~- ~~Use snappy compression for storage~~+- atleast 1 token match for proceeding with drop tokens+- support wildcard query with filters- API for optimizing on disk storage- Jemalloc- Exact searchdiff --git a/include/array_utils.h b/include/array_utils.hindex 05390238..81a0576c 100644--- a/include/array_utils.h+++ b/include/array_utils.h@@ -13,4 +13,7 @@ public:static size_t and_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t **out);static size_t or_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t **out);++ static size_t exclude_scalar(const uint32_t *src, const size_t lenSrc, const uint32_t *filter, const size_t lenFilter,+ uint32_t **out);};\ No newline at end of filediff --git a/include/collection.h b/include/collection.hindex 9dae85ae..f2164a11 100644--- a/include/collection.h+++ b/include/collection.h@@ -17,6 +17,56 @@#include <field.h>#include <option.h>++struct override_t {+ static const std::string MATCH_EXACT;+ static const std::string MATCH_CONTAINS;++ struct rule_t {+ std::string query;+ std::string match;+ };++ struct add_hit_t {+ std::string doc_id;+ uint32_t position;+ };++ struct drop_hit_t {+ std::string doc_id;+ };++ std::string id;+ rule_t rule;+ std::vector<add_hit_t> add_hits;+ std::vector<drop_hit_t> drop_hits;++ override_t() {}++ override_t(const nlohmann::json & override) {+ id = override["id"].get<std::string>();+ rule.query = override["rule"]["query"].get<std::string>();+ rule.match = override["rule"]["match"].get<std::string>();++ if (override.count("includes") != 0) {+ for(const auto & include: override["includes"]) {+ add_hit_t add_hit;+ add_hit.doc_id = include["id"].get<std::string>();+ add_hit.position = include["position"].get<uint32_t>();+ add_hits.push_back(add_hit);+ }+ }++ if (override.count("excludes") != 0) {+ for(const auto & exclude: override["excludes"]) {+ drop_hit_t drop_hit;+ drop_hit.doc_id = exclude["id"].get<std::string>();+ drop_hits.push_back(drop_hit);+ }+ }+ }+};+class Collection {private:@@ -35,6 +85,8 @@ private:}};+ std::map<std::string, override_t> overrides;+std::string name;uint32_t collection_id;@@ -69,11 +121,14 @@ private:std::string get_seq_id_key(uint32_t seq_id);void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,- const Topster<512>::KV &field_order_kv, const nlohmann::json &document,+ const KV &field_order_kv, const nlohmann::json &document,StringUtils & string_utils, highlight_t &highlight);void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);+ void populate_overrides(std::string query, std::map<uint32_t, size_t> & id_pos_map,+ std::vector<uint32_t> & included_ids, std::vector<uint32_t> & excluded_ids);+public:Collection() = delete;@@ -103,7 +158,7 @@ public:void increment_next_seq_id_field();- Option<uint32_t> doc_id_to_seq_id(std::string doc_id);+ Option<uint32_t> doc_id_to_seq_id(const std::string & doc_id);std::vector<std::string> get_facet_fields();@@ -135,9 +190,28 @@ public:Option<std::string> remove(const std::string & id, const bool remove_from_store = true);+ // FIXME: add persistence+ bool add_override(override_t & override) {+ if(overrides.count("id") != 0) {+ return false;+ }++ overrides[override.id] = override;+ return true;+ }++ bool remove_override(const std::string & id) {+ if(overrides.count("id") != 0) {+ overrides.erase(id);+ return true;+ }++ return false;+ }+size_t get_num_indices();- static uint32_t get_seq_id_key(const std::string & key);+ static uint32_t get_seq_id_from_key(const std::string & key);Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);diff --git a/include/index.h b/include/index.hindex c527f969..39abd0ac 100644--- a/include/index.h+++ b/include/index.h@@ -26,6 +26,8 @@ struct search_args {std::vector<std::string> search_fields;std::vector<filter> filters;std::vector<facet> facets;+ std::vector<uint32_t> included_ids;+ std::vector<uint32_t> excluded_ids;std::vector<sort_by> sort_fields_std;int num_typos;size_t max_facet_values;@@ -34,9 +36,10 @@ struct search_args {token_ordering token_order;bool prefix;size_t drop_tokens_threshold;- std::vector<Topster<512>::KV> field_order_kvs;+ std::vector<KV> raw_result_kvs;size_t all_result_ids_len;std::vector<std::vector<art_leaf*>> searched_queries;+ std::vector<KV> override_result_kvs;Option<uint32_t> outcome;search_args(): outcome(0) {@@ -44,10 +47,12 @@ struct search_args {}search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,- std::vector<facet> facets, std::vector<sort_by> sort_fields_std, int num_typos, size_t max_facet_values,+ std::vector<facet> facets, std::vector<uint32_t> included_ids, std::vector<uint32_t> excluded_ids,+ std::vector<sort_by> sort_fields_std, int num_typos, size_t max_facet_values,size_t per_page, size_t page, token_ordering token_order, bool prefix, size_t drop_tokens_threshold):- query(query), search_fields(search_fields), filters(filters), facets(facets),- sort_fields_std(sort_fields_std), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),+ query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),+ excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), num_typos(num_typos),+ max_facet_values(max_facet_values), per_page(per_page),page(page), token_order(token_order), prefix(prefix), drop_tokens_threshold(drop_tokens_threshold),all_result_ids_len(0), outcome(0) {@@ -132,7 +137,9 @@ private:Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);- void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);+ void do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size);++ void drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> ids);void search_field(const uint8_t & field_id, std::string & query,const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,@@ -177,6 +184,10 @@ private:void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,const uint32_t indices_length);+ void collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,+ const std::vector<uint32_t> & included_ids,+ Topster<32> & curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);+public:Index() = delete;@@ -187,13 +198,14 @@ public:void run_search();- void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,+ void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> & search_fields,const std::vector<filter> & filters, std::vector<facet> & facets,- std::vector<sort_by> sort_fields_std, const int num_typos,- const size_t per_page, const size_t page,- const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold,- std::vector<Topster<512>::KV> & field_order_kv,- size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);+ const std::vector<uint32_t> & included_ids, const std::vector<uint32_t> & excluded_ids,+ const std::vector<sort_by> & sort_fields_std, const int num_typos,+ const size_t per_page, const size_t page, const token_ordering token_order,+ const bool prefix, const size_t drop_tokens_threshold, std::vector<KV> & raw_result_kvs,+ size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries,+ std::vector<KV> & override_result_kvs);Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);diff --git a/include/string_utils.h b/include/string_utils.hindex a4d33c0c..de261652 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -150,6 +150,10 @@ struct StringUtils {std::transform(str.begin(), str.end(), str.begin(), ::toupper);}+ static void tolowercase(std::string& str) {+ std::transform(str.begin(), str.end(), str.begin(), ::tolower);+ }+void unicode_normalize(std::string& str) const;/* https://stackoverflow.com/a/34571089/131050 */diff --git a/include/topster.h b/include/topster.hindex 0e158c5b..0adde65a 100644--- a/include/topster.h+++ b/include/topster.h@@ -8,21 +8,21 @@#include <match_score.h>#include <number.h>+struct KV {+ uint8_t field_id;+ uint16_t query_index;+ uint16_t array_index;+ uint64_t key;+ uint64_t match_score;+ number_t primary_attr;+ number_t secondary_attr;+};+/** Remembers the max-K elements seen so far using a min-heap*/template <size_t MAX_SIZE=512>struct Topster {- struct KV {- uint8_t field_id;- uint16_t query_index;- uint16_t array_index;- uint64_t key;- uint64_t match_score;- number_t primary_attr;- number_t secondary_attr;- };-KV *data;uint32_t size;diff --git a/src/array_utils.cpp b/src/array_utils.cppindex 4459cac7..9d93fa46 100644--- a/src/array_utils.cpp+++ b/src/array_utils.cpp@@ -41,6 +41,10 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,const uint32_t *B, const size_t lenB, uint32_t **out) {size_t indexA = 0, indexB = 0, res_index = 0;+ if(A == nullptr && B == nullptr) {+ return 0;+ }+if(A == nullptr) {*out = new uint32_t[lenB];memcpy(*out, B, lenB * sizeof(uint32_t));@@ -95,5 +99,57 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,memcpy(*out, results, res_index * sizeof(uint32_t));delete[] results;+ return res_index;+}++size_t ArrayUtils::exclude_scalar(const uint32_t *A, const size_t lenA,+ const uint32_t *B, const size_t lenB, uint32_t **out) {+ size_t indexA = 0, indexB = 0, res_index = 0;++ if(A == nullptr && B == nullptr) {+ return 0;+ }++ if(A == nullptr) {+ return 0;+ }++ if(B == nullptr) {+ *out = new uint32_t[lenA];+ memcpy(*out, A, lenA * sizeof(uint32_t));+ return lenA;+ }++ uint32_t* results = new uint32_t[lenA];++ while (indexA < lenA && indexB < lenB) {+ if (A[indexA] < B[indexB]) {+ results[res_index] = A[indexA];+ res_index++;+ indexA++;+ } else {+ if (A[indexA] == B[indexB]) {+ indexA++;+ indexB++;+ } else {+ results[res_index] = A[indexA];+ res_index++;+ indexA++;+ indexB++;+ }+ }+ }++ while (indexA < lenA) {+ results[res_index] = A[indexA];+ res_index++;+ indexA++;+ }++ // shrink fit+ *out = new uint32_t[res_index];+ memcpy(*out, results, res_index * sizeof(uint32_t));+ delete[] results;+return res_index;}\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex e230bfcd..38d7365b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -13,6 +13,9 @@#include "topster.h"#include "logger.h"+const std::string override_t::MATCH_EXACT = "exact";+const std::string override_t::MATCH_CONTAINS = "contains";+struct match_index_t {Match match;uint64_t match_score = 0;@@ -273,6 +276,33 @@ void Collection::prune_document(nlohmann::json &document, const spp::sparse_hash}}+void Collection::populate_overrides(std::string query, std::map<uint32_t, size_t> & id_pos_map,+ std::vector<uint32_t> & included_ids, std::vector<uint32_t> & excluded_ids) {+ StringUtils::tolowercase(query);++ for(const auto & override_kv: overrides) {+ const auto & override = override_kv.second;++ if( (override.rule.match == override_t::MATCH_EXACT && override.rule.query == query) ||+ (override.rule.match == override_t::MATCH_CONTAINS && query.find(override.rule.query) != std::string::npos) ) {+ for(const auto & hit: override.add_hits) {+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);+ if(seq_id_op.ok()) {+ included_ids.push_back(seq_id_op.get());+ id_pos_map[seq_id_op.get()] = hit.position;+ }+ }++ for(const auto & hit: override.drop_hits) {+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);+ if(seq_id_op.ok()) {+ excluded_ids.push_back(seq_id_op.get());+ }+ }+ }+ }+}+Option<nlohmann::json> Collection::search(std::string query, const std::vector<std::string> search_fields,const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_by> & sort_fields, const int num_typos,@@ -282,6 +312,25 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sconst spp::sparse_hash_set<std::string> include_fields,const spp::sparse_hash_set<std::string> exclude_fields,const size_t max_facet_values) {++ std::vector<uint32_t> included_ids;+ std::vector<uint32_t> excluded_ids;+ std::map<uint32_t, size_t> id_pos_map;+ populate_overrides(query, id_pos_map, included_ids, excluded_ids);++ std::map<uint32_t, std::vector<uint32_t>> index_to_included_ids;+ std::map<uint32_t, std::vector<uint32_t>> index_to_excluded_ids;++ for(auto seq_id: included_ids) {+ auto index_id = (seq_id % num_indices);+ index_to_included_ids[index_id].push_back(seq_id);+ }++ for(auto seq_id: excluded_ids) {+ auto index_id = (seq_id % num_indices);+ index_to_excluded_ids[index_id].push_back(seq_id);+ }+std::vector<facet> facets;// validate search fields@@ -448,22 +497,26 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sreturn Option<nlohmann::json>(422, message);}- // all search queries that were used for generating the results- std::vector<std::vector<art_leaf*>> searched_queries;- std::vector<Topster<512>::KV> field_order_kvs;+ std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results+ std::vector<KV> raw_result_kvs;+ std::vector<KV> override_result_kvs;+size_t total_found = 0;// send data to individual index threads+ size_t index_id = 0;for(Index* index: indices) {- index->search_params = search_args(query, search_fields, filters, facets, sort_fields_std,- num_typos, max_facet_values, per_page, page, token_order, prefix,- drop_tokens_threshold);+ index->search_params = search_args(query, search_fields, filters, facets,+ index_to_included_ids[index_id], index_to_excluded_ids[index_id],+ sort_fields_std, num_typos, max_facet_values, per_page, page,+ token_order, prefix, drop_tokens_threshold);{std::lock_guard<std::mutex> lk(index->m);index->ready = true;index->processed = false;}index->cv.notify_one();+ index_id++;}Option<nlohmann::json> index_search_op({}); // stores the last error across all index threads@@ -485,9 +538,14 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<scontinue;}- for(auto & field_order_kv: index->search_params.field_order_kvs) {+ for(auto & field_order_kv: index->search_params.raw_result_kvs) {field_order_kv.query_index += searched_queries.size();- field_order_kvs.push_back(field_order_kv);+ raw_result_kvs.push_back(field_order_kv);+ }++ for(auto & field_order_kv: index->search_params.override_result_kvs) {+ field_order_kv.query_index += searched_queries.size();+ override_result_kvs.push_back(field_order_kv);}searched_queries.insert(searched_queries.end(), index->search_params.searched_queries.begin(),@@ -519,25 +577,60 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}// All fields are sorted descending- std::sort(field_order_kvs.begin(), field_order_kvs.end(), Topster<>::is_greater_kv_value);+ std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster<>::is_greater_kv_value);- nlohmann::json result = nlohmann::json::object();+ // Sort based on position in overriden list+ std::sort(+ override_result_kvs.begin(), override_result_kvs.end(),+ [&id_pos_map](const KV & a, const KV & b) -> bool {+ return id_pos_map[a.key] < id_pos_map[b.key];+ }+ );- result["hits"] = nlohmann::json::array();- result["found"] = total_found;+ nlohmann::json result = nlohmann::json::object();const int start_result_index = (page - 1) * per_page;- const int kvsize = field_order_kvs.size();+ const int kvsize = raw_result_kvs.size() + override_result_kvs.size();if(start_result_index > (kvsize - 1)) {return Option<nlohmann::json>(result);}- const int end_result_index = std::min(int(num_results), kvsize) - 1;+ std::vector<KV> result_kvs;+ size_t override_kv_index = 0;+ size_t raw_results_index = 0;++ // merge raw results and override results+ while(override_kv_index < override_result_kvs.size() && raw_results_index < raw_result_kvs.size()) {+ if(override_kv_index < override_result_kvs.size() &&+ id_pos_map.count(override_result_kvs[override_kv_index].key) != 0 &&+ result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index].key]) {+ result_kvs.push_back(override_result_kvs[override_kv_index]);+ override_kv_index++;+ } else {+ result_kvs.push_back(raw_result_kvs[raw_results_index]);+ raw_results_index++;+ }+ }++ while(override_kv_index < override_result_kvs.size()) {+ result_kvs.push_back(override_result_kvs[override_kv_index]);+ override_kv_index++;+ }++ while(raw_results_index < raw_result_kvs.size()) {+ result_kvs.push_back(raw_result_kvs[raw_results_index]);+ raw_results_index++;+ }++ size_t end_result_index = std::min(num_results, result_kvs.size()) - 1;++ result["hits"] = nlohmann::json::array();+ result["found"] = total_found;// construct results array- for(int field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {- const auto & field_order_kv = field_order_kvs[field_order_kv_index];+ for(size_t result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {+ const auto & field_order_kv = result_kvs[result_kvs_index];const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);std::string json_doc_str;@@ -634,7 +727,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<svoid Collection::highlight_result(const field &search_field,const std::vector<std::vector<art_leaf *>> &searched_queries,- const Topster<512>::KV & field_order_kv, const nlohmann::json & document,+ const KV & field_order_kv, const nlohmann::json & document,StringUtils & string_utils, highlight_t & highlight) {spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;@@ -838,7 +931,7 @@ size_t Collection::get_num_indices() {return num_indices;}-uint32_t Collection::get_seq_id_key(const std::string & key) {+uint32_t Collection::get_seq_id_from_key(const std::string & key) {// last 4 bytes of the key would be the serialized version of the sequence idstd::string serialized_seq_id = key.substr(key.length() - 4);return StringUtils::deserialize_uint32_t(serialized_seq_id);@@ -875,7 +968,7 @@ uint32_t Collection::get_collection_id() {return collection_id;}-Option<uint32_t> Collection::doc_id_to_seq_id(std::string doc_id) {+Option<uint32_t> Collection::doc_id_to_seq_id(const std::string & doc_id) {std::string seq_id_str;StoreStatus status = store->get(get_doc_id_key(doc_id), seq_id_str);if(status == StoreStatus::FOUND) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 47331a92..36ced335 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -109,7 +109,7 @@ Option<bool> CollectionManager::init(Store *store,}while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {- const uint32_t seq_id = Collection::get_seq_id_key(iter->key().ToString());+ const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());nlohmann::json document;diff --git a/src/index.cpp b/src/index.cppindex 30a1096e..ff623363 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2,6 +2,7 @@#include <numeric>#include <chrono>+#include <set>#include <unordered_map>#include <array_utils.h>#include <match_score.h>@@ -509,7 +510,7 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin}}-void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {+void Index::do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size) {for(auto & a_facet: facets) {// assumed that facet fields have already been validated upstreamconst field & facet_field = facet_schema.at(a_facet.field_name);@@ -529,6 +530,28 @@ void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t}}+void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> ids) {+ for(auto & a_facet: facets) {+ // assumed that facet fields have already been validated upstream+ const field & facet_field = facet_schema.at(a_facet.field_name);+ const facet_value & fvalue = facet_index.at(facet_field.name);++ for(const uint32_t doc_seq_id: ids) {+ if(fvalue.doc_values.count(doc_seq_id) != 0) {+ // for every result document, get the values associated and decrement counter+ const std::vector<uint32_t> & value_indices = fvalue.doc_values.at(doc_seq_id);+ for(size_t j = 0; j < value_indices.size(); j++) {+ const std::string & facet_value = fvalue.index_value.at(value_indices.at(j));+ a_facet.result_map[facet_value] -= 1;+ if(a_facet.result_map[facet_value] == 0) {+ a_facet.result_map.erase(facet_value);+ }+ }+ }+ }+ }+}+void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,const std::vector<sort_by> & sort_fields,std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,@@ -770,10 +793,11 @@ void Index::run_search() {// after the wait, we own the lock.search(search_params.outcome, search_params.query, search_params.search_fields,- search_params.filters, search_params.facets,- search_params.sort_fields_std, search_params.num_typos, search_params.per_page, search_params.page,- search_params.token_order, search_params.prefix, search_params.drop_tokens_threshold,- search_params.field_order_kvs, search_params.all_result_ids_len, search_params.searched_queries);+ search_params.filters, search_params.facets, search_params.included_ids,+ search_params.excluded_ids, search_params.sort_fields_std, search_params.num_typos,+ search_params.per_page, search_params.page, search_params.token_order,+ search_params.prefix, search_params.drop_tokens_threshold, search_params.raw_result_kvs,+ search_params.all_result_ids_len, search_params.searched_queries, search_params.override_result_kvs);// hand control back to main threadprocessed = true;@@ -785,17 +809,90 @@ void Index::run_search() {}}-void Index::search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,- const std::vector<filter> & filters, std::vector<facet> & facets,- std::vector<sort_by> sort_fields_std, const int num_typos,- const size_t per_page, const size_t page, const token_ordering token_order,- const bool prefix, const size_t drop_tokens_threshold,- std::vector<Topster<512>::KV> & field_order_kvs,- size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {+void Index::collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,+ const std::vector<uint32_t> & included_ids,+ Topster<32> & curated_topster,+ std::vector<std::vector<art_leaf*>> & searched_queries) {++ if(included_ids.size() == 0) {+ return;+ }++ // calculate match_score and add to topster independently+ std::vector<std::string> tokens;+ StringUtils::split(query, tokens, " ");++ std::vector<art_leaf *> override_query;++ for(size_t token_index = 0; token_index < tokens.size(); token_index++) {+ const auto token = tokens[token_index];+ const size_t token_len = token.length();+ string_utils.unicode_normalize(tokens[token_index]);++ std::vector<art_leaf*> leaves;+ art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,+ 0, 0, 1, token_ordering::MAX_SCORE, false, leaves);++ if(leaves.size() > 0) {+ override_query.push_back(leaves[0]);+ }+ }++ if(override_query.size() == 0) {+ // happens when the curated hit's field has no overlap with query string+ //return ;+ }++ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;++ for (art_leaf *token_leaf : override_query) {+ uint32_t *indices = new uint32_t[included_ids.size()];+ token_leaf->values->ids.indexOf(&included_ids[0], included_ids.size(), indices);+ leaf_to_indices.emplace(token_leaf, indices);+ }++ for(size_t j=0; j<included_ids.size(); j++) {+ const uint32_t seq_id = included_ids[j];++ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ populate_token_positions(override_query, leaf_to_indices, j, array_token_positions);++ uint64_t match_score = 0;++ for(const std::vector<std::vector<uint16_t>> & token_positions: array_token_positions) {+ if(token_positions.size() == 0) {+ continue;+ }+ const Match & match = Match::match(seq_id, token_positions);+ uint64_t this_match_score = match.get_match_score(0, field_id);++ if(this_match_score > match_score) {+ match_score = this_match_score;+ }+ }++ curated_topster.add(seq_id, field_id, searched_queries.size(), match_score,+ number_t(int64_t(1)), number_t(int64_t(1)));++ searched_queries.push_back(override_query);+ }+}++void Index::search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> & search_fields,+ const std::vector<filter> & filters, std::vector<facet> & facets,+ const std::vector<uint32_t> & included_ids,+ const std::vector<uint32_t> & excluded_ids,+ const std::vector<sort_by> & sort_fields_std, const int num_typos,+ const size_t per_page, const size_t page, const token_ordering token_order,+ const bool prefix, const size_t drop_tokens_threshold,+ std::vector<KV> & raw_result_kvs,+ size_t & all_result_ids_len,+ std::vector<std::vector<art_leaf*>> & searched_queries,+ std::vector<KV> & override_result_kvs) {const size_t num_results = (page * per_page);- // process the filters first+ // process the filtersuint32_t* filter_ids = nullptr;Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, filters);@@ -811,36 +908,66 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vecuint32_t* all_result_ids = nullptr;Topster<512> topster;+ Topster<32> curated_topster;if(query == "*") {- uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);+ const std::string & field = search_fields[0];+score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},filter_ids, filter_ids_length);+ collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);do_facets(facets, filter_ids, filter_ids_length);all_result_ids_len = filter_ids_length;} else {const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);for(size_t i = 0; i < num_search_fields; i++) {- const std::string & field = search_fields[i];// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.size() == 0 || filter_ids_length > 0) {- uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort results+ const std::string & field = search_fields[i];+search_field(field_id, query, field, filter_ids, filter_ids_length, facets, sort_fields_std,num_typos, num_results, searched_queries, topster, &all_result_ids, all_result_ids_len,token_order, prefix, drop_tokens_threshold);+ collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);}}}+ do_facets(facets, &included_ids[0], included_ids.size());+// must be sorted before iterated upon to remove "empty" array entriestopster.sort();+ curated_topster.sort();++ std::set<uint32_t> ids_to_remove(included_ids.begin(), included_ids.end());+ ids_to_remove.insert(excluded_ids.begin(), excluded_ids.end());++ std::vector<uint32_t> dropped_ids;- // order of fields specified matter: matching docs from earlier fields are more important+ // loop through topster and remove elements from included and excluded id listsfor(uint32_t t = 0; t < topster.size && t < num_results; t++) {- Topster<512>::KV* kv = topster.getKV(t);- field_order_kvs.push_back(*kv);+ KV* kv = topster.getKV(t);++ if(ids_to_remove.count(kv->key) != 0) {+ dropped_ids.push_back((uint32_t)kv->key);+ } else {+ raw_result_kvs.push_back(*kv);+ }}+ for(uint32_t t = 0; t < curated_topster.size && t < num_results; t++) {+ KV* kv = curated_topster.getKV(t);+ override_result_kvs.push_back(*kv);+ }++ // for the ids that are dropped, remove their corresponding facet components from facet results+ drop_facets(facets, dropped_ids);++ all_result_ids_len -= dropped_ids.size();+ all_result_ids_len += curated_topster.size;+delete [] filter_ids;delete [] all_result_ids;diff --git a/src/replicator.cpp b/src/replicator.cppindex a3c75e36..69ff52cc 100644--- a/src/replicator.cpp+++ b/src/replicator.cpp@@ -163,7 +163,7 @@ void Replicator::on_replication_event(void *data) {Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));nlohmann::json document = nlohmann::json::parse(replication_event->value);- uint32_t seq_id = Collection::get_seq_id_key(replication_event->key);+ uint32_t seq_id = Collection::get_seq_id_from_key(replication_event->key);collection->index_in_memory(document, seq_id);}diff --git a/test/array_utils_test.cpp b/test/array_utils_test.cppindex 018f613d..ccaf3dd0 100644--- a/test/array_utils_test.cpp+++ b/test/array_utils_test.cpp@@ -113,4 +113,48 @@ TEST(SortedArrayTest, OrScalarWithEitherArrayAsNull) {delete[] results;results = nullptr;+}++TEST(SortedArrayTest, FilterArray) {+ const size_t size1 = 9;+ uint32_t *arr1 = new uint32_t[size1];+ for(auto i = 0; i < 9; i++) {+ arr1[i] = i;+ }++ std::vector<uint32_t> vec2 = {0, 1, 5, 7, 8};+ uint32_t *arr2 = new uint32_t[vec2.size()];+ auto j = 0;+ for(auto i: vec2) {+ arr2[j++] = i;+ }++ uint32_t *results = nullptr;+ uint32_t results_size = ArrayUtils::exclude_scalar(arr1, size1, arr2, vec2.size(), &results);+ ASSERT_EQ(4, results_size);++ std::vector<uint32_t> expected = {2, 3, 4, 6};++ for(size_t i = 0; i < results_size; i++) {+ ASSERT_EQ(expected[i], results[i]);+ }++ delete[] arr2;+ delete[] results;++ vec2 = {0, 1, 2, 3, 4, 5, 6, 7, 8};+ arr2 = new uint32_t[vec2.size()];++ j = 0;+ for(auto i: vec2) {+ arr2[j++] = i;+ }++ results = nullptr;+ results_size = ArrayUtils::exclude_scalar(arr1, size1, arr2, vec2.size(), &results);+ ASSERT_EQ(0, results_size);++ delete[] arr2;+ delete[] arr1;+ delete[] results;}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex 7729ce3d..d9470efa 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -783,6 +783,134 @@ TEST_F(CollectionTest, MultipleFields) {collectionManager.drop_collection("coll_mul_fields");}+TEST_F(CollectionTest, ExcludeIncludeExactQueryMatch) {+ Collection *coll_mul_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("cast", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, false)+ };++ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ if(coll_mul_fields == nullptr) {+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_mul_fields->add(json_line);+ }++ infile.close();++ // with override+ nlohmann::json override_json = {+ {"id", "exclude-rule"},+ {+ "rule", {+ {"query", "of"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };+ override_json["excludes"] = nlohmann::json::array();+ override_json["excludes"][0] = nlohmann::json::object();+ override_json["excludes"][0]["id"] = "4";++ override_json["excludes"][1] = nlohmann::json::object();+ override_json["excludes"][1]["id"] = "11";++ override_t override(override_json);++ coll_mul_fields->add_override(override);++ std::vector<std::string> facets = {"cast"};++ Option<nlohmann::json> res_op = coll_mul_fields->search("of", {"title"}, "", facets, sort_fields, 0, 10);+ ASSERT_TRUE(res_op.ok());+ nlohmann::json results = res_op.get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(3, results["found"].get<uint32_t>());+ ASSERT_EQ(6, results["facet_counts"][0]["counts"].size());++ // include+ nlohmann::json override_json_include = {+ {"id", "include-rule"},+ {+ "rule", {+ {"query", "in"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };+ override_json_include["includes"] = nlohmann::json::array();+ override_json_include["includes"][0] = nlohmann::json::object();+ override_json_include["includes"][0]["id"] = "0";+ override_json_include["includes"][0]["position"] = 1;++ override_json_include["includes"][1] = nlohmann::json::object();+ override_json_include["includes"][1]["id"] = "3";+ override_json_include["includes"][1]["position"] = 2;++ override_t override_include(override_json_include);++ coll_mul_fields->add_override(override_include);++ res_op = coll_mul_fields->search("in", {"title"}, "", {}, sort_fields, 0, 10);+ ASSERT_TRUE(res_op.ok());+ results = res_op.get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(3, results["found"].get<uint32_t>());++ coll_mul_fields->remove_override("exclude-rule");+ coll_mul_fields->remove_override("include-rule");++ // contains cases++ nlohmann::json override_contains_inc = {+ {"id", "include-rule"},+ {+ "rule", {+ {"query", "will"},+ {"match", override_t::MATCH_CONTAINS}+ }+ }+ };+ override_contains_inc["includes"] = nlohmann::json::array();+ override_contains_inc["includes"][0] = nlohmann::json::object();+ override_contains_inc["includes"][0]["id"] = "0";+ override_contains_inc["includes"][0]["position"] = 1;++ override_contains_inc["includes"][1] = nlohmann::json::object();+ override_contains_inc["includes"][1]["id"] = "1";+ override_contains_inc["includes"][1]["position"] = 7; // purposely setting it way out++ override_t override_inc_contains(override_contains_inc);+ coll_mul_fields->add_override(override_inc_contains);++ res_op = coll_mul_fields->search("will smith", {"title"}, "", {}, sort_fields, 0, 10);+ ASSERT_TRUE(res_op.ok());+ results = res_op.get();++ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ(4, results["found"].get<uint32_t>());++ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("2", results["hits"][2]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][3]["document"]["id"].get<std::string>().c_str());++ coll_mul_fields->remove_override("include-rule");++ collectionManager.drop_collection("coll_mul_fields");+}+TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {Collection *coll_mul_fields;