Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Lay foundations for overriding search results.

Kishore Nallanc1b950135161
diff --git a/TODO.md b/TODO.md
index 048da518..69298247 100644
--- a/TODO.md
+++ b/TODO.md
@@ -97,6 +97,8 @@
- ~~Have a LOG(ERROR) level~~
- ~~Handle SIGTERM which is sent when process is killed~~
- ~~Use snappy compression for storage~~
+- atleast 1 token match for proceeding with drop tokens
+- support wildcard query with filters
- API for optimizing on disk storage
- Jemalloc
- Exact search
diff --git a/include/array_utils.h b/include/array_utils.h
index 05390238..81a0576c 100644
--- a/include/array_utils.h
+++ b/include/array_utils.h
@@ -13,4 +13,7 @@ public:
static size_t and_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t **out);
static size_t or_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t **out);
+
+ static size_t exclude_scalar(const uint32_t *src, const size_t lenSrc, const uint32_t *filter, const size_t lenFilter,
+ uint32_t **out);
};
\ No newline at end of file
diff --git a/include/collection.h b/include/collection.h
index 9dae85ae..f2164a11 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -17,6 +17,56 @@
#include <field.h>
#include <option.h>
+
+struct override_t {
+ static const std::string MATCH_EXACT;
+ static const std::string MATCH_CONTAINS;
+
+ struct rule_t {
+ std::string query;
+ std::string match;
+ };
+
+ struct add_hit_t {
+ std::string doc_id;
+ uint32_t position;
+ };
+
+ struct drop_hit_t {
+ std::string doc_id;
+ };
+
+ std::string id;
+ rule_t rule;
+ std::vector<add_hit_t> add_hits;
+ std::vector<drop_hit_t> drop_hits;
+
+ override_t() {}
+
+ override_t(const nlohmann::json & override) {
+ id = override["id"].get<std::string>();
+ rule.query = override["rule"]["query"].get<std::string>();
+ rule.match = override["rule"]["match"].get<std::string>();
+
+ if (override.count("includes") != 0) {
+ for(const auto & include: override["includes"]) {
+ add_hit_t add_hit;
+ add_hit.doc_id = include["id"].get<std::string>();
+ add_hit.position = include["position"].get<uint32_t>();
+ add_hits.push_back(add_hit);
+ }
+ }
+
+ if (override.count("excludes") != 0) {
+ for(const auto & exclude: override["excludes"]) {
+ drop_hit_t drop_hit;
+ drop_hit.doc_id = exclude["id"].get<std::string>();
+ drop_hits.push_back(drop_hit);
+ }
+ }
+ }
+};
+
class Collection {
private:
@@ -35,6 +85,8 @@ private:
}
};
+ std::map<std::string, override_t> overrides;
+
std::string name;
uint32_t collection_id;
@@ -69,11 +121,14 @@ private:
std::string get_seq_id_key(uint32_t seq_id);
void highlight_result(const field &search_field, const std::vector<std::vector<art_leaf *>> &searched_queries,
- const Topster<512>::KV &field_order_kv, const nlohmann::json &document,
+ const KV &field_order_kv, const nlohmann::json &document,
StringUtils & string_utils, highlight_t &highlight);
void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
+ void populate_overrides(std::string query, std::map<uint32_t, size_t> & id_pos_map,
+ std::vector<uint32_t> & included_ids, std::vector<uint32_t> & excluded_ids);
+
public:
Collection() = delete;
@@ -103,7 +158,7 @@ public:
void increment_next_seq_id_field();
- Option<uint32_t> doc_id_to_seq_id(std::string doc_id);
+ Option<uint32_t> doc_id_to_seq_id(const std::string & doc_id);
std::vector<std::string> get_facet_fields();
@@ -135,9 +190,28 @@ public:
Option<std::string> remove(const std::string & id, const bool remove_from_store = true);
+ // FIXME: add persistence
+ bool add_override(override_t & override) {
+ if(overrides.count("id") != 0) {
+ return false;
+ }
+
+ overrides[override.id] = override;
+ return true;
+ }
+
+ bool remove_override(const std::string & id) {
+ if(overrides.count("id") != 0) {
+ overrides.erase(id);
+ return true;
+ }
+
+ return false;
+ }
+
size_t get_num_indices();
- static uint32_t get_seq_id_key(const std::string & key);
+ static uint32_t get_seq_id_from_key(const std::string & key);
Option<uint32_t> index_in_memory(const nlohmann::json & document, uint32_t seq_id);
diff --git a/include/index.h b/include/index.h
index c527f969..39abd0ac 100644
--- a/include/index.h
+++ b/include/index.h
@@ -26,6 +26,8 @@ struct search_args {
std::vector<std::string> search_fields;
std::vector<filter> filters;
std::vector<facet> facets;
+ std::vector<uint32_t> included_ids;
+ std::vector<uint32_t> excluded_ids;
std::vector<sort_by> sort_fields_std;
int num_typos;
size_t max_facet_values;
@@ -34,9 +36,10 @@ struct search_args {
token_ordering token_order;
bool prefix;
size_t drop_tokens_threshold;
- std::vector<Topster<512>::KV> field_order_kvs;
+ std::vector<KV> raw_result_kvs;
size_t all_result_ids_len;
std::vector<std::vector<art_leaf*>> searched_queries;
+ std::vector<KV> override_result_kvs;
Option<uint32_t> outcome;
search_args(): outcome(0) {
@@ -44,10 +47,12 @@ struct search_args {
}
search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,
- std::vector<facet> facets, std::vector<sort_by> sort_fields_std, int num_typos, size_t max_facet_values,
+ std::vector<facet> facets, std::vector<uint32_t> included_ids, std::vector<uint32_t> excluded_ids,
+ std::vector<sort_by> sort_fields_std, int num_typos, size_t max_facet_values,
size_t per_page, size_t page, token_ordering token_order, bool prefix, size_t drop_tokens_threshold):
- query(query), search_fields(search_fields), filters(filters), facets(facets),
- sort_fields_std(sort_fields_std), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),
+ query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
+ excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), num_typos(num_typos),
+ max_facet_values(max_facet_values), per_page(per_page),
page(page), token_order(token_order), prefix(prefix), drop_tokens_threshold(drop_tokens_threshold),
all_result_ids_len(0), outcome(0) {
@@ -132,7 +137,9 @@ private:
Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);
- void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);
+ void do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size);
+
+ void drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> ids);
void search_field(const uint8_t & field_id, std::string & query,
const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
@@ -177,6 +184,10 @@ private:
void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
const uint32_t indices_length);
+ void collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,
+ const std::vector<uint32_t> & included_ids,
+ Topster<32> & curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);
+
public:
Index() = delete;
@@ -187,13 +198,14 @@ public:
void run_search();
- void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,
+ void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> & search_fields,
const std::vector<filter> & filters, std::vector<facet> & facets,
- std::vector<sort_by> sort_fields_std, const int num_typos,
- const size_t per_page, const size_t page,
- const token_ordering token_order, const bool prefix, const size_t drop_tokens_threshold,
- std::vector<Topster<512>::KV> & field_order_kv,
- size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries);
+ const std::vector<uint32_t> & included_ids, const std::vector<uint32_t> & excluded_ids,
+ const std::vector<sort_by> & sort_fields_std, const int num_typos,
+ const size_t per_page, const size_t page, const token_ordering token_order,
+ const bool prefix, const size_t drop_tokens_threshold, std::vector<KV> & raw_result_kvs,
+ size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries,
+ std::vector<KV> & override_result_kvs);
Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document);
diff --git a/include/string_utils.h b/include/string_utils.h
index a4d33c0c..de261652 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -150,6 +150,10 @@ struct StringUtils {
std::transform(str.begin(), str.end(), str.begin(), ::toupper);
}
+ static void tolowercase(std::string& str) {
+ std::transform(str.begin(), str.end(), str.begin(), ::tolower);
+ }
+
void unicode_normalize(std::string& str) const;
/* https://stackoverflow.com/a/34571089/131050 */
diff --git a/include/topster.h b/include/topster.h
index 0e158c5b..0adde65a 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -8,21 +8,21 @@
#include <match_score.h>
#include <number.h>
+struct KV {
+ uint8_t field_id;
+ uint16_t query_index;
+ uint16_t array_index;
+ uint64_t key;
+ uint64_t match_score;
+ number_t primary_attr;
+ number_t secondary_attr;
+};
+
/*
* Remembers the max-K elements seen so far using a min-heap
*/
template <size_t MAX_SIZE=512>
struct Topster {
- struct KV {
- uint8_t field_id;
- uint16_t query_index;
- uint16_t array_index;
- uint64_t key;
- uint64_t match_score;
- number_t primary_attr;
- number_t secondary_attr;
- };
-
KV *data;
uint32_t size;
diff --git a/src/array_utils.cpp b/src/array_utils.cpp
index 4459cac7..9d93fa46 100644
--- a/src/array_utils.cpp
+++ b/src/array_utils.cpp
@@ -41,6 +41,10 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,
const uint32_t *B, const size_t lenB, uint32_t **out) {
size_t indexA = 0, indexB = 0, res_index = 0;
+ if(A == nullptr && B == nullptr) {
+ return 0;
+ }
+
if(A == nullptr) {
*out = new uint32_t[lenB];
memcpy(*out, B, lenB * sizeof(uint32_t));
@@ -95,5 +99,57 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,
memcpy(*out, results, res_index * sizeof(uint32_t));
delete[] results;
+ return res_index;
+}
+
+size_t ArrayUtils::exclude_scalar(const uint32_t *A, const size_t lenA,
+ const uint32_t *B, const size_t lenB, uint32_t **out) {
+ size_t indexA = 0, indexB = 0, res_index = 0;
+
+ if(A == nullptr && B == nullptr) {
+ return 0;
+ }
+
+ if(A == nullptr) {
+ return 0;
+ }
+
+ if(B == nullptr) {
+ *out = new uint32_t[lenA];
+ memcpy(*out, A, lenA * sizeof(uint32_t));
+ return lenA;
+ }
+
+ uint32_t* results = new uint32_t[lenA];
+
+ while (indexA < lenA && indexB < lenB) {
+ if (A[indexA] < B[indexB]) {
+ results[res_index] = A[indexA];
+ res_index++;
+ indexA++;
+ } else {
+ if (A[indexA] == B[indexB]) {
+ indexA++;
+ indexB++;
+ } else {
+ results[res_index] = A[indexA];
+ res_index++;
+ indexA++;
+ indexB++;
+ }
+ }
+ }
+
+ while (indexA < lenA) {
+ results[res_index] = A[indexA];
+ res_index++;
+ indexA++;
+ }
+
+ // shrink fit
+ *out = new uint32_t[res_index];
+ memcpy(*out, results, res_index * sizeof(uint32_t));
+ delete[] results;
+
return res_index;
}
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index e230bfcd..38d7365b 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -13,6 +13,9 @@
#include "topster.h"
#include "logger.h"
+const std::string override_t::MATCH_EXACT = "exact";
+const std::string override_t::MATCH_CONTAINS = "contains";
+
struct match_index_t {
Match match;
uint64_t match_score = 0;
@@ -273,6 +276,33 @@ void Collection::prune_document(nlohmann::json &document, const spp::sparse_hash
}
}
+void Collection::populate_overrides(std::string query, std::map<uint32_t, size_t> & id_pos_map,
+ std::vector<uint32_t> & included_ids, std::vector<uint32_t> & excluded_ids) {
+ StringUtils::tolowercase(query);
+
+ for(const auto & override_kv: overrides) {
+ const auto & override = override_kv.second;
+
+ if( (override.rule.match == override_t::MATCH_EXACT && override.rule.query == query) ||
+ (override.rule.match == override_t::MATCH_CONTAINS && query.find(override.rule.query) != std::string::npos) ) {
+ for(const auto & hit: override.add_hits) {
+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);
+ if(seq_id_op.ok()) {
+ included_ids.push_back(seq_id_op.get());
+ id_pos_map[seq_id_op.get()] = hit.position;
+ }
+ }
+
+ for(const auto & hit: override.drop_hits) {
+ Option<uint32_t> seq_id_op = doc_id_to_seq_id(hit.doc_id);
+ if(seq_id_op.ok()) {
+ excluded_ids.push_back(seq_id_op.get());
+ }
+ }
+ }
+ }
+}
+
Option<nlohmann::json> Collection::search(std::string query, const std::vector<std::string> search_fields,
const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const int num_typos,
@@ -282,6 +312,25 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
const spp::sparse_hash_set<std::string> include_fields,
const spp::sparse_hash_set<std::string> exclude_fields,
const size_t max_facet_values) {
+
+ std::vector<uint32_t> included_ids;
+ std::vector<uint32_t> excluded_ids;
+ std::map<uint32_t, size_t> id_pos_map;
+ populate_overrides(query, id_pos_map, included_ids, excluded_ids);
+
+ std::map<uint32_t, std::vector<uint32_t>> index_to_included_ids;
+ std::map<uint32_t, std::vector<uint32_t>> index_to_excluded_ids;
+
+ for(auto seq_id: included_ids) {
+ auto index_id = (seq_id % num_indices);
+ index_to_included_ids[index_id].push_back(seq_id);
+ }
+
+ for(auto seq_id: excluded_ids) {
+ auto index_id = (seq_id % num_indices);
+ index_to_excluded_ids[index_id].push_back(seq_id);
+ }
+
std::vector<facet> facets;
// validate search fields
@@ -448,22 +497,26 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
return Option<nlohmann::json>(422, message);
}
- // all search queries that were used for generating the results
- std::vector<std::vector<art_leaf*>> searched_queries;
- std::vector<Topster<512>::KV> field_order_kvs;
+ std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the results
+ std::vector<KV> raw_result_kvs;
+ std::vector<KV> override_result_kvs;
+
size_t total_found = 0;
// send data to individual index threads
+ size_t index_id = 0;
for(Index* index: indices) {
- index->search_params = search_args(query, search_fields, filters, facets, sort_fields_std,
- num_typos, max_facet_values, per_page, page, token_order, prefix,
- drop_tokens_threshold);
+ index->search_params = search_args(query, search_fields, filters, facets,
+ index_to_included_ids[index_id], index_to_excluded_ids[index_id],
+ sort_fields_std, num_typos, max_facet_values, per_page, page,
+ token_order, prefix, drop_tokens_threshold);
{
std::lock_guard<std::mutex> lk(index->m);
index->ready = true;
index->processed = false;
}
index->cv.notify_one();
+ index_id++;
}
Option<nlohmann::json> index_search_op({}); // stores the last error across all index threads
@@ -485,9 +538,14 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
continue;
}
- for(auto & field_order_kv: index->search_params.field_order_kvs) {
+ for(auto & field_order_kv: index->search_params.raw_result_kvs) {
field_order_kv.query_index += searched_queries.size();
- field_order_kvs.push_back(field_order_kv);
+ raw_result_kvs.push_back(field_order_kv);
+ }
+
+ for(auto & field_order_kv: index->search_params.override_result_kvs) {
+ field_order_kv.query_index += searched_queries.size();
+ override_result_kvs.push_back(field_order_kv);
}
searched_queries.insert(searched_queries.end(), index->search_params.searched_queries.begin(),
@@ -519,25 +577,60 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
// All fields are sorted descending
- std::sort(field_order_kvs.begin(), field_order_kvs.end(), Topster<>::is_greater_kv_value);
+ std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster<>::is_greater_kv_value);
- nlohmann::json result = nlohmann::json::object();
+ // Sort based on position in overriden list
+ std::sort(
+ override_result_kvs.begin(), override_result_kvs.end(),
+ [&id_pos_map](const KV & a, const KV & b) -> bool {
+ return id_pos_map[a.key] < id_pos_map[b.key];
+ }
+ );
- result["hits"] = nlohmann::json::array();
- result["found"] = total_found;
+ nlohmann::json result = nlohmann::json::object();
const int start_result_index = (page - 1) * per_page;
- const int kvsize = field_order_kvs.size();
+ const int kvsize = raw_result_kvs.size() + override_result_kvs.size();
if(start_result_index > (kvsize - 1)) {
return Option<nlohmann::json>(result);
}
- const int end_result_index = std::min(int(num_results), kvsize) - 1;
+ std::vector<KV> result_kvs;
+ size_t override_kv_index = 0;
+ size_t raw_results_index = 0;
+
+ // merge raw results and override results
+ while(override_kv_index < override_result_kvs.size() && raw_results_index < raw_result_kvs.size()) {
+ if(override_kv_index < override_result_kvs.size() &&
+ id_pos_map.count(override_result_kvs[override_kv_index].key) != 0 &&
+ result_kvs.size() + 1 == id_pos_map[override_result_kvs[override_kv_index].key]) {
+ result_kvs.push_back(override_result_kvs[override_kv_index]);
+ override_kv_index++;
+ } else {
+ result_kvs.push_back(raw_result_kvs[raw_results_index]);
+ raw_results_index++;
+ }
+ }
+
+ while(override_kv_index < override_result_kvs.size()) {
+ result_kvs.push_back(override_result_kvs[override_kv_index]);
+ override_kv_index++;
+ }
+
+ while(raw_results_index < raw_result_kvs.size()) {
+ result_kvs.push_back(raw_result_kvs[raw_results_index]);
+ raw_results_index++;
+ }
+
+ size_t end_result_index = std::min(num_results, result_kvs.size()) - 1;
+
+ result["hits"] = nlohmann::json::array();
+ result["found"] = total_found;
// construct results array
- for(int field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {
- const auto & field_order_kv = field_order_kvs[field_order_kv_index];
+ for(size_t result_kvs_index = start_result_index; result_kvs_index <= end_result_index; result_kvs_index++) {
+ const auto & field_order_kv = result_kvs[result_kvs_index];
const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.key);
std::string json_doc_str;
@@ -634,7 +727,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
void Collection::highlight_result(const field &search_field,
const std::vector<std::vector<art_leaf *>> &searched_queries,
- const Topster<512>::KV & field_order_kv, const nlohmann::json & document,
+ const KV & field_order_kv, const nlohmann::json & document,
StringUtils & string_utils, highlight_t & highlight) {
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
@@ -838,7 +931,7 @@ size_t Collection::get_num_indices() {
return num_indices;
}
-uint32_t Collection::get_seq_id_key(const std::string & key) {
+uint32_t Collection::get_seq_id_from_key(const std::string & key) {
// last 4 bytes of the key would be the serialized version of the sequence id
std::string serialized_seq_id = key.substr(key.length() - 4);
return StringUtils::deserialize_uint32_t(serialized_seq_id);
@@ -875,7 +968,7 @@ uint32_t Collection::get_collection_id() {
return collection_id;
}
-Option<uint32_t> Collection::doc_id_to_seq_id(std::string doc_id) {
+Option<uint32_t> Collection::doc_id_to_seq_id(const std::string & doc_id) {
std::string seq_id_str;
StoreStatus status = store->get(get_doc_id_key(doc_id), seq_id_str);
if(status == StoreStatus::FOUND) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 47331a92..36ced335 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -109,7 +109,7 @@ Option<bool> CollectionManager::init(Store *store,
}
while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
- const uint32_t seq_id = Collection::get_seq_id_key(iter->key().ToString());
+ const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());
nlohmann::json document;
diff --git a/src/index.cpp b/src/index.cpp
index 30a1096e..ff623363 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2,6 +2,7 @@
#include <numeric>
#include <chrono>
+#include <set>
#include <unordered_map>
#include <array_utils.h>
#include <match_score.h>
@@ -509,7 +510,7 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin
}
}
-void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {
+void Index::do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size) {
for(auto & a_facet: facets) {
// assumed that facet fields have already been validated upstream
const field & facet_field = facet_schema.at(a_facet.field_name);
@@ -529,6 +530,28 @@ void Index::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t
}
}
+void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> ids) {
+ for(auto & a_facet: facets) {
+ // assumed that facet fields have already been validated upstream
+ const field & facet_field = facet_schema.at(a_facet.field_name);
+ const facet_value & fvalue = facet_index.at(facet_field.name);
+
+ for(const uint32_t doc_seq_id: ids) {
+ if(fvalue.doc_values.count(doc_seq_id) != 0) {
+ // for every result document, get the values associated and decrement counter
+ const std::vector<uint32_t> & value_indices = fvalue.doc_values.at(doc_seq_id);
+ for(size_t j = 0; j < value_indices.size(); j++) {
+ const std::string & facet_value = fvalue.index_value.at(value_indices.at(j));
+ a_facet.result_map[facet_value] -= 1;
+ if(a_facet.result_map[facet_value] == 0) {
+ a_facet.result_map.erase(facet_value);
+ }
+ }
+ }
+ }
+ }
+}
+
void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
const std::vector<sort_by> & sort_fields,
std::vector<token_candidates> & token_candidates_vec, const token_ordering token_order,
@@ -770,10 +793,11 @@ void Index::run_search() {
// after the wait, we own the lock.
search(search_params.outcome, search_params.query, search_params.search_fields,
- search_params.filters, search_params.facets,
- search_params.sort_fields_std, search_params.num_typos, search_params.per_page, search_params.page,
- search_params.token_order, search_params.prefix, search_params.drop_tokens_threshold,
- search_params.field_order_kvs, search_params.all_result_ids_len, search_params.searched_queries);
+ search_params.filters, search_params.facets, search_params.included_ids,
+ search_params.excluded_ids, search_params.sort_fields_std, search_params.num_typos,
+ search_params.per_page, search_params.page, search_params.token_order,
+ search_params.prefix, search_params.drop_tokens_threshold, search_params.raw_result_kvs,
+ search_params.all_result_ids_len, search_params.searched_queries, search_params.override_result_kvs);
// hand control back to main thread
processed = true;
@@ -785,17 +809,90 @@ void Index::run_search() {
}
}
-void Index::search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,
- const std::vector<filter> & filters, std::vector<facet> & facets,
- std::vector<sort_by> sort_fields_std, const int num_typos,
- const size_t per_page, const size_t page, const token_ordering token_order,
- const bool prefix, const size_t drop_tokens_threshold,
- std::vector<Topster<512>::KV> & field_order_kvs,
- size_t & all_result_ids_len, std::vector<std::vector<art_leaf*>> & searched_queries) {
+void Index::collate_curated_ids(const std::string & query, const std::string & field, const uint8_t field_id,
+ const std::vector<uint32_t> & included_ids,
+ Topster<32> & curated_topster,
+ std::vector<std::vector<art_leaf*>> & searched_queries) {
+
+ if(included_ids.size() == 0) {
+ return;
+ }
+
+ // calculate match_score and add to topster independently
+ std::vector<std::string> tokens;
+ StringUtils::split(query, tokens, " ");
+
+ std::vector<art_leaf *> override_query;
+
+ for(size_t token_index = 0; token_index < tokens.size(); token_index++) {
+ const auto token = tokens[token_index];
+ const size_t token_len = token.length();
+ string_utils.unicode_normalize(tokens[token_index]);
+
+ std::vector<art_leaf*> leaves;
+ art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
+ 0, 0, 1, token_ordering::MAX_SCORE, false, leaves);
+
+ if(leaves.size() > 0) {
+ override_query.push_back(leaves[0]);
+ }
+ }
+
+ if(override_query.size() == 0) {
+ // happens when the curated hit's field has no overlap with query string
+ //return ;
+ }
+
+ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
+
+ for (art_leaf *token_leaf : override_query) {
+ uint32_t *indices = new uint32_t[included_ids.size()];
+ token_leaf->values->ids.indexOf(&included_ids[0], included_ids.size(), indices);
+ leaf_to_indices.emplace(token_leaf, indices);
+ }
+
+ for(size_t j=0; j<included_ids.size(); j++) {
+ const uint32_t seq_id = included_ids[j];
+
+ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
+ populate_token_positions(override_query, leaf_to_indices, j, array_token_positions);
+
+ uint64_t match_score = 0;
+
+ for(const std::vector<std::vector<uint16_t>> & token_positions: array_token_positions) {
+ if(token_positions.size() == 0) {
+ continue;
+ }
+ const Match & match = Match::match(seq_id, token_positions);
+ uint64_t this_match_score = match.get_match_score(0, field_id);
+
+ if(this_match_score > match_score) {
+ match_score = this_match_score;
+ }
+ }
+
+ curated_topster.add(seq_id, field_id, searched_queries.size(), match_score,
+ number_t(int64_t(1)), number_t(int64_t(1)));
+
+ searched_queries.push_back(override_query);
+ }
+}
+
+void Index::search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> & search_fields,
+ const std::vector<filter> & filters, std::vector<facet> & facets,
+ const std::vector<uint32_t> & included_ids,
+ const std::vector<uint32_t> & excluded_ids,
+ const std::vector<sort_by> & sort_fields_std, const int num_typos,
+ const size_t per_page, const size_t page, const token_ordering token_order,
+ const bool prefix, const size_t drop_tokens_threshold,
+ std::vector<KV> & raw_result_kvs,
+ size_t & all_result_ids_len,
+ std::vector<std::vector<art_leaf*>> & searched_queries,
+ std::vector<KV> & override_result_kvs) {
const size_t num_results = (page * per_page);
- // process the filters first
+ // process the filters
uint32_t* filter_ids = nullptr;
Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, filters);
@@ -811,36 +908,66 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec
uint32_t* all_result_ids = nullptr;
Topster<512> topster;
+ Topster<32> curated_topster;
if(query == "*") {
- uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
+ const std::string & field = search_fields[0];
+
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
filter_ids, filter_ids_length);
+ collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);
do_facets(facets, filter_ids, filter_ids_length);
all_result_ids_len = filter_ids_length;
} else {
const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);
for(size_t i = 0; i < num_search_fields; i++) {
- const std::string & field = search_fields[i];
// proceed to query search only when no filters are provided or when filtering produces results
if(filters.size() == 0 || filter_ids_length > 0) {
- uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort results
+ const std::string & field = search_fields[i];
+
search_field(field_id, query, field, filter_ids, filter_ids_length, facets, sort_fields_std,
num_typos, num_results, searched_queries, topster, &all_result_ids, all_result_ids_len,
token_order, prefix, drop_tokens_threshold);
+ collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);
}
}
}
+ do_facets(facets, &included_ids[0], included_ids.size());
+
// must be sorted before iterated upon to remove "empty" array entries
topster.sort();
+ curated_topster.sort();
+
+ std::set<uint32_t> ids_to_remove(included_ids.begin(), included_ids.end());
+ ids_to_remove.insert(excluded_ids.begin(), excluded_ids.end());
+
+ std::vector<uint32_t> dropped_ids;
- // order of fields specified matter: matching docs from earlier fields are more important
+ // loop through topster and remove elements from included and excluded id lists
for(uint32_t t = 0; t < topster.size && t < num_results; t++) {
- Topster<512>::KV* kv = topster.getKV(t);
- field_order_kvs.push_back(*kv);
+ KV* kv = topster.getKV(t);
+
+ if(ids_to_remove.count(kv->key) != 0) {
+ dropped_ids.push_back((uint32_t)kv->key);
+ } else {
+ raw_result_kvs.push_back(*kv);
+ }
}
+ for(uint32_t t = 0; t < curated_topster.size && t < num_results; t++) {
+ KV* kv = curated_topster.getKV(t);
+ override_result_kvs.push_back(*kv);
+ }
+
+ // for the ids that are dropped, remove their corresponding facet components from facet results
+ drop_facets(facets, dropped_ids);
+
+ all_result_ids_len -= dropped_ids.size();
+ all_result_ids_len += curated_topster.size;
+
delete [] filter_ids;
delete [] all_result_ids;
diff --git a/src/replicator.cpp b/src/replicator.cpp
index a3c75e36..69ff52cc 100644
--- a/src/replicator.cpp
+++ b/src/replicator.cpp
@@ -163,7 +163,7 @@ void Replicator::on_replication_event(void *data) {
Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));
nlohmann::json document = nlohmann::json::parse(replication_event->value);
- uint32_t seq_id = Collection::get_seq_id_key(replication_event->key);
+ uint32_t seq_id = Collection::get_seq_id_from_key(replication_event->key);
collection->index_in_memory(document, seq_id);
}
diff --git a/test/array_utils_test.cpp b/test/array_utils_test.cpp
index 018f613d..ccaf3dd0 100644
--- a/test/array_utils_test.cpp
+++ b/test/array_utils_test.cpp
@@ -113,4 +113,48 @@ TEST(SortedArrayTest, OrScalarWithEitherArrayAsNull) {
delete[] results;
results = nullptr;
+}
+
+TEST(SortedArrayTest, FilterArray) {
+ const size_t size1 = 9;
+ uint32_t *arr1 = new uint32_t[size1];
+ for(auto i = 0; i < 9; i++) {
+ arr1[i] = i;
+ }
+
+ std::vector<uint32_t> vec2 = {0, 1, 5, 7, 8};
+ uint32_t *arr2 = new uint32_t[vec2.size()];
+ auto j = 0;
+ for(auto i: vec2) {
+ arr2[j++] = i;
+ }
+
+ uint32_t *results = nullptr;
+ uint32_t results_size = ArrayUtils::exclude_scalar(arr1, size1, arr2, vec2.size(), &results);
+ ASSERT_EQ(4, results_size);
+
+ std::vector<uint32_t> expected = {2, 3, 4, 6};
+
+ for(size_t i = 0; i < results_size; i++) {
+ ASSERT_EQ(expected[i], results[i]);
+ }
+
+ delete[] arr2;
+ delete[] results;
+
+ vec2 = {0, 1, 2, 3, 4, 5, 6, 7, 8};
+ arr2 = new uint32_t[vec2.size()];
+
+ j = 0;
+ for(auto i: vec2) {
+ arr2[j++] = i;
+ }
+
+ results = nullptr;
+ results_size = ArrayUtils::exclude_scalar(arr1, size1, arr2, vec2.size(), &results);
+ ASSERT_EQ(0, results_size);
+
+ delete[] arr2;
+ delete[] arr1;
+ delete[] results;
}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 7729ce3d..d9470efa 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -783,6 +783,134 @@ TEST_F(CollectionTest, MultipleFields) {
collectionManager.drop_collection("coll_mul_fields");
}
+TEST_F(CollectionTest, ExcludeIncludeExactQueryMatch) {
+ Collection *coll_mul_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, false),
+ field("cast", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, false)
+ };
+
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ if(coll_mul_fields == nullptr) {
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_mul_fields->add(json_line);
+ }
+
+ infile.close();
+
+ // with override
+ nlohmann::json override_json = {
+ {"id", "exclude-rule"},
+ {
+ "rule", {
+ {"query", "of"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ }
+ };
+ override_json["excludes"] = nlohmann::json::array();
+ override_json["excludes"][0] = nlohmann::json::object();
+ override_json["excludes"][0]["id"] = "4";
+
+ override_json["excludes"][1] = nlohmann::json::object();
+ override_json["excludes"][1]["id"] = "11";
+
+ override_t override(override_json);
+
+ coll_mul_fields->add_override(override);
+
+ std::vector<std::string> facets = {"cast"};
+
+ Option<nlohmann::json> res_op = coll_mul_fields->search("of", {"title"}, "", facets, sort_fields, 0, 10);
+ ASSERT_TRUE(res_op.ok());
+ nlohmann::json results = res_op.get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(3, results["found"].get<uint32_t>());
+ ASSERT_EQ(6, results["facet_counts"][0]["counts"].size());
+
+ // include
+ nlohmann::json override_json_include = {
+ {"id", "include-rule"},
+ {
+ "rule", {
+ {"query", "in"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ }
+ };
+ override_json_include["includes"] = nlohmann::json::array();
+ override_json_include["includes"][0] = nlohmann::json::object();
+ override_json_include["includes"][0]["id"] = "0";
+ override_json_include["includes"][0]["position"] = 1;
+
+ override_json_include["includes"][1] = nlohmann::json::object();
+ override_json_include["includes"][1]["id"] = "3";
+ override_json_include["includes"][1]["position"] = 2;
+
+ override_t override_include(override_json_include);
+
+ coll_mul_fields->add_override(override_include);
+
+ res_op = coll_mul_fields->search("in", {"title"}, "", {}, sort_fields, 0, 10);
+ ASSERT_TRUE(res_op.ok());
+ results = res_op.get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ(3, results["found"].get<uint32_t>());
+
+ coll_mul_fields->remove_override("exclude-rule");
+ coll_mul_fields->remove_override("include-rule");
+
+ // contains cases
+
+ nlohmann::json override_contains_inc = {
+ {"id", "include-rule"},
+ {
+ "rule", {
+ {"query", "will"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ }
+ };
+ override_contains_inc["includes"] = nlohmann::json::array();
+ override_contains_inc["includes"][0] = nlohmann::json::object();
+ override_contains_inc["includes"][0]["id"] = "0";
+ override_contains_inc["includes"][0]["position"] = 1;
+
+ override_contains_inc["includes"][1] = nlohmann::json::object();
+ override_contains_inc["includes"][1]["id"] = "1";
+ override_contains_inc["includes"][1]["position"] = 7; // purposely setting it way out
+
+ override_t override_inc_contains(override_contains_inc);
+ coll_mul_fields->add_override(override_inc_contains);
+
+ res_op = coll_mul_fields->search("will smith", {"title"}, "", {}, sort_fields, 0, 10);
+ ASSERT_TRUE(res_op.ok());
+ results = res_op.get();
+
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ(4, results["found"].get<uint32_t>());
+
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("2", results["hits"][2]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][3]["document"]["id"].get<std::string>().c_str());
+
+ coll_mul_fields->remove_override("include-rule");
+
+ collectionManager.drop_collection("coll_mul_fields");
+}
+
TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {
Collection *coll_mul_fields;