Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactored the facet implementation and as well as the query interface.

Kishore Nallan7af95e7f2295
diff --git a/TODO.md b/TODO.md
index e53459c7..2d6f9044 100644
--- a/TODO.md
+++ b/TODO.md
@@ -21,35 +21,37 @@
- ~~storage key prefix should include collection name~~
- ~~Index and search on multi-valued field~~
- ~~range search for art_int~~
-- Proper score field for ranking tokens
-- Support nested fields via "."
- ~~Restore records as well on restart (like for meta)~~
- ~~drop collection should remove all records from the store~~
-- Pagination parameter
-- UTF-8 support for fuzzy search
- ~~Multi-key binary search during scoring~~
- ~~Assumption that all tokens match for scoring is no longer true~~
-- Handle searching for non-existing fields gracefully
-- Intersection without unpacking
- ~~Filters~~
- ~~Facets~~
+- Prevent string copy during indexing
+- Schema validation during insertion
+- clean special chars before indexing
+- Proper score field for ranking tokens
+- Minimum results should be a variable instead of blindly going with max_results
+- Pagination parameter
- Iterator
- Highlight
+- UTF-8 support for fuzzy search
+- Handle searching for non-existing fields gracefully
+- test for same match score but different primary, secondary attr
+- only last token should be prefix searched
+- Intersection without unpacking
+- Support nested fields via "."
- Support search operators like +, - etc.
- Prefix-search strings should not be null terminated
- string_utils::tokenize should not have max length
-- only last token should be prefix searched
- art float search
-- Minimum results should be a variable instead of blindly going with max_results
- Benchmark with -ffast-math
- Space sensitivity
- Use bitmap index instead of compressed array for doc list
- Throw errors when schema is broken
- Primary_rank_scores and secondary_rank_scores hashmaps should be combined
- Proper logging
-- clean special chars before indexing
- d-ary heap?
-- test for same match score but different primary, secondary attr
**API**
diff --git a/include/collection.h b/include/collection.h
index 0749782e..05f7fa64 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -19,17 +19,19 @@ private:
// Auto incrementing record ID used internally for indexing - not exposed to the client
uint32_t next_seq_id;
- spp::sparse_hash_map<std::string, field> schema;
+ spp::sparse_hash_map<std::string, field> search_schema;
+
+ spp::sparse_hash_map<std::string, field> facet_schema;
std::vector<std::string> rank_fields;
Store* store;
- spp::sparse_hash_map<std::string, art_tree*> index_map;
+ spp::sparse_hash_map<std::string, art_tree*> search_index;
- spp::sparse_hash_map<uint32_t, int64_t> primary_rank_scores;
+ spp::sparse_hash_map<std::string, art_tree*> facet_index;
- spp::sparse_hash_map<uint32_t, int64_t> secondary_rank_scores;
+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> rank_index;
std::string get_doc_id_key(std::string doc_id);
@@ -46,17 +48,21 @@ private:
void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);
- void search(uint32_t *filter_ids, size_t filter_ids_length, std::vector<facet> &facets, std::string &query,
- const std::string &field, const int num_typos, const size_t num_results, Topster<100> &topster,
- size_t &num_found, const token_ordering token_order = FREQUENCY, const bool prefix = false);
+ void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
+ std::vector<facet> & facets, const std::vector<std::string> & rank_fields, const int num_typos,
+ const size_t num_results, Topster<100> &topster, size_t & num_found,
+ const token_ordering token_order = FREQUENCY, const bool prefix = false);
void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
- int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
+ const std::vector<std::string> & rank_fields, int & token_rank,
+ std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
size_t & total_results, size_t & num_found, const size_t & max_results);
- void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+ void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,
+ const bool verbatim) const;
- void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+ void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
+ uint32_t seq_id, const bool verbatim) const;
void index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const;
@@ -73,7 +79,8 @@ public:
Collection() = delete;
Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> & search_fields, const std::vector<std::string> rank_fields);
+ const std::vector<field> & search_fields, const std::vector<field> & facet_fields,
+ const std::vector<std::string> & rank_fields);
~Collection();
@@ -89,20 +96,24 @@ public:
uint32_t doc_id_to_seq_id(std::string doc_id);
+ std::vector<std::string> get_facet_fields();
+
std::vector<std::string> get_rank_fields();
spp::sparse_hash_map<std::string, field> get_schema();
std::string add(std::string json_str);
- nlohmann::json search(std::string query, const std::vector<std::string> fields, const std::string &simple_filter_query,
- std::vector<facet> & facets, const int num_typos, const size_t num_results,
- const token_ordering token_order = FREQUENCY, const bool prefix = false);
+ nlohmann::json search(std::string query, const std::vector<std::string> search_fields,
+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
+ const std::vector<std::string> & rank_fields, const int num_typos,
+ const size_t num_results, const token_ordering token_order = FREQUENCY, const bool prefix = false);
void remove(std::string id);
- void score_results(Topster<100> &topster, const int & token_rank, const std::vector<art_leaf *> &query_suggestion,
- const uint32_t *result_ids, const size_t result_size) const;
+ void score_results(const std::vector<std::string> & rank_fields, const int & token_rank, Topster<100> &topster,
+ const std::vector<art_leaf *> & query_suggestion, const uint32_t *result_ids,
+ const size_t result_size) const;
void index_in_memory(const nlohmann::json &document, uint32_t seq_id);
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 675cdf30..2191ca8e 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -21,6 +21,7 @@ private:
static constexpr const char* COLLECTION_NAME_KEY = "name";
static constexpr const char* COLLECTION_ID_KEY = "id";
static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "search_fields";
+ static constexpr const char* COLLECTION_FACET_FIELDS_KEY = "facet_fields";
static constexpr const char* COLLECTION_RANK_FIELDS_KEY = "rank_fields";
CollectionManager();
@@ -39,7 +40,8 @@ public:
void init(Store *store);
Collection* create_collection(std::string name, const std::vector<field> & search_fields,
- const std::vector<std::string> & rank_fields);
+ const std::vector<field> & facet_fields,
+ const std::vector<std::string> & rank_fields);
Collection* get_collection(std::string collection_name);
diff --git a/src/collection.cpp b/src/collection.cpp
index 8492206e..5d9db218 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -8,23 +8,46 @@
#include <art.h>
Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> &search_fields, const std::vector<std::string> rank_fields):
+ const std::vector<field> &search_fields, const std::vector<field> & facet_fields,
+ const std::vector<std::string> & rank_fields):
name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store), rank_fields(rank_fields) {
for(const field& field: search_fields) {
art_tree *t = new art_tree;
art_tree_init(t);
- index_map.emplace(field.name, t);
- schema.emplace(field.name, field);
+ search_index.emplace(field.name, t);
+ search_schema.emplace(field.name, field);
+ }
+
+ for(const field& field: facet_fields) {
+ art_tree *t = new art_tree;
+ art_tree_init(t);
+ facet_index.emplace(field.name, t);
+ facet_schema.emplace(field.name, field);
+ }
+
+ for(const std::string & rank_field: rank_fields) {
+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
+ rank_index.emplace(rank_field, doc_to_score);
}
}
Collection::~Collection() {
- for(std::pair<std::string, field> name_field: schema) {
- art_tree *t = index_map.at(name_field.first);
+ for(std::pair<std::string, field> name_field: search_schema) {
+ art_tree *t = search_index.at(name_field.first);
art_tree_destroy(t);
t = nullptr;
}
+
+ for(std::pair<std::string, field> name_field: facet_schema) {
+ art_tree *t = facet_index.at(name_field.first);
+ art_tree_destroy(t);
+ t = nullptr;
+ }
+
+ for(std::pair<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> name_map: rank_index) {
+ delete name_map.second;
+ }
}
uint32_t Collection::get_next_seq_id() {
@@ -50,18 +73,20 @@ std::string Collection::add(std::string json_str) {
}
void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id) {
- for(const std::pair<std::string, field> & field_pair: schema) {
- const std::string & field_name = field_pair.first;
- art_tree *t = index_map.at(field_name);
+ // FIXME: field might not exist in the document or field type might be invalid - need to validate!
- uint32_t points = 0;
- if(document.count("points") != 0) {
- points = document["points"];
- }
+ uint32_t points = 0;
+ if(document.count("points") != 0) {
+ points = document["points"];
+ }
+
+ for(const std::pair<std::string, field> & field_pair: search_schema) {
+ const std::string & field_name = field_pair.first;
+ art_tree *t = search_index.at(field_name);
if(field_pair.second.type == field_types::STRING) {
const std::string & text = document[field_name];
- index_string_field(text, points, t, seq_id);
+ index_string_field(text, points, t, seq_id, false);
} else if(field_pair.second.type == field_types::INT32) {
uint32_t value = document[field_name];
index_int32_field(value, points, t, seq_id);
@@ -70,7 +95,7 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id
index_int64_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
std::vector<std::string> strings = document[field_name];
- index_string_array_field(strings, points, t, seq_id);
+ index_string_array_field(strings, points, t, seq_id, false);
} else if(field_pair.second.type == field_types::INT32_ARRAY) {
std::vector<int32_t> values = document[field_name];
index_int32_array_field(values, points, t, seq_id);
@@ -80,12 +105,24 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id
}
}
- if(rank_fields.size() > 0 && document.count(rank_fields[0])) {
- primary_rank_scores[seq_id] = document[rank_fields[0]].get<int64_t>();
+ for(const std::pair<std::string, field> & field_pair: facet_schema) {
+ const std::string & field_name = field_pair.first;
+ art_tree *t = facet_index.at(field_name);
+ if(field_pair.second.type == field_types::STRING) {
+ const std::string & text = document[field_name];
+ index_string_field(text, points, t, seq_id, true);
+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {
+ std::vector<std::string> strings = document[field_name];
+ index_string_array_field(strings, points, t, seq_id, true);
+ }
}
- if(rank_fields.size() > 1 && document.count(rank_fields[1])) {
- secondary_rank_scores[seq_id] = document[rank_fields[1]].get<int64_t>();
+ for(const std::string & rank_field: rank_fields) {
+ if(rank_index.count(rank_field) > 0) {
+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = rank_index.at(rank_field);
+ doc_to_score->emplace(seq_id, document[rank_fields[0]].get<int64_t>());
+ }
+ // FIXME: handle else (return error)
}
}
@@ -135,15 +172,21 @@ void Collection::index_int64_field(const int64_t value, uint32_t score, art_tree
art_insert(t, key, KEY_LEN, &art_doc, num_hits);
}
-void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const {
+void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t,
+ uint32_t seq_id, const bool verbatim) const {
std::vector<std::string> tokens;
- StringUtils::tokenize(text, tokens, " ", true);
-
std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
- for(uint32_t i=0; i<tokens.size(); i++) {
- auto token = tokens[i];
- transform(token.begin(), token.end(), token.begin(), tolower);
- token_to_offsets[token].push_back(i);
+
+ if(verbatim) {
+ tokens.push_back(text);
+ token_to_offsets[text].push_back(0);
+ } else {
+ StringUtils::tokenize(text, tokens, " ", true);
+ for(uint32_t i=0; i<tokens.size(); i++) {
+ auto token = tokens[i];
+ transform(token.begin(), token.end(), token.begin(), tolower);
+ token_to_offsets[token].push_back(i);
+ }
}
for(auto & kv: token_to_offsets) {
@@ -175,9 +218,9 @@ void Collection::index_string_field(const std::string & text, const uint32_t sco
}
void Collection::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
- uint32_t seq_id) const {
+ uint32_t seq_id, const bool verbatim) const {
for(const std::string & str: strings) {
- index_string_field(str, score, t, seq_id);
+ index_string_field(str, score, t, seq_id, verbatim);
}
}
@@ -198,11 +241,11 @@ void Collection::index_int64_array_field(const std::vector<int64_t> & values, co
void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {
for(auto & a_facet: facets) {
// assumed that facet fields have already been validated upstream
- const field & facet_field = schema.at(a_facet.field_name);
+ const field & facet_field = facet_schema.at(a_facet.field_name);
// loop through the field, get all keys and intersect those ids with result ids
- if(index_map.count(facet_field.name) != 0) {
- art_tree *t = index_map.at(facet_field.name);
+ if(facet_index.count(facet_field.name) != 0) {
+ art_tree *t = facet_index.at(facet_field.name);
std::vector<art_leaf *> leaves;
art_topk_iter(t->root, MAX_SCORE, 10, leaves);
@@ -226,9 +269,9 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, si
}
void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,
- int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves,
- Topster<100> & topster, size_t & total_results, size_t & num_found,
- const size_t & max_results) {
+ const std::vector<std::string> & rank_fields, int & token_rank,
+ std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,
+ size_t & total_results, size_t & num_found, const size_t & max_results) {
const size_t combination_limit = 10;
auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);
@@ -261,7 +304,7 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
do_facets(facets, filtered_result_ids, filtered_results_size);
// go through each matching document id and calculate match score
- score_results(topster, token_rank, query_suggestion, filtered_result_ids, filtered_results_size);
+ score_results(rank_fields, token_rank, topster, query_suggestion, filtered_result_ids, filtered_results_size);
num_found += filtered_results_size;
delete[] filtered_result_ids;
@@ -269,7 +312,7 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
} else {
do_facets(facets, result_ids, result_size);
- score_results(topster, token_rank, query_suggestion, result_ids, result_size);
+ score_results(rank_fields, token_rank, topster, query_suggestion, result_ids, result_size);
num_found += result_size;
delete[] result_ids;
}
@@ -317,11 +360,11 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
}
const std::string & field_name = expression_parts[0];
- if(schema.count(field_name) == 0) {
+ if(search_schema.count(field_name) == 0) {
return Option<>(400, "Could not find a filter field named `" + field_name + "` in the schema.");
}
- field _field = schema.at(field_name);
+ field _field = search_schema.at(field_name);
const std::string & raw_value = expression_parts[1];
filter f;
@@ -381,9 +424,9 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
// process the filters first
for(const filter & a_filter: filters) {
- if(index_map.count(a_filter.field_name) != 0) {
- art_tree* t = index_map.at(a_filter.field_name);
- field f = schema.at(a_filter.field_name);
+ if(search_index.count(a_filter.field_name) != 0) {
+ art_tree* t = search_index.at(a_filter.field_name);
+ field f = search_schema.at(a_filter.field_name);
std::vector<const art_leaf*> leaves;
if(f.integer()) {
@@ -425,22 +468,41 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
return Option<>(filter_ids_length);
}
-nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields,
- const std::string & simple_filter_query, std::vector<facet> & facets,
- const int num_typos, const size_t num_results, const token_ordering token_order,
- const bool prefix) {
+nlohmann::json Collection::search(std::string query, const std::vector<std::string> search_fields,
+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
+ const std::vector<std::string> & rank_fields, const int num_typos,
+ const size_t num_results, const token_ordering token_order, const bool prefix) {
size_t num_found = 0;
nlohmann::json result = nlohmann::json::object();
+ std::vector<facet> facets;
+
+ // validate search fields
+ for(const std::string & field_name: search_fields) {
+ if(search_schema.count(field_name) == 0) {
+ result["error"] = "Could not find a search field named `" + field_name + "` in the schema.";
+ return result;
+ }
+
+ field search_field = search_schema.at(field_name);
+ if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {
+ result["error"] = "Search field `" + field_name + "` should be a string or a string array.";
+ return result;
+ }
+ }
// validate facet fields
- for(const facet & a_facet: facets) {
- if(schema.count(a_facet.field_name) == 0) {
- result["error"] = "Could not find a facet field named `" + a_facet.field_name + "` in the schema.";
+ for(const std::string & field_name: facet_fields) {
+ if(facet_schema.count(field_name) == 0) {
+ result["error"] = "Could not find a facet field named `" + field_name + "` in the schema.";
return result;
}
- field facet_field = schema.at(a_facet.field_name);
- if(facet_field.type != field_types::STRING && facet_field.type != field_types::STRING_ARRAY) {
- result["error"] = "Facet field `" + a_facet.field_name + "` should be a string or a string array.";
+ facets.push_back(facet(field_name));
+ }
+
+ // validate rank fields
+ for(const std::string & field_name: rank_fields) {
+ if(rank_index.count(field_name) == 0) {
+ result["error"] = "Could not find a rank field named `" + field_name + "` in the schema.";
return result;
}
}
@@ -459,18 +521,18 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
auto begin = std::chrono::high_resolution_clock::now();
std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;
- for(int i = 0; i < fields.size(); i++) {
+ for(int i = 0; i < search_fields.size(); i++) {
Topster<100> topster;
- const std::string & field = fields[i];
+ const std::string & field = search_fields[i];
// proceed to query search only when no filters are provided or when filtering produces results
if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
- search(filter_ids, filter_ids_length, facets, query, field, num_typos, num_results, topster, num_found,
- token_order, prefix);
+ search_field(query, field, filter_ids, filter_ids_length, facets, rank_fields, num_typos, num_results,
+ topster, num_found, token_order, prefix);
topster.sort();
}
for(auto t = 0; t < topster.size && t < num_results; t++) {
- field_order_kvs.push_back(std::make_pair(fields.size() - i, topster.getKV(t)));
+ field_order_kvs.push_back(std::make_pair(search_fields.size() - i, topster.getKV(t)));
}
}
@@ -497,14 +559,22 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
result["found"] = num_found;
- result["facets"] = nlohmann::json::array();
+ result["facet_counts"] = nlohmann::json::array();
// populate facets
for(const facet & a_facet: facets) {
nlohmann::json facet_result = nlohmann::json::object();
facet_result["field_name"] = a_facet.field_name;
- facet_result["counts"] = a_facet.result_map;
- result["facets"].push_back(facet_result);
+ facet_result["counts"] = nlohmann::json::array();
+
+ for(auto kv: a_facet.result_map) {
+ nlohmann::json facet_value_count = nlohmann::json::object();
+ facet_value_count["value"] = kv.first;
+ facet_value_count["count"] = kv.second;
+ facet_result["counts"].push_back(facet_value_count);
+ }
+
+ result["facet_counts"].push_back(facet_result);
}
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
@@ -522,9 +592,10 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vector<facet> &facets, std::string &query,
- const std::string &field, const int num_typos, const size_t num_results, Topster<100> &topster,
- size_t &num_found, const token_ordering token_order, const bool prefix) {
+void Collection::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
+ std::vector<facet> & facets, const std::vector<std::string> & rank_fields, const int num_typos,
+ const size_t num_results, Topster<100> &topster, size_t & num_found,
+ const token_ordering token_order, const bool prefix) {
std::vector<std::string> tokens;
StringUtils::tokenize(query, tokens, " ", true);
@@ -590,7 +661,7 @@ void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vec
std::cout << "\n";
}
- art_fuzzy_search(index_map.at(field), (const unsigned char *) token.c_str(), token_len,
+ art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
costs[token_index], costs[token_index], 3, token_order, prefix, leaves);
if(!leaves.empty()) {
@@ -627,7 +698,7 @@ void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vec
if(token_leaves.size() != 0 && token_leaves.size() == tokens.size()) {
// If all tokens were found, go ahead and search for candidates with what we have so far
- search_candidates(filter_ids, filter_ids_length, facets, token_rank, token_leaves, topster,
+ search_candidates(filter_ids, filter_ids_length, facets, rank_fields, token_rank, token_leaves, topster,
total_results, num_found, max_results);
if (total_results >= max_results) {
@@ -661,8 +732,8 @@ void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vec
}
}
- return search(filter_ids, filter_ids_length, facets, truncated_query, field, num_typos, num_results, topster,
- num_found, token_order, prefix);
+ return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, rank_fields, num_typos,
+ num_results, topster, num_found, token_order, prefix);
}
}
@@ -677,9 +748,9 @@ void Collection::log_leaves(const int cost, const std::string &token, const std:
}
}
-void Collection::score_results(Topster<100> & topster, const int & token_rank,
- const std::vector<art_leaf *> &query_suggestion, const uint32_t *result_ids,
- const size_t result_size) const {
+void Collection::score_results(const std::vector<std::string> & rank_fields, const int & token_rank,
+ Topster<100> & topster, const std::vector<art_leaf *> &query_suggestion,
+ const uint32_t *result_ids, const size_t result_size) const {
const int max_token_rank = 250;
spp::sparse_hash_map<art_leaf*, uint32_t*> leaf_to_indices;
@@ -693,6 +764,18 @@ void Collection::score_results(Topster<100> & topster, const int & token_rank,
}
}
+ spp::sparse_hash_map<uint32_t, int64_t> * primary_rank_scores = nullptr;
+ spp::sparse_hash_map<uint32_t, int64_t> * secondary_rank_scores = nullptr;
+
+ if(rank_fields.size() > 0) {
+ // assumed that rank field exists in the index - checked earlier in the chain
+ primary_rank_scores = rank_index.at(rank_fields[0]);
+ }
+
+ if(rank_fields.size() > 1) {
+ secondary_rank_scores = rank_index.at(rank_fields[1]);
+ }
+
for(auto i=0; i<result_size; i++) {
uint32_t seq_id = result_ids[i];
std::vector<std::vector<uint16_t>> token_positions;
@@ -733,8 +816,9 @@ void Collection::score_results(Topster<100> & topster, const int & token_rank,
((uint64_t)(mscore.words_present) << 8) +
(MAX_SEARCH_TOKENS - mscore.distance);
- int64_t primary_rank_score = primary_rank_scores.count(seq_id) > 0 ? primary_rank_scores.at(seq_id) : 0;
- int64_t secondary_rank_score = secondary_rank_scores.count(seq_id) > 0 ? secondary_rank_scores.at(seq_id) : 0;
+ int64_t primary_rank_score = primary_rank_scores->count(seq_id) > 0 ? primary_rank_scores->at(seq_id) : 0;
+ int64_t secondary_rank_score = (secondary_rank_scores && secondary_rank_scores->count(seq_id) > 0) ?
+ secondary_rank_scores->at(seq_id) : 0;
topster.add(seq_id, match_score, primary_rank_score, secondary_rank_score);
/*std::cout << "token_rank_score: " << token_rank_score << ", match_score: "
<< match_score << ", primary_rank_score: " << primary_rank_score << ", seq_id: " << seq_id << std::endl;*/
@@ -818,7 +902,7 @@ void Collection::remove(std::string id) {
const unsigned char *key = (const unsigned char *) token.c_str();
int key_len = (int) (token.length() + 1);
- art_leaf* leaf = (art_leaf *) art_search(index_map.at("title"), key, key_len);
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at("title"), key, key_len);
if(leaf != NULL) {
uint32_t seq_id_values[1] = {seq_id};
@@ -841,7 +925,7 @@ void Collection::remove(std::string id) {
std::cout << "----" << std::endl;*/
if(leaf->values->ids.getLength() == 0) {
- art_delete(index_map.at("title"), key, key_len);
+ art_delete(search_index.at("title"), key, key_len);
}
}
}
@@ -880,12 +964,21 @@ uint32_t Collection::doc_id_to_seq_id(std::string doc_id) {
return seq_id;
}
+std::vector<std::string> Collection::get_facet_fields() {
+ std::vector<std::string> facet_fields_copy;
+ for(auto it = facet_schema.begin(); it != facet_schema.end(); ++it) {
+ facet_fields_copy.push_back(it->first);
+ }
+
+ return facet_fields_copy;
+}
+
std::vector<std::string> Collection::get_rank_fields() {
return rank_fields;
}
spp::sparse_hash_map<std::string, field> Collection::get_schema() {
- return schema;
+ return search_schema;
};
std::string Collection::get_meta_key(std::string collection_name) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 475b81b4..96b2cafd 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -28,11 +28,18 @@ void CollectionManager::init(Store *store) {
std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();
std::vector<field> search_fields;
- nlohmann::json fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];
+ nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];
- for (nlohmann::json::iterator it = fields_map.begin(); it != fields_map.end(); ++it) {
+ for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {
search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
}
+
+ std::vector<field> facet_fields;
+ nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];
+
+ for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {
+ facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
+ }
std::string collection_next_seq_id_str;
store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);
@@ -46,6 +53,7 @@ void CollectionManager::init(Store *store) {
collection_next_seq_id,
store,
search_fields,
+ facet_fields,
collection_rank_fields);
// Fetch records from the store and re-create memory index
@@ -70,7 +78,8 @@ void CollectionManager::init(Store *store) {
}
Collection* CollectionManager::create_collection(std::string name, const std::vector<field> & search_fields,
- const std::vector<std::string> & rank_fields) {
+ const std::vector<field> & facet_fields,
+ const std::vector<std::string> & rank_fields) {
if(store->contains(Collection::get_meta_key(name))) {
return nullptr;
}
@@ -85,12 +94,21 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve
search_fields_json.push_back(field_val);
}
+ nlohmann::json facet_fields_json = nlohmann::json::array();;
+ for(const field& facet_field: facet_fields) {
+ nlohmann::json field_val;
+ field_val[fields::name] = facet_field.name;
+ field_val[fields::type] = facet_field.type;
+ facet_fields_json.push_back(field_val);
+ }
+
collection_meta[COLLECTION_NAME_KEY] = name;
collection_meta[COLLECTION_ID_KEY] = next_collection_id;
collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = search_fields_json;
+ collection_meta[COLLECTION_FACET_FIELDS_KEY] = facet_fields_json;
collection_meta[COLLECTION_RANK_FIELDS_KEY] = rank_fields;
- Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, rank_fields);
+ Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, facet_fields, rank_fields);
store->insert(Collection::get_meta_key(name), collection_meta.dump());
store->insert(Collection::get_next_seq_id_key(name), std::to_string(0));
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 70bcadbe..e1fc5c83 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -10,7 +10,8 @@ protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
Collection *collection1;
- std::vector<field> fields;
+ std::vector<field> search_fields;
+ std::vector<field> facet_fields;
std::vector<std::string> rank_fields;
void setupCollection() {
@@ -21,9 +22,11 @@ protected:
store = new Store(state_dir_path);
collectionManager.init(store);
- fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};
+ search_fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};
+ facet_fields = {field("starring", field_types::STRING)};
rank_fields = {"points"};
- collection1 = collectionManager.create_collection("collection1", fields, rank_fields);
+
+ collection1 = collectionManager.create_collection("collection1", search_fields, facet_fields, rank_fields);
}
virtual void SetUp() {
@@ -47,9 +50,9 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
infile.close();
std::vector<std::string> search_fields = {"starring", "title"};
- std::vector<facet> facets;
+ std::vector<std::string> facets;
- nlohmann::json results = collection1->search("thomas", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ nlohmann::json results = collection1->search("thomas", search_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
spp::sparse_hash_map<std::string, field> schema = collection1->get_schema();
@@ -61,12 +64,15 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
collection1 = collectionManager2.get_collection("collection1");
ASSERT_NE(nullptr, collection1);
+ std::vector<std::string> facet_fields_expected = {facet_fields[0].name};
+
ASSERT_EQ(0, collection1->get_collection_id());
ASSERT_EQ(18, collection1->get_next_seq_id());
+ ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());
ASSERT_EQ(rank_fields, collection1->get_rank_fields());
ASSERT_EQ(schema.size(), collection1->get_schema().size());
- results = collection1->search("thomas", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ results = collection1->search("thomas", search_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 23409ab6..c485c460 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -8,9 +8,11 @@
class CollectionTest : public ::testing::Test {
protected:
Collection *collection;
- std::vector<std::string> search_fields;
+ std::vector<std::string> query_fields;
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::vector<std::string> rank_fields;
+ std::vector<field> facet_fields;
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/collection";
@@ -21,13 +23,15 @@ protected:
collectionManager.init(store);
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
- std::vector<field> fields = {field("title", field_types::STRING)};
- std::vector<std::string> rank_fields = {"points"};
- search_fields = {"title"};
+ std::vector<field> search_fields = {field("title", field_types::STRING)};
+
+ query_fields = {"title"};
+ facet_fields = { };
+ rank_fields = {"points"};
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
- collection = collectionManager.create_collection("collection", fields, rank_fields);
+ collection = collectionManager.create_collection("collection", search_fields, facet_fields, rank_fields);
}
std::string json_line;
@@ -54,8 +58,8 @@ protected:
};
TEST_F(CollectionTest, ExactSearchShouldBeStable) {
- std::vector<facet> facets;
- nlohmann::json results = collection->search("the", search_fields, "", facets, 0, 10);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("the", query_fields, "", facets, rank_fields, 0, 10);
ASSERT_EQ(7, results["hits"].size());
ASSERT_EQ(7, results["found"].get<int>());
@@ -71,8 +75,8 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {
}
TEST_F(CollectionTest, ExactPhraseSearch) {
- std::vector<facet> facets;
- nlohmann::json results = collection->search("rocket launch", search_fields, "", facets, 0, 10);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, rank_fields, 0, 10);
ASSERT_EQ(5, results["hits"].size());
/*
@@ -94,7 +98,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
}
// Check pagination
- results = collection->search("rocket launch", search_fields, "", facets, 0, 3);
+ results = collection->search("rocket launch", query_fields, "", facets, rank_fields, 0, 3);
ASSERT_EQ(3, results["hits"].size());
for(size_t i = 0; i < 3; i++) {
nlohmann::json result = results["hits"].at(i);
@@ -106,8 +110,8 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
// Tokens that are not found in the index should be skipped
- std::vector<facet> facets;
- nlohmann::json results = collection->search("DoesNotExist from", search_fields, "", facets, 0, 10);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("DoesNotExist from", query_fields, "", facets, rank_fields, 0, 10);
ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"2", "17"};
@@ -120,7 +124,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
}
// with non-zero cost
- results = collection->search("DoesNotExist from", search_fields, "", facets, 1, 10);
+ results = collection->search("DoesNotExist from", query_fields, "", facets, rank_fields, 1, 10);
ASSERT_EQ(2, results["hits"].size());
for(size_t i = 0; i < results["hits"].size(); i++) {
@@ -131,7 +135,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
}
// with 2 indexed words
- results = collection->search("from DoesNotExist insTruments", search_fields, "", facets, 1, 10);
+ results = collection->search("from DoesNotExist insTruments", query_fields, "", facets, rank_fields, 1, 10);
ASSERT_EQ(2, results["hits"].size());
ids = {"2", "17"};
@@ -143,17 +147,17 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
}
results.clear();
- results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, "", facets, 0, 10);
+ results = collection->search("DoesNotExist1 DoesNotExist2", query_fields, "", facets, rank_fields, 0, 10);
ASSERT_EQ(0, results["hits"].size());
results.clear();
- results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, "", facets, 2, 10);
+ results = collection->search("DoesNotExist1 DoesNotExist2", query_fields, "", facets, rank_fields, 2, 10);
ASSERT_EQ(0, results["hits"].size());
}
TEST_F(CollectionTest, PartialPhraseSearch) {
- std::vector<facet> facets;
- nlohmann::json results = collection->search("rocket research", search_fields, "", facets, 0, 10);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("rocket research", query_fields, "", facets, rank_fields, 0, 10);
ASSERT_EQ(4, results["hits"].size());
std::vector<std::string> ids = {"1", "8", "16", "17"};
@@ -167,8 +171,8 @@ TEST_F(CollectionTest, PartialPhraseSearch) {
}
TEST_F(CollectionTest, QueryWithTypo) {
- std::vector<facet> facets;
- nlohmann::json results = collection->search("kind biologcal", search_fields, "", facets, 2, 3);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("kind biologcal", query_fields, "", facets, rank_fields, 2, 3);
ASSERT_EQ(3, results["hits"].size());
std::vector<std::string> ids = {"19", "20", "21"};
@@ -181,7 +185,7 @@ TEST_F(CollectionTest, QueryWithTypo) {
}
results.clear();
- results = collection->search("fer thx", search_fields, "", facets, 1, 3);
+ results = collection->search("fer thx", query_fields, "", facets, rank_fields, 1, 3);
ids = {"1", "10", "13"};
ASSERT_EQ(3, results["hits"].size());
@@ -195,8 +199,8 @@ TEST_F(CollectionTest, QueryWithTypo) {
}
TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
- std::vector<facet> facets;
- nlohmann::json results = collection->search("loox", search_fields, "", facets, 1, 2, MAX_SCORE, false);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 2, MAX_SCORE, false);
ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"22", "23"};
@@ -207,7 +211,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- results = collection->search("loox", search_fields, "", facets, 1, 3, FREQUENCY, false);
+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 3, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
ids = {"3", "12", "24"};
@@ -219,19 +223,19 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
}
// Check pagination
- results = collection->search("loox", search_fields, "", facets, 1, 1, FREQUENCY, false);
+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 1, FREQUENCY, false);
ASSERT_EQ(3, results["found"].get<int>());
ASSERT_EQ(1, results["hits"].size());
std::string solo_id = results["hits"].at(0)["id"];
ASSERT_STREQ("3", solo_id.c_str());
- results = collection->search("loox", search_fields, "", facets, 1, 2, FREQUENCY, false);
+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 2, FREQUENCY, false);
ASSERT_EQ(3, results["found"].get<int>());
ASSERT_EQ(2, results["hits"].size());
// Check total ordering
- results = collection->search("loox", search_fields, "", facets, 1, 10, FREQUENCY, false);
+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 10, FREQUENCY, false);
ASSERT_EQ(5, results["hits"].size());
ids = {"3", "12", "24", "22", "23"};
@@ -242,7 +246,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- results = collection->search("loox", search_fields, "", facets, 1, 10, MAX_SCORE, false);
+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 10, MAX_SCORE, false);
ASSERT_EQ(5, results["hits"].size());
ids = {"22", "23", "3", "12", "24"};
@@ -256,8 +260,8 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
TEST_F(CollectionTest, TextContainingAnActualTypo) {
// A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens
- std::vector<facet> facets;
- nlohmann::json results = collection->search("ISX what", search_fields, "", facets, 1, 4, FREQUENCY, false);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("ISX what", query_fields, "", facets, rank_fields, 1, 4, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
std::vector<std::string> ids = {"19", "6", "21", "8"};
@@ -270,7 +274,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
}
// Record containing exact token match should appear first
- results = collection->search("ISX", search_fields, "", facets, 1, 10, FREQUENCY, false);
+ results = collection->search("ISX", query_fields, "", facets, rank_fields, 1, 10, FREQUENCY, false);
ASSERT_EQ(8, results["hits"].size());
ids = {"20", "19", "6", "3", "21", "4", "10", "8"};
@@ -284,8 +288,8 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
}
TEST_F(CollectionTest, PrefixSearching) {
- std::vector<facet> facets;
- nlohmann::json results = collection->search("ex", search_fields, "", facets, 0, 10, FREQUENCY, true);
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("ex", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, true);
ASSERT_EQ(2, results["hits"].size());
std::vector<std::string> ids = {"12", "6"};
@@ -296,7 +300,7 @@ TEST_F(CollectionTest, PrefixSearching) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- results = collection->search("ex", search_fields, "", facets, 0, 10, MAX_SCORE, true);
+ results = collection->search("ex", query_fields, "", facets, rank_fields, 0, 10, MAX_SCORE, true);
ASSERT_EQ(2, results["hits"].size());
ids = {"6", "12"};
@@ -318,7 +322,7 @@ TEST_F(CollectionTest, MultipleFields) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, rank_fields);
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, rank_fields);
}
std::string json_line;
@@ -329,9 +333,9 @@ TEST_F(CollectionTest, MultipleFields) {
infile.close();
- search_fields = {"title", "starring"};
- std::vector<facet> facets;
- nlohmann::json results = coll_mul_fields->search("Will", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ query_fields = {"title", "starring"};
+ std::vector<std::string> facets;
+ nlohmann::json results = coll_mul_fields->search("Will", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
std::vector<std::string> ids = {"3", "2", "1", "0"};
@@ -345,8 +349,8 @@ TEST_F(CollectionTest, MultipleFields) {
// when "starring" takes higher priority than "title"
- search_fields = {"starring", "title"};
- results = coll_mul_fields->search("thomas", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ query_fields = {"starring", "title"};
+ results = coll_mul_fields->search("thomas", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
ids = {"15", "14", "12", "13"};
@@ -358,12 +362,12 @@ TEST_F(CollectionTest, MultipleFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- search_fields = {"starring", "title", "cast"};
- results = coll_mul_fields->search("ben affleck", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ query_fields = {"starring", "title", "cast"};
+ results = coll_mul_fields->search("ben affleck", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(1, results["hits"].size());
- search_fields = {"cast"};
- results = coll_mul_fields->search("chris", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ query_fields = {"cast"};
+ results = coll_mul_fields->search("chris", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
ids = {"6", "1", "7"};
@@ -374,8 +378,8 @@ TEST_F(CollectionTest, MultipleFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- search_fields = {"cast"};
- results = coll_mul_fields->search("chris pine", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ query_fields = {"cast"};
+ results = coll_mul_fields->search("chris pine", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
ids = {"7", "6", "1"};
@@ -398,7 +402,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);
}
std::string json_line;
@@ -410,9 +414,9 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
infile.close();
// Plain search with no filters - results should be sorted by rank fields
- search_fields = {"name"};
- std::vector<facet> facets;
- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ query_fields = {"name"};
+ std::vector<std::string> facets;
+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(5, results["hits"].size());
std::vector<std::string> ids = {"3", "1", "4", "0", "2"};
@@ -425,7 +429,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
}
// Searching on an int32 field
- results = coll_array_fields->search("Jeremy", search_fields, "age:>24", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "age:>24", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
ids = {"3", "1", "4"};
@@ -437,14 +441,14 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- results = coll_array_fields->search("Jeremy", search_fields, "age:>=24", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "age:>=24", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
- results = coll_array_fields->search("Jeremy", search_fields, "age:24", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "age:24", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(1, results["hits"].size());
// Searching a number against an int32 array field
- results = coll_array_fields->search("Jeremy", search_fields, "years:>2002", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "years:>2002", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
ids = {"1", "0", "2"};
@@ -455,7 +459,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- results = coll_array_fields->search("Jeremy", search_fields, "years:<1989", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "years:<1989", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(1, results["hits"].size());
ids = {"3"};
@@ -467,7 +471,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
}
// multiple filters
- results = coll_array_fields->search("Jeremy", search_fields, "years:<2005 && years:>1987", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "years:<2005 && years:>1987", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(1, results["hits"].size());
ids = {"4"};
@@ -479,7 +483,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
}
// multiple search values (works like SQL's IN operator) against a single int field
- results = coll_array_fields->search("Jeremy", search_fields, "age:[21, 24, 63]", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "age:[21, 24, 63]", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
ids = {"3", "0", "2"};
@@ -491,7 +495,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
}
// multiple search values against an int32 array field - also use extra padding between symbols
- results = coll_array_fields->search("Jeremy", search_fields, "years : [ 2015, 1985 , 1999]", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "years : [ 2015, 1985 , 1999]", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
ids = {"3", "1", "4", "0"};
@@ -503,7 +507,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
}
// searching on an int64 array field - also ensure that padded space causes no issues
- results = coll_array_fields->search("Jeremy", search_fields, "timestamps : > 475205222", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps : > 475205222", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
ids = {"1", "4", "0", "2"};
@@ -516,7 +520,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
}
// when filters don't match any record, no results should be returned
- results = coll_array_fields->search("Jeremy", search_fields, "timestamps:<1", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps:<1", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll_array_fields");
@@ -533,7 +537,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);
}
std::string json_line;
@@ -544,9 +548,9 @@ TEST_F(CollectionTest, FilterOnTextFields) {
infile.close();
- search_fields = {"name"};
- std::vector<facet> facets;
- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "tags: gold", facets, 0, 10, FREQUENCY, false);
+ query_fields = {"name"};
+ std::vector<std::string> facets;
+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "tags: gold", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
std::vector<std::string> ids = {"1", "4", "0", "2"};
@@ -558,7 +562,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- results = coll_array_fields->search("Jeremy", search_fields, "tags : bronze", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "tags : bronze", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(2, results["hits"].size());
ids = {"4", "2"};
@@ -571,7 +575,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {
}
// search with a list of tags, also testing extra padding of space
- results = coll_array_fields->search("Jeremy", search_fields, "tags: [bronze, silver]", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: [bronze, silver]", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
ids = {"3", "4", "0", "2"};
@@ -584,7 +588,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {
}
// should be exact matches (no normalization or fuzzy searching should happen)
- results = coll_array_fields->search("Jeremy", search_fields, "tags: BRONZE", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: BRONZE", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
collectionManager.drop_collection("coll_array_fields");
@@ -603,7 +607,7 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);
}
std::string json_line;
@@ -614,32 +618,34 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {
infile.close();
- search_fields = {"name"};
- std::vector<facet> facets;
+ query_fields = {"name"};
+ std::vector<std::string> facets;
// when filter field does not exist in the schema
- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "tagzz: gold", facets, 0, 10, FREQUENCY, false);
+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "tagzz: gold", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
// searching using a string for a numeric field
- results = coll_array_fields->search("Jeremy", search_fields, "age: abcdef", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "age: abcdef", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
// searching using a string for a numeric array field
- results = coll_array_fields->search("Jeremy", search_fields, "timestamps: abcdef", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps: abcdef", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
// malformed k:v syntax
- results = coll_array_fields->search("Jeremy", search_fields, "timestamps abcdef", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps abcdef", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
// just empty spaces
- results = coll_array_fields->search("Jeremy", search_fields, " ", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, " ", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
// wrapping number with quotes
- results = coll_array_fields->search("Jeremy", search_fields, "age: '21'", facets, 0, 10, FREQUENCY, false);
+ results = coll_array_fields->search("Jeremy", query_fields, "age: '21'", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(0, results["hits"].size());
+
+ collectionManager.drop_collection("coll_array_fields");
}
TEST_F(CollectionTest, FacetCounts) {
@@ -650,11 +656,12 @@ TEST_F(CollectionTest, FacetCounts) {
field("years", field_types::INT32_ARRAY),
field("timestamps", field_types::INT64_ARRAY),
field("tags", field_types::STRING_ARRAY)};
+ facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};
std::vector<std::string> rank_fields = {"age"};
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);
}
std::string json_line;
@@ -665,45 +672,113 @@ TEST_F(CollectionTest, FacetCounts) {
infile.close();
- search_fields = {"name"};
- std::vector<facet> facets = {facet("tags")};
+ query_fields = {"name"};
+ std::vector<std::string> facets = {"tags"};
// single facet with no filters
- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
ASSERT_EQ(5, results["hits"].size());
- ASSERT_EQ(1, results["facets"].size());
- ASSERT_EQ(2, results["facets"][0].size());
- ASSERT_EQ("tags", results["facets"][0]["field_name"]);
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0].size());
+ ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
+
+ ASSERT_EQ("gold", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ(4, (int) results["facet_counts"][0]["counts"][1]["count"]);
+
+ ASSERT_EQ("silver", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][2]["count"]);
- ASSERT_EQ(4, (int) results["facets"][0]["counts"]["gold"]);
- ASSERT_EQ(3, (int) results["facets"][0]["counts"]["silver"]);
- ASSERT_EQ(2, (int) results["facets"][0]["counts"]["bronze"]);
+ ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
// 2 facets, 1 text filter with no filters
facets.clear();
- facets.push_back(facet("tags"));
- facets.push_back(facet("name"));
- results = coll_array_fields->search("Jeremy", search_fields, "", facets, 0, 10, FREQUENCY, false);
+ facets.push_back("tags");
+ facets.push_back("name");
+ results = coll_array_fields->search("Jeremy", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);
+
ASSERT_EQ(5, results["hits"].size());
- ASSERT_EQ(2, results["facets"].size());
+ ASSERT_EQ(2, results["facet_counts"].size());
- ASSERT_EQ("tags", results["facets"][0]["field_name"]);
- ASSERT_EQ("name", results["facets"][1]["field_name"]);
+ ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
+ ASSERT_EQ("name", results["facet_counts"][1]["field_name"]);
- // text is tokenized and standardized
- ASSERT_EQ(5, (int) results["facets"][1]["counts"]["howard"]);
- ASSERT_EQ(5, (int) results["facets"][1]["counts"]["jeremy"]);
+ // facet value must one that's stored, not indexed (i.e. no tokenization/standardization)
+ ASSERT_EQ("Jeremy Howard", results["facet_counts"][1]["counts"][0]["value"]);
+ ASSERT_EQ(5, (int) results["facet_counts"][1]["counts"][0]["count"]);
// facet with filters
facets.clear();
- facets.push_back(facet("tags"));
- results = coll_array_fields->search("Jeremy", search_fields, "age: >24", facets, 0, 10, FREQUENCY, false);
+ facets.push_back("tags");
+ results = coll_array_fields->search("Jeremy", query_fields, "age: >24", facets, rank_fields, 0, 10, FREQUENCY, false);
+
ASSERT_EQ(3, results["hits"].size());
- ASSERT_EQ(1, results["facets"].size());
+ ASSERT_EQ(1, results["facet_counts"].size());
+
+ ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][1]["count"]);
+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][2]["count"]);
+
+ ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("gold", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("silver", results["facet_counts"][0]["counts"][2]["value"]);
+
+ collectionManager.drop_collection("coll_array_fields");
+}
+
+TEST_F(CollectionTest, SearchingWithMissingFields) {
+ // return error without crashing when searching for fields that do not conform to the schema
+ Collection *coll_array_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
+ std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
+ field("years", field_types::INT32_ARRAY),
+ field("timestamps", field_types::INT64_ARRAY),
+ field("tags", field_types::STRING_ARRAY)};
+ facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};
+ std::vector<std::string> rank_fields = {"age"};
+
+ coll_array_fields = collectionManager.get_collection("coll_array_fields");
+ if(coll_array_fields == nullptr) {
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_array_fields->add(json_line);
+ }
+
+ infile.close();
+
+ // when a query field mentioned in schema does not exist
+ std::vector<std::string> facets;
+ std::vector<std::string> query_fields_not_found = {"titlez"};
+
+ nlohmann::json res = coll_array_fields->search("the", query_fields_not_found, "", facets, rank_fields, 0, 10);
+ ASSERT_EQ(0, res["hits"].size());
+ ASSERT_STREQ("Could not find a search field named `titlez` in the schema.",res["error"].get<std::string>().c_str());
- ASSERT_EQ("tags", results["facets"][0]["field_name"]);
- ASSERT_EQ(2, (int) results["facets"][0]["counts"]["gold"]);
- ASSERT_EQ(2, (int) results["facets"][0]["counts"]["silver"]);
- ASSERT_EQ(1, (int) results["facets"][0]["counts"]["bronze"]);
+ // when a query field is an integer field
+ res = coll_array_fields->search("the", {"age"}, "", facets, rank_fields, 0, 10);
+ ASSERT_EQ(0, res["hits"].size());
+ ASSERT_STREQ("Search field `age` should be a string or a string array.", res["error"].get<std::string>().c_str());
+
+ // when a facet field is not defined in the schema
+ res = coll_array_fields->search("the", {"name"}, "", {"timestamps"}, rank_fields, 0, 10);
+ ASSERT_EQ(0, res["hits"].size());
+ ASSERT_STREQ("Could not find a facet field named `timestamps` in the schema.", res["error"].get<std::string>().c_str());
+
+ // when a rank field is not defined in the schema
+ res = coll_array_fields->search("the", {"name"}, "", {}, {"timestamps"}, 0, 10);
+ ASSERT_EQ(0, res["hits"].size());
+ ASSERT_STREQ("Could not find a rank field named `timestamps` in the schema.", res["error"].get<std::string>().c_str());
+
+ res = coll_array_fields->search("the", {"name"}, "", {}, {"_rank"}, 0, 10);
+ ASSERT_EQ(0, res["hits"].size());
+ ASSERT_STREQ("Could not find a rank field named `_rank` in the schema.", res["error"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll_array_fields");
}
\ No newline at end of file