Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Filtering on string field should be verbatim by default.

kishorencaa9e4a226e57
diff --git a/include/art.h b/include/art.h
index 1d3a286d..9c6276d0 100644
--- a/include/art.h
+++ b/include/art.h
@@ -130,6 +130,7 @@ enum NUM_COMPARATOR {
LESS_THAN,
LESS_THAN_EQUALS,
EQUALS,
+ CONTAINS,
GREATER_THAN,
GREATER_THAN_EQUALS
};
diff --git a/include/index.h b/include/index.h
index ce2995f0..b146a2c0 100644
--- a/include/index.h
+++ b/include/index.h
@@ -236,7 +236,7 @@ public:
void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const uint32_t *result_ids, const size_t result_size) const;
+ const uint32_t *result_ids, const size_t result_size);
static int32_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);
@@ -288,5 +288,11 @@ public:
static int64_t float_to_in64_t(float n);
uint64_t get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id, const uint32_t seq_id) const;
+
+ void get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index);
+
+ void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
+ const std::vector<art_leaf *> &query_suggestion,
+ uint32_t *exact_strt_ids, size_t& exact_strt_size) const;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 4ebd6de3..72469b49 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -610,12 +610,21 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
} else if(_field.is_string()) {
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ size_t filter_value_index = 0;
+ NUM_COMPARATOR str_comparator = EQUALS;
+
+ if(raw_value[0] == '~') {
+ // string filter should be evaluated in "contains" mode
+ str_comparator = CONTAINS;
+ while(raw_value[++filter_value_index] == ' ');
+ }
+
+ if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
std::vector<std::string> filter_values;
StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
- f = {field_name, filter_values, EQUALS};
+ f = {field_name, filter_values, str_comparator};
} else {
- f = {field_name, {raw_value}, EQUALS};
+ f = {field_name, {raw_value}, str_comparator};
}
} else {
return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Unidentified field type.");
diff --git a/src/index.cpp b/src/index.cpp
index 41f82f7e..a55775f9 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -96,6 +96,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
}
// initialize facet index since it will be updated as well during search indexing
+ // even if a field is optional, a facet position will be available in the vector for that field
std::vector<std::vector<uint64_t>> values(facet_schema.size());
facet_index_v2.emplace(seq_id, values);
@@ -630,12 +631,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
const uint32_t* result_ids, size_t results_size) {
std::unordered_map<std::string, size_t> facet_to_index;
-
- size_t i_facet = 0;
- for(const auto & facet: facet_schema) {
- facet_to_index[facet.first] = i_facet;
- i_facet++;
- }
+ get_facet_to_index(facet_to_index);
// assumed that facet fields have already been validated upstream
for(auto & a_facet: facets) {
@@ -781,6 +777,14 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
}
}
+void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) {
+ size_t i_facet = 0;
+ for(const auto & facet_kv: facet_schema) {
+ facet_to_index.emplace(facet_kv.first, i_facet);
+ i_facet++;
+ }
+}
+
void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,
const std::vector<uint32_t>& curated_ids,
const std::vector<sort_by> & sort_fields,
@@ -884,6 +888,9 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
uint32_t* filter_ids = nullptr;
uint32_t filter_ids_length = 0;
+ std::unordered_map<std::string, size_t> facet_to_index;
+ get_facet_to_index(facet_to_index);
+
for(size_t i = 0; i < filters.size(); i++) {
const filter & a_filter = filters[i];
if(search_index.count(a_filter.field_name) == 0) {
@@ -945,9 +952,11 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
uint32_t* strt_ids = nullptr;
size_t strt_ids_size = 0;
+ std::vector<art_leaf *> query_suggestion;
+
// there could be multiple tokens in a filter value, which we have to treat as ANDs
// e.g. country: South Africa
- for(auto & str_token : str_tokens) {
+ for(auto& str_token: str_tokens) {
string_utils.unicode_normalize(str_token);
art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
@@ -956,6 +965,8 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
continue;
}
+ query_suggestion.push_back(leaf);
+
if(strt_ids == nullptr) {
strt_ids = leaf->values->ids.uncompress();
strt_ids_size = leaf->values->ids.getLength();
@@ -971,6 +982,24 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
}
}
+ if(a_filter.compare_operator == EQUALS) {
+ // need to do exact match (unlike CONTAINS) by iterating the result ids and calculating scores
+ if(query_suggestion.size() == str_tokens.size()) {
+ uint32_t *exact_strt_ids = new uint32_t[strt_ids_size];
+ size_t exact_strt_size = 0;
+
+ eq_str_filter_plain(strt_ids, strt_ids_size, query_suggestion, exact_strt_ids, exact_strt_size);
+
+ delete[] strt_ids;
+ strt_ids = exact_strt_ids;
+ strt_ids_size = exact_strt_size;
+ }
+ }
+
+ // (NOT implemented) if the query is wrapped by double quotes, ensure exact match
+ // bool exact_match = (filter_value.front() == '"' && filter_value.back() == '"');
+
+ // Otherwise, we just ensure that given record contains tokens in the filter query
uint32_t* out = nullptr;
ids_size = ArrayUtils::or_scalar(ids, ids_size, strt_ids, strt_ids_size, &out);
delete[] strt_ids;
@@ -999,6 +1028,59 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto
return Option<>(filter_ids_length);
}
+void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,
+ const std::vector<art_leaf *> &query_suggestion, uint32_t *exact_strt_ids,
+ size_t& exact_strt_size) const {
+ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
+ for (art_leaf *token_leaf: query_suggestion) {
+ if(token_leaf == nullptr) {
+ leaf_to_indices.emplace(token_leaf, nullptr);
+ continue;
+ }
+
+ uint32_t *indices = new uint32_t[strt_ids_size];
+ token_leaf->values->ids.indexOf(strt_ids, strt_ids_size, indices);
+ leaf_to_indices.emplace(token_leaf, indices);
+ }
+
+ // e.g. First In First Out => hash([0, 1, 0, 2])
+ spp::sparse_hash_map<size_t, uint32_t> leaf_to_id;
+ size_t next_id = 1;
+ size_t filter_hash = 1;
+
+ for(size_t leaf_index=0; leaf_index<query_suggestion.size(); leaf_index++) {
+ if(leaf_to_id.count(leaf_index) == 0) {
+ leaf_to_id.emplace(leaf_index, next_id++);
+ }
+
+ uint32_t leaf_id = leaf_to_id[leaf_index];
+ filter_hash *= (1779033703 + 2*leaf_id*(leaf_index+1));
+ }
+
+ for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {
+ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;
+ populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);
+ // iterate array_token_positions and compute hash
+
+ for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {
+ auto& token_positions = array_token_positions[array_index];
+ size_t this_hash = 1;
+
+ for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {
+ auto& positions = token_positions[token_index];
+ for(auto pos: positions) {
+ this_hash *= (1779033703 + 2*(token_index+1)*(pos+1));
+ }
+ }
+
+ if(this_hash == filter_hash) {
+ exact_strt_ids[exact_strt_size++] = strt_ids[strt_ids_index];
+ break;
+ }
+ }
+ }
+}
+
uint32_t* Index::collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const {
std::vector<uint32_t> ids;
@@ -1421,16 +1503,21 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
const uint8_t & field_id, const uint32_t total_cost, Topster* topster,
const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const uint32_t *result_ids, const size_t result_size) const {
+ const uint32_t *result_ids, const size_t result_size) {
spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;
-
for (art_leaf *token_leaf : query_suggestion) {
uint32_t *indices = new uint32_t[result_size];
token_leaf->values->ids.indexOf(result_ids, result_size, indices);
leaf_to_indices.emplace(token_leaf, indices);
}
+ std::unordered_map<std::string, size_t> facet_to_index;
+ get_facet_to_index(facet_to_index);
+
+ Match single_token_match = Match(1, 0);
+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);
+
int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
spp::sparse_hash_map<uint32_t, int64_t>* field_values[3];
@@ -1447,19 +1534,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
//auto begin = std::chrono::high_resolution_clock::now();
- Match single_token_match = Match(1, 0);
- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);
-
- std::unordered_map<std::string, size_t> facet_to_id;
-
- if(search_params->group_limit > 0) {
- size_t i_facet = 0;
- for(const auto & facet: facet_schema) {
- facet_to_id[facet.first] = i_facet;
- i_facet++;
- }
- }
-
for(size_t i=0; i<result_size; i++) {
const uint32_t seq_id = result_ids[i];
@@ -1537,7 +1611,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
uint64_t distinct_id = seq_id;
if(search_params->group_limit != 0) {
- distinct_id = get_distinct_id(facet_to_id, seq_id);
+ distinct_id = get_distinct_id(facet_to_index, seq_id);
groups_processed.emplace(distinct_id);
}
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 041f7f07..6c1e8835 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -7,10 +7,11 @@
std::string lower_and_no_special_chars(const std::string & str) {
std::stringstream ss;
- for(const auto c: str) {
- bool should_remove = ( (int)(c) >= 0 && // check for ASCII range
- !std::isalnum(c) );
- if(!should_remove) {
+ for(char c : str) {
+ bool is_ascii = ((int)(c) >= 0);
+ bool keep_char = !is_ascii || std::isalnum(c);
+
+ if(keep_char) {
ss << (char) std::tolower(c);
}
}
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 82db30a4..daf51598 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -140,6 +140,23 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
+ // facet with filter on string array field must fail when partial token is used
+ facets.clear();
+ facets.push_back("tags");
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(0, results["found"].get<size_t>());
+
+ // partial token should be used along with ~
+ results = coll_array_fields->search("Jeremy", query_fields, "tags:~ PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ // or when tokens match facet value exactly
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
// facet with wildcard query
facets.clear();
facets.push_back("tags");
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 3eee6ddf..14033a57 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -847,7 +847,7 @@ TEST_F(CollectionTest, MultipleFields) {
// filtering on unfaceted multi-valued string field
query_fields = {"title"};
- results = coll_mul_fields->search("captain", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ results = coll_mul_fields->search("captain", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(1, results["hits"].size());
ids = {"6"};
for(size_t i = 0; i < results["hits"].size(); i++) {
@@ -928,7 +928,7 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {
// filtering on facet field should be possible (supports partial word search but without typo tolerance)
query_fields = {"title"};
- result_op = coll_mul_fields->search("*", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1,
+ result_op = coll_mul_fields->search("*", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1,
FREQUENCY, false);
ASSERT_EQ(true, result_op.ok());
results = result_op.get();