Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Implement facet filtering.

kishorenc4fa89ebf007f
diff --git a/include/collection.h b/include/collection.h
index c97c9b36..1c478e09 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -221,7 +221,8 @@ public:
size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
const spp::sparse_hash_set<std::string> & include_fields = spp::sparse_hash_set<std::string>(),
const spp::sparse_hash_set<std::string> & exclude_fields = spp::sparse_hash_set<std::string>(),
- size_t max_facet_values=10, size_t max_hits=500);
+ size_t max_facet_values=10, size_t max_hits=500,
+ const std::string & simple_facet_query = "");
Option<nlohmann::json> get(const std::string & id);
diff --git a/include/field.h b/include/field.h
index a011b0fe..162151a8 100644
--- a/include/field.h
+++ b/include/field.h
@@ -133,4 +133,9 @@ struct facet {
facet(const std::string & field_name): field_name(field_name) {
}
+};
+
+struct facet_query_t {
+ std::string field_name;
+ std::string query;
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index f77fdbc2..e91e35e4 100644
--- a/include/index.h
+++ b/include/index.h
@@ -29,6 +29,7 @@ struct search_args {
std::vector<uint32_t> included_ids;
std::vector<uint32_t> excluded_ids;
std::vector<sort_by> sort_fields_std;
+ facet_query_t facet_query;
int num_typos;
size_t max_facet_values;
size_t max_hits;
@@ -49,11 +50,11 @@ struct search_args {
search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,
std::vector<facet> facets, std::vector<uint32_t> included_ids, std::vector<uint32_t> excluded_ids,
- std::vector<sort_by> sort_fields_std, int num_typos, size_t max_facet_values,
+ std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,
size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,
size_t drop_tokens_threshold):
query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
- excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), num_typos(num_typos),
+ excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),
max_facet_values(max_facet_values), max_hits(max_hits), per_page(per_page),
page(page), token_order(token_order), prefix(prefix), drop_tokens_threshold(drop_tokens_threshold),
all_result_ids_len(0), outcome(0) {
@@ -139,7 +140,8 @@ private:
Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);
- void do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size);
+ void do_facets(std::vector<facet> & facets, const facet_query_t & facet_query,
+ const uint32_t* result_ids, size_t results_size);
void drop_facets(std::vector<facet> & facets, const std::vector<uint32_t> & ids);
@@ -202,6 +204,7 @@ public:
void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> & search_fields,
const std::vector<filter> & filters, std::vector<facet> & facets,
+ const facet_query_t & facet_query,
const std::vector<uint32_t> & included_ids, const std::vector<uint32_t> & excluded_ids,
const std::vector<sort_by> & sort_fields_std, const int num_typos,
const size_t max_hits, const size_t per_page, const size_t page, const token_ordering token_order,
@@ -271,8 +274,9 @@ public:
search_args search_params;
- static void
- populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions,
- const art_leaf *token_leaf, uint32_t doc_index);
+ static void populate_array_token_positions(std::vector<std::vector<std::vector<uint16_t>>> & array_token_positions,
+ const art_leaf *token_leaf, uint32_t doc_index);
+
+ int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 27db4b63..62510d6e 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -310,7 +310,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const size_t drop_tokens_threshold,
const spp::sparse_hash_set<std::string> & include_fields,
const spp::sparse_hash_set<std::string> & exclude_fields,
- const size_t max_facet_values, const size_t max_hits) {
+ const size_t max_facet_values, const size_t max_hits,
+ const std::string & simple_facet_query) {
std::vector<uint32_t> included_ids;
std::vector<uint32_t> excluded_ids;
@@ -453,6 +454,24 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
facets.emplace_back(field_name);
}
+ // parse facet query
+ std::vector<std::string> facet_query_vec;
+ facet_query_t facet_query;
+
+ if(!simple_facet_query.empty() && simple_facet_query.find(':') == std::string::npos) {
+ std::string error = "Facet query must be in the `facet_field: value` format.";
+ return Option<nlohmann::json>(404, error);
+ }
+
+ StringUtils::split(simple_facet_query, facet_query_vec, ":");
+ if(!facet_query_vec.empty()) {
+ facet_query = { StringUtils::trim(facet_query_vec[0]), StringUtils::trim(facet_query_vec[1]) };
+ if(facet_schema.count(facet_query.field_name) == 0) {
+ std::string error = "Could not find a facet field named `" + facet_query.field_name + "` in the schema.";
+ return Option<nlohmann::json>(404, error);
+ }
+ }
+
// validate sort fields and standardize
std::vector<sort_by> sort_fields_std;
@@ -502,7 +521,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
for(Index* index: indices) {
index->search_params = search_args(query, search_fields, filters, facets,
index_to_included_ids[index_id], index_to_excluded_ids[index_id],
- sort_fields_std, num_typos, max_facet_values, max_hits,
+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
per_page, page, token_order, prefix, drop_tokens_threshold);
{
std::lock_guard<std::mutex> lk(index->m);
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 07d73821..4ed6f085 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -174,6 +174,7 @@ void get_search(http_req & req, http_res & res) {
const char *SORT_BY = "sort_by";
const char *FACET_BY = "facet_by";
+ const char *FACET_QUERY = "facet_query";
const char *MAX_FACET_VALUES = "max_facet_values";
const char *MAX_HITS = "max_hits";
@@ -252,6 +253,10 @@ void get_search(http_req & req, http_res & res) {
std::vector<std::string> facet_fields;
StringUtils::split(req.params[FACET_BY], facet_fields, ",");
+ if(req.params.count(FACET_QUERY) == 0) {
+ req.params[FACET_QUERY] = "";
+ }
+
std::vector<std::string> include_fields_vec;
StringUtils::split(req.params[INCLUDE_FIELDS], include_fields_vec, ",");
@@ -307,7 +312,8 @@ void get_search(http_req & req, http_res & res) {
token_order, prefix, drop_tokens_threshold,
include_fields, exclude_fields,
static_cast<size_t>(std::stoi(req.params[MAX_FACET_VALUES])),
- static_cast<size_t>(std::stoi(req.params[MAX_HITS])));
+ static_cast<size_t>(std::stoi(req.params[MAX_HITS])),
+ req.params[FACET_QUERY]);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/src/index.cpp b/src/index.cpp
index c33a4aa9..e2670820 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -522,7 +522,8 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin
}
}
-void Index::do_facets(std::vector<facet> & facets, const uint32_t* result_ids, size_t results_size) {
+void Index::do_facets(std::vector<facet> & facets, const facet_query_t & facet_query,
+ const uint32_t* result_ids, size_t results_size) {
std::map<std::string, size_t> facet_to_index;
size_t i_facet = 0;
@@ -533,12 +534,35 @@ void Index::do_facets(std::vector<facet> & facets, const uint32_t* result_ids, s
for(auto & a_facet: facets) {
// assumed that facet fields have already been validated upstream
+ spp::sparse_hash_set<int64_t> facet_filter_set;
+ bool use_facet_query = false;
+
+ if(a_facet.field_name == facet_query.field_name && !facet_query.query.empty()) {
+ use_facet_query = true;
+ int bounded_cost = get_bounded_typo_cost(2, facet_query.query.size());
+ std::vector<art_leaf*> leaves;
+ art_fuzzy_search(search_index.at(a_facet.field_name), (const unsigned char *) facet_query.query.c_str(),
+ facet_query.query.size(),0, bounded_cost, 10000,
+ token_ordering::MAX_SCORE, true, leaves);
+
+ for(const auto & leaf: leaves) {
+ // calculate hash without terminating null char
+ uint64_t hash = wyhash(leaf->key, leaf->key_len-1, 0, _wyp);
+ facet_filter_set.insert(hash);
+ }
+ }
+
for(size_t i = 0; i < results_size; i++) {
uint32_t doc_seq_id = result_ids[i];
+
if(facet_index_v2.count(doc_seq_id) != 0) {
const std::vector<uint64_t> & values = facet_index_v2[doc_seq_id][facet_to_index[a_facet.field_name]];
for(size_t j = 0; j < values.size(); j++) {
+ if(use_facet_query && facet_filter_set.find(values[j]) == facet_filter_set.end()) {
+ // this particular facet value is not found in facet filter, so ignore
+ continue;
+ }
a_facet.result_map[values[j]].count += 1;
a_facet.result_map[values[j]].doc_id = doc_seq_id;
a_facet.result_map[values[j]].array_pos = j;
@@ -804,7 +828,7 @@ void Index::run_search() {
// after the wait, we own the lock.
search(search_params.outcome, search_params.query, search_params.search_fields,
- search_params.filters, search_params.facets, search_params.included_ids,
+ search_params.filters, search_params.facets, search_params.facet_query, search_params.included_ids,
search_params.excluded_ids, search_params.sort_fields_std, search_params.num_typos,
search_params.max_hits, search_params.per_page, search_params.page, search_params.token_order,
search_params.prefix, search_params.drop_tokens_threshold, search_params.raw_result_kvs,
@@ -887,7 +911,8 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f
void Index::search(Option<uint32_t> & outcome,
std::string query,
const std::vector<std::string> & search_fields,
- const std::vector<filter> & filters, std::vector<facet> & facets,
+ const std::vector<filter> & filters,
+ std::vector<facet> & facets, const facet_query_t & facet_query,
const std::vector<uint32_t> & included_ids,
const std::vector<uint32_t> & excluded_ids,
const std::vector<sort_by> & sort_fields_std, const int num_typos, const size_t max_hits,
@@ -925,13 +950,13 @@ void Index::search(Option<uint32_t> & outcome,
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
filter_ids, filter_ids_length);
collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);
- do_facets(facets, filter_ids, filter_ids_length);
+ do_facets(facets, facet_query, filter_ids, filter_ids_length);
all_result_ids_len = filter_ids_length;
} else {
const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);
for(size_t i = 0; i < num_search_fields; i++) {
// proceed to query search only when no filters are provided or when filtering produces results
- if(filters.size() == 0 || filter_ids_length > 0) {
+ if(filters.empty() || filter_ids_length > 0) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort results
const std::string & field = search_fields[i];
@@ -941,10 +966,10 @@ void Index::search(Option<uint32_t> & outcome,
collate_curated_ids(query, field, field_id, included_ids, curated_topster, searched_queries);
}
}
- do_facets(facets, all_result_ids, all_result_ids_len);
+ do_facets(facets, facet_query, all_result_ids, all_result_ids_len);
}
- do_facets(facets, &included_ids[0], included_ids.size());
+ do_facets(facets, facet_query, &included_ids[0], included_ids.size());
// must be sorted before iterated upon to remove "empty" array entries
topster.sort();
@@ -1019,10 +1044,7 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st
const size_t token_len = tokens[token_index].length();
// This ensures that we don't end up doing a cost of 1 for a single char etc.
- int bounded_cost = max_cost;
- if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {
- bounded_cost = token_len - 1;
- }
+ int bounded_cost = get_bounded_typo_cost(max_cost, token_len);
for(int cost = 0; cost <= bounded_cost; cost++) {
all_costs.push_back(cost);
@@ -1154,6 +1176,14 @@ void Index::search_field(const uint8_t & field_id, std::string & query, const st
}
}
+int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const {
+ int bounded_cost = max_cost;
+ if(token_len > 0 && max_cost >= token_len && (token_len == 1 || token_len == 2)) {
+ bounded_cost = token_len - 1;
+ }
+ return bounded_cost;
+}
+
void Index::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {
LOG(INFO) << "Token: " << token << ", cost: " << cost;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 955c5b95..146aaed6 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1843,6 +1843,47 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
+ // facet with facet filter query (allows typo correction!)
+ results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
+ false, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, " tags : sliver").get();
+
+ ASSERT_EQ(5, results["hits"].size());
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+
+ // facet query that does not match any indexed value
+ results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
+ false, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, " tags : notfound").get();
+
+ ASSERT_EQ(5, results["hits"].size());
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["facet_counts"][0]["counts"].size());
+
+ // bad facet query syntax
+ auto res_op = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
+ false, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "foobar");
+
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("Facet query must be in the `facet_field: value` format.", res_op.error().c_str());
+
+ // unknown facet field
+ res_op = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,
+ false, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "foobar: baz");
+
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_STREQ("Could not find a facet field named `foobar` in the schema.", res_op.error().c_str());
+
collectionManager.drop_collection("coll_array_fields");
}