Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow non-string fields to be facetable.
kishorenc95c8fb70824c
diff --git a/include/field.h b/include/field.hindex 389e9eb0..6a27ae80 100644--- a/include/field.h+++ b/include/field.h@@ -65,6 +65,16 @@ struct field {bool is_facet() const {return facet;}++ bool is_array() const {+ return (type == field_types::STRING_ARRAY || type == field_types::INT32_ARRAY ||+ type == field_types::FLOAT_ARRAY ||+ type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY);+ }++ std::string faceted_name() const {+ return (facet && !is_string()) ? "_fstr_" + name : name;+ }};struct filter {diff --git a/include/index.h b/include/index.hindex a95d17b3..1e019997 100644--- a/include/index.h+++ b/include/index.h@@ -252,15 +252,6 @@ public:// in the query that have the least individual hits one by one until enough results are found.static const int DROP_TOKENS_THRESHOLD = 10;- // strings under this length will be fully highlighted, instead of showing a snippet of relevant portion- enum {SNIPPET_STR_ABOVE_LEN = 30};-- // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store- static constexpr const char* COLLECTION_META_PREFIX = "$CM";- static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";- static constexpr const char* SEQ_ID_PREFIX = "$SI";- static constexpr const char* DOC_ID_PREFIX = "$DI";-/** Concurrency Primitives*/diff --git a/src/collection.cpp b/src/collection.cppindex 6fbe185b..9852ed23 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -749,6 +749,26 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::value = document[a_facet.field_name];} else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {value = document[a_facet.field_name][facet_count.array_pos];+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {+ value = std::to_string(document[a_facet.field_name].get<int32_t>());+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {+ value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<int32_t>());+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {+ value = std::to_string(document[a_facet.field_name].get<int64_t>());+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {+ value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<int64_t>());+ } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {+ value = std::to_string(document[a_facet.field_name].get<float>());+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros+ } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {+ value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<float>());+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros+ } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {+ value = std::to_string(document[a_facet.field_name].get<bool>());+ value = (value == "1") ? "true" : "false";+ } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {+ value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());+ value = (value == "1") ? "true" : "false";}std::vector<std::string> tokens;diff --git a/src/index.cpp b/src/index.cppindex 3d3717c5..93571f64 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -15,10 +15,16 @@ Index::Index(const std::string name, const std::unordered_map<std::string, fieldname(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {for(const auto & pair: search_schema) {- // NOTE: facet fields are also part of search schemaart_tree *t = new art_tree;art_tree_init(t);search_index.emplace(pair.first, t);++ // initialize for non-string facet fields+ if(pair.second.facet && !pair.second.is_string()) {+ art_tree *ft = new art_tree;+ art_tree_init(ft);+ search_index.emplace(pair.second.faceted_name(), ft);+ }}for(const auto & pair: sort_schema) {@@ -86,13 +92,56 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t// assumes that validation has already been donefor(const std::pair<std::string, field> & field_pair: search_schema) {const std::string & field_name = field_pair.first;- art_tree *t = search_index.at(field_name);int facet_id = -1;if(facet_schema.count(field_name) != 0) {facet_id = facet_to_id[field_name];}+ // non-string faceted field should be indexed as faceted string field as well+ if(field_pair.second.facet && !field_pair.second.is_string()) {+ art_tree *t = search_index.at(field_pair.second.faceted_name());++ if(field_pair.second.is_array()) {+ std::vector<std::string> strings;++ if(field_pair.second.type == field_types::INT32_ARRAY) {+ for(int32_t value: document[field_name]){+ strings.push_back(std::to_string(value));+ }+ } else if(field_pair.second.type == field_types::INT64_ARRAY) {+ for(int64_t value: document[field_name]){+ strings.push_back(std::to_string(value));+ }+ } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {+ for(float value: document[field_name]){+ strings.push_back(std::to_string(value));+ }+ } else if(field_pair.second.type == field_types::BOOL_ARRAY) {+ for(bool value: document[field_name]){+ strings.push_back(std::to_string(value));+ }+ }+ index_string_array_field(strings, points, t, seq_id, facet_id);+ } else {+ std::string text;++ if(field_pair.second.type == field_types::INT32) {+ text = std::to_string(document[field_name].get<int32_t>());+ } else if(field_pair.second.type == field_types::INT64) {+ text = std::to_string(document[field_name].get<int64_t>());+ } else if(field_pair.second.type == field_types::FLOAT) {+ text = std::to_string(document[field_name].get<float>());+ } else if(field_pair.second.type == field_types::BOOL) {+ text = std::to_string(document[field_name].get<bool>());+ }++ index_string_field(text, points, t, seq_id, facet_id);+ }+ }++ art_tree *t = search_index.at(field_name);+if(field_pair.second.type == field_types::STRING) {const std::string & text = document[field_name];index_string_field(text, points, t, seq_id, facet_id);@@ -241,6 +290,7 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,}}+ // since every facet field has to be a search field, we don't have to revalidate types herefor(const std::pair<std::string, field> & field_pair: facet_schema) {const std::string & field_name = field_pair.first;@@ -248,22 +298,6 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,return Option<>(400, "Field `" + field_name + "` has been declared as a facet field in the schema, ""but is not found in the document.");}-- if(field_pair.second.type == field_types::STRING) {- if(!document[field_name].is_string()) {- return Option<>(400, "Facet field `" + field_name + "` must be a string.");- }- } else if(field_pair.second.type == field_types::STRING_ARRAY) {- if(!document[field_name].is_array()) {- return Option<>(400, "Facet field `" + field_name + "` must be a string array.");- }-- if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {- return Option<>(400, "Facet field `" + field_name + "` must be a string array.");- }- } else {- return Option<>(400, "Facet field `" + field_name + "` must be a string or a string[].");- }}return Option<>(200);@@ -384,7 +418,6 @@ void Index::index_bool_field(const bool value, const uint32_t score, art_tree *tconst int KEY_LEN = 1;unsigned char key[KEY_LEN];key[0] = value ? '1' : '0';- //key[1] = '\0';uint32_t num_hits = 0;art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);@@ -548,9 +581,21 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,if(a_facet.field_name == facet_query.field_name && !facet_query.query.empty()) {use_facet_query = true;+ const field & facet_field = facet_schema.at(a_facet.field_name);++ if(facet_field.is_bool()) {+ if(facet_query.query == "true") {+ facet_query.query = "1";+ } else if(facet_query.query == "false") {+ facet_query.query = "0";+ }+ }+std::vector<std::string> query_tokens;StringUtils::split(facet_query.query, query_tokens, " ");+ art_tree *t = search_index.at(facet_field.faceted_name());+for(size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {auto & q = query_tokens[qtoken_index];string_utils.unicode_normalize(q);@@ -558,9 +603,11 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,bool prefix_search = (qtoken_index == (query_tokens.size()-1)); // only last token must be used as prefixstd::vector<art_leaf*> leaves;- art_fuzzy_search(search_index.at(a_facet.field_name), (const unsigned char *) q.c_str(),++ art_fuzzy_search(t, (const unsigned char *) q.c_str(),q.size(), 0, bounded_cost, 10000,token_ordering::MAX_SCORE, prefix_search, leaves);+for(size_t i = 0; i < leaves.size(); i++) {const auto & leaf = leaves[i];// calculate hash without terminating null chardiff --git a/test/collection_test.cpp b/test/collection_test.cppindex b2e47020..ecbd90fc 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1773,9 +1773,10 @@ TEST_F(CollectionTest, FacetCounts) {std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");std::vector<field> fields = {field("name", field_types::STRING, false),field("name_facet", field_types::STRING, true),- field("age", field_types::INT32, false),- field("years", field_types::INT32_ARRAY, false),- field("timestamps", field_types::INT64_ARRAY, false),+ field("age", field_types::INT32, true),+ field("years", field_types::INT32_ARRAY, true),+ field("rating", field_types::FLOAT, true),+ field("timestamps", field_types::INT64_ARRAY, true),field("tags", field_types::STRING_ARRAY, true)};std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };@@ -1938,8 +1939,53 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ // facet query on an integer field+ results = coll_array_fields->search("*", query_fields, "", {"age"}, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "age: 2").get();++ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_STREQ("age", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("24", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());++ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_STREQ("21", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());++ // facet query on a float field+ results = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "rating: 7").get();++ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_STREQ("rating", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("7.812", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>7</mark>.812", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());++ // facet query on a arrary integer field++ results = coll_array_fields->search("*", query_fields, "", {"timestamps"}, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "timestamps: 142189002").get();++ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_STREQ("timestamps", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("1421890022", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>142189002</mark>2", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+// facet query that does not match any indexed value- results = coll_array_fields->search("*", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,+ results = coll_array_fields->search("*", query_fields, "", {facets}, sort_fields, 0, 10, 1, FREQUENCY,false, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, 500, " tags : notfound").get();@@ -2101,6 +2147,60 @@ TEST_F(CollectionTest, FacetCountsHighlighting) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionTest, FacetCountsBool) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),+ field("in_stock", field_types::BOOL, true)};++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1");+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", fields, "points").get();+ }++ nlohmann::json doc;+ doc["id"] = "100";+ doc["title"] = "Ford Mustang";+ doc["points"] = 25;+ doc["in_stock"] = true;++ coll1->add(doc.dump());++ doc["id"] = "101";+ doc["title"] = "Tesla Model S";+ doc["points"] = 40;+ doc["in_stock"] = false;++ coll1->add(doc.dump());++ doc["id"] = "102";+ doc["title"] = "Chevrolet Beat";+ doc["points"] = 10;+ doc["in_stock"] = true;++ coll1->add(doc.dump());++ std::vector<std::string> facets = {"in_stock"};++ nlohmann::json results = coll1->search("*", {"title"}, "in_stock:true", facets, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "in_stock:true").get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());++ ASSERT_STREQ("in_stock", results["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("true", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>true</mark>",+ results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionTest, SortingOrder) {Collection *coll_mul_fields;