Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow faceting to be configurable via dynamic fields.
kishorenc0fb5c25d2cb7
diff --git a/include/collection.h b/include/collection.hindex 5b1697bd..256100fc 100644--- a/include/collection.h+++ b/include/collection.h@@ -321,10 +321,12 @@ private:const float max_memory_ratio;- const std::vector<Index*> indices;-const std::string fallback_field_type;+ std::vector<field> dynamic_fields;++ const std::vector<Index*> indices;+// methodsstd::string get_doc_id_key(const std::string & doc_id) const;@@ -453,6 +455,8 @@ public:std::vector<field> get_fields();+ std::vector<field> get_dynamic_fields();+std::unordered_map<std::string, field> get_schema();std::string get_default_sorting_field();diff --git a/include/field.h b/include/field.hindex bf29ee64..870e6c1c 100644--- a/include/field.h+++ b/include/field.h@@ -32,6 +32,7 @@ namespace fields {static const std::string type = "type";static const std::string facet = "facet";static const std::string optional = "optional";+ static const std::string index = "index";static const std::string geo_resolution = "geo_resolution";}@@ -43,22 +44,13 @@ struct field {std::string type;bool facet;bool optional;+ bool index;uint8_t geo_resolution;- field(const std::string & name, const std::string & type, const bool facet):- name(name), type(type), facet(facet), optional(false), geo_resolution(DEFAULT_GEO_RESOLUTION) {-- }-- field(const std::string & name, const std::string & type, const bool facet, const bool optional):- name(name), type(type), facet(facet), optional(optional), geo_resolution(DEFAULT_GEO_RESOLUTION) {-- }-- field(const std::string & name, const std::string & type, const bool facet, const bool optional,- const uint8_t geo_resolution):- name(name), type(type), facet(facet), optional(optional), geo_resolution(geo_resolution) {+ field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,+ bool index = true, const uint8_t geo_resolution = DEFAULT_GEO_RESOLUTION) :+ name(name), type(type), facet(facet), optional(optional), index(index), geo_resolution(geo_resolution) {}@@ -121,6 +113,10 @@ struct field {return !is_array();}+ bool is_dynamic() const {+ return name.find(".*") != std::string::npos;+ }+bool has_numerical_index() const {return (type == field_types::INT32 || type == field_types::INT64 ||type == field_types::FLOAT || type == field_types::BOOL);@@ -226,6 +222,10 @@ struct field {found_default_sorting_field = true;}++ if(field.is_dynamic() && !field.optional) {+ return Option<bool>(400, "Field `" + field.name + "` with wildcard name must be an optional field.");+ }}if(!default_sorting_field.empty() && !found_default_sorting_field) {@@ -309,7 +309,12 @@ struct field {}if(field_json.count("optional") == 0) {- field_json["optional"] = false;+ // actual value will depend on whether field name is dynamic or not (i.e. contains .*)+ if(field_json["name"].get<std::string>().find(".*") != std::string::npos) {+ field_json["optional"] = true;+ } else {+ field_json["optional"] = false;+ }}fields.emplace_back(@@ -473,6 +478,10 @@ struct facet_hash_values_t {uint32_t length = 0;uint64_t* hashes = nullptr;+ ~facet_hash_values_t() {+ delete [] hashes;+ }+uint64_t size() const {return length;}diff --git a/src/collection.cpp b/src/collection.cppindex 1acb0a45..8c6f866f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -13,6 +13,7 @@#include <tokenizer.h>#include <collection_manager.h>#include <h3api.h>+#include <regex>#include "topster.h"#include "logger.h"@@ -46,7 +47,8 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, cofields(fields), default_sorting_field(default_sorting_field),num_memory_shards(num_memory_shards),max_memory_ratio(max_memory_ratio),- indices(init_indices()), fallback_field_type(fallback_field_type) {+ fallback_field_type(fallback_field_type), dynamic_fields({}),+ indices(init_indices()) {this->num_documents = 0;}@@ -154,6 +156,7 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::type] = coll_field.type;field_json[fields::facet] = coll_field.facet;field_json[fields::optional] = coll_field.optional;+ field_json[fields::index] = coll_field.index;if(coll_field.is_geopoint()) {field_json[fields::geo_resolution] = size_t(coll_field.geo_resolution);@@ -223,8 +226,8 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaget_document_from_store(get_seq_id_key(seq_id), record.old_doc);}- // if `fallback_field_type` is enabled, we will have to update schema first before indexing- if(!fallback_field_type.empty()) {+ // if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing+ if(!fallback_field_type.empty() || !dynamic_fields.empty()) {Option<bool> schema_change_op = check_and_update_schema(record.doc, dirty_values);if(!schema_change_op.ok()) {record.index_failure(schema_change_op.code(), schema_change_op.error());@@ -1292,6 +1295,10 @@ bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_tconst nlohmann::json &document, std::string &value) const {if(document.count(a_facet.field_name) == 0) {+ if(search_schema.at(a_facet.field_name).optional) {+ return false;+ }+LOG(ERROR) << "Could not find field " << a_facet.field_name << " in document during faceting.";LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;LOG(ERROR) << "Actual document: " << document;@@ -1740,6 +1747,11 @@ std::vector<field> Collection::get_fields() {return fields;}+std::vector<field> Collection::get_dynamic_fields() {+ std::shared_lock lock(mutex);+ return dynamic_fields;+}+std::unordered_map<std::string, field> Collection::get_schema() {std::shared_lock lock(mutex);return search_schema;@@ -2261,11 +2273,30 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, conststd::vector<field> new_fields;auto kv = document.begin();- for(; kv != document.end(); ) {- // we will not index the special "id" or doc meta keys+ while(kv != document.end()) {+ // we will not index the special "id" keyif (search_schema.count(kv.key()) == 0 && kv.key() != "id") {+ const std::string &fname = kv.key();+ field new_field(fname, field_types::STRING, false, true);std::string field_type;- bool parseable = field::get_type(kv.value(), field_type);+ bool parseable;++ // check against dynamic field definitions+ for(const auto& dynamic_field: dynamic_fields) {+ if(std::regex_match (kv.key(), std::regex(dynamic_field.name))) {+ new_field = dynamic_field;+ new_field.name = fname;+ goto UPDATE_SCHEMA;+ }+ }++ if(fallback_field_type.empty()) {+ // we will not auto detect schema if auto detection is not enabled+ kv++;+ continue;+ }++ parseable = field::get_type(kv.value(), field_type);if(!parseable) {if(dirty_values == DIRTY_VALUES::REJECT || dirty_values == DIRTY_VALUES::COERCE_OR_REJECT) {return Option<bool>(400, "Type of field `" + kv.key() + "` is invalid.");@@ -2276,29 +2307,32 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const}}- const std::string &fname = kv.key();- field new_field(fname, field_type, false, true);-- if(!fallback_field_type.empty()) {- if (field_types::is_string_or_array(fallback_field_type)) {- // Supporting single/array field detection only for strings, as it does not seem to be too useful for- // other field types.- if (new_field.is_array()) {- new_field.type = field_types::STRING_ARRAY;- } else {- new_field.type = field_types::STRING;- }- } else if(fallback_field_type != field_types::AUTO) {- new_field.type = fallback_field_type;+ new_field.type = field_type;++ if (field_types::is_string_or_array(fallback_field_type)) {+ // Supporting single/array field detection only for strings, as it does not seem to be too useful for+ // other field types.+ if (new_field.is_array()) {+ new_field.type = field_types::STRING_ARRAY;+ } else {+ new_field.type = field_types::STRING;}+ } else if(fallback_field_type != field_types::AUTO) {+ new_field.type = fallback_field_type;}- search_schema.emplace(fname, new_field);+ UPDATE_SCHEMA:++ search_schema.emplace(new_field.name, new_field);fields.emplace_back(new_field);new_fields.emplace_back(new_field);if (new_field.is_sortable()) {sort_schema.emplace(new_field.name, new_field);}++ if(new_field.is_facet()) {+ facet_schema.emplace(new_field.name, new_field);+ }}kv++;@@ -2348,6 +2382,12 @@ std::vector<Index *> Collection::init_indices() {std::vector<Index*> index_list;for(const field& field: fields) {+ if(field.is_dynamic()) {+ // regexp fields are treated as dynamic fields+ dynamic_fields.push_back(field);+ continue;+ }+search_schema.emplace(field.name, field);if(field.is_facet()) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 26e91d94..dd40aed1 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -25,13 +25,17 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::optional] = false;}+ if(field_obj.count(fields::index) == 0) {+ field_obj[fields::index] = true;+ }+// handle older records indexed before geo_resolution field introductionif(field_obj.count(fields::geo_resolution) == 0) {field_obj[fields::geo_resolution] = size_t(DEFAULT_GEO_RESOLUTION);}fields.push_back({field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],- field_obj[fields::optional], field_obj[fields::geo_resolution]});+ field_obj[fields::optional], field_obj[fields::index], field_obj[fields::geo_resolution]});}std::string default_sorting_field = collection_meta[Collection::COLLECTION_DEFAULT_SORTING_FIELD_KEY].get<std::string>();diff --git a/src/core_api.cpp b/src/core_api.cppindex c259b548..78906027 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -140,15 +140,6 @@ bool post_create_collection(const std::shared_ptr<http_req>& req, const std::shareturn false;}- std::string fallback_field_type;- std::vector<field> fields;- auto parse_op = field::json_fields_to_fields(req_json["fields"], fallback_field_type, fields);-- if(!parse_op.ok()) {- res->set(parse_op.code(), parse_op.error());- return false;- }-const std::string & default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get<std::string>();const Option<Collection*> & collection_op =collectionManager.create_collection(req_json, num_memory_shards, default_sorting_field);diff --git a/src/index.cpp b/src/index.cppindex e6096f1a..5f679432 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -855,6 +855,9 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,facet_count_t& facet_count = a_facet.result_map[fhash];+ /*LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id+ << ", hash: " << fhash;*/+facet_count.doc_id = doc_seq_id;facet_count.array_pos = array_pos;@@ -2470,6 +2473,18 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {}}+ if(new_field.is_facet()) {+ facet_schema.emplace(new_field.name, new_field);+ facet_index_v3.emplace(new_field.name, spp::sparse_hash_map<uint32_t, facet_hash_values_t>());++ // initialize for non-string facet fields+ if(!new_field.is_string()) {+ art_tree *ft = new art_tree;+ art_tree_init(ft);+ search_index.emplace(new_field.faceted_name(), ft);+ }+ }+if(sort_index.count(new_field.name) == 0) {spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();sort_index.emplace(new_field.name, doc_to_score);diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 40a434d6..d97a4377 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -552,3 +552,217 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {ASSERT_FALSE(parse_op.ok());ASSERT_EQ("The `geo_resolution` property of the field `loc` should be between 0 and 15.", parse_op.error());}++TEST_F(CollectionAllFieldsTest, WildcardFacetFieldsOnAutoSchema) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field(".*_name", field_types::STRING, true, true),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO).get();+ }++ nlohmann::json doc;+ doc["title"] = "Org";+ doc["org_name"] = "Amazon";++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["title"] = "Org";+ doc["org_name"] = "Walmart";++ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto results = coll1->search("org", {"title"}, "", {"org_name"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("Walmart", results["hits"][0]["document"]["org_name"].get<std::string>());+ ASSERT_EQ("Amazon", results["hits"][1]["document"]["org_name"].get<std::string>());++ ASSERT_EQ("Amazon", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);++ ASSERT_EQ("Walmart", results["facet_counts"][0]["counts"][1]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);++ // add another type of .*_name field++ doc.clear();+ doc["title"] = "Company";+ doc["company_name"] = "Stark";++ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("*", {"title"}, "", {"company_name", "org_name"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(3, results["hits"].size());++ ASSERT_EQ("company_name", results["facet_counts"][0]["field_name"].get<std::string>());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Stark", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);++ ASSERT_EQ("org_name", results["facet_counts"][1]["field_name"].get<std::string>());+ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());+ ASSERT_EQ("Amazon", results["facet_counts"][1]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][1]["counts"][0]["count"]);++ ASSERT_EQ("Walmart", results["facet_counts"][1]["counts"][1]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][1]["counts"][1]["count"]);++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionAllFieldsTest, WildcardFacetFieldsWithoutAutoSchema) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field(".*_name", field_types::STRING, true, true),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0).get();+ }++ nlohmann::json doc;+ doc["title"] = "Org";+ doc["org_name"] = "Amazon";++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["title"] = "Org";+ doc["org_name"] = "Walmart";++ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto results = coll1->search("org", {"title"}, "", {"org_name"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("Walmart", results["hits"][0]["document"]["org_name"].get<std::string>());+ ASSERT_EQ("Amazon", results["hits"][1]["document"]["org_name"].get<std::string>());++ ASSERT_EQ("Amazon", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);++ ASSERT_EQ("Walmart", results["facet_counts"][0]["counts"][1]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);++ // add another type of .*_name field++ doc.clear();+ doc["title"] = "Company";+ doc["company_name"] = "Stark";++ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("*", {"title"}, "", {"company_name", "org_name"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(3, results["hits"].size());++ ASSERT_EQ("company_name", results["facet_counts"][0]["field_name"].get<std::string>());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Stark", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);++ ASSERT_EQ("org_name", results["facet_counts"][1]["field_name"].get<std::string>());+ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());+ ASSERT_EQ("Amazon", results["facet_counts"][1]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][1]["counts"][0]["count"]);++ ASSERT_EQ("Walmart", results["facet_counts"][1]["counts"][1]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][1]["counts"][1]["count"]);++ // Don't allow auto detection of schema when AUTO mode is not chosen+ doc["description"] = "Stark company.";+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto res_op = coll1->search("*", {"description"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `description` in the schema.", res_op.error());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionAllFieldsTest, DynamicFieldsMustOnlyBeOptional) {+ Collection *coll1;++ std::vector<field> bad_fields = {field("title", field_types::STRING, true),+ field(".*_name", field_types::STRING, true, false),};++ auto op = collectionManager.create_collection("coll1", 1, bad_fields, "", 0);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("Field `.*_name` with wildcard name must be an optional field.", op.error());++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field(".*_name", field_types::STRING, true, true),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ op = collectionManager.create_collection("coll1", 1, fields, "", 0);+ ASSERT_TRUE(op.ok());+ coll1 = op.get();+ }++ ASSERT_TRUE(coll1->get_dynamic_fields()[0].optional);++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionAllFieldsTest, BothFallbackAndDynamicFields) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field(".*_name", field_types::STRING, false, true),+ field(".*_year", field_types::INT32, true, true),+ field("*", field_types::AUTO, false, true)};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);+ ASSERT_TRUE(op.ok());+ coll1 = op.get();+ }++ ASSERT_EQ(4, coll1->get_fields().size());+ ASSERT_EQ(2, coll1->get_dynamic_fields().size());++ ASSERT_EQ(".*_name", coll1->get_dynamic_fields()[0].name);+ ASSERT_TRUE(coll1->get_dynamic_fields()[0].optional);+ ASSERT_FALSE(coll1->get_dynamic_fields()[0].facet);++ ASSERT_EQ(".*_year", coll1->get_dynamic_fields()[1].name);+ ASSERT_TRUE(coll1->get_dynamic_fields()[0].optional);+ ASSERT_FALSE(coll1->get_dynamic_fields()[0].facet);++ nlohmann::json doc;+ doc["title"] = "Amazon Inc.";+ doc["org_name"] = "Amazon";+ doc["org_year"] = 1994;+ doc["rand_int"] = 42;+ doc["rand_str"] = "fizzbuzz";++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto res_op = coll1->search("Amazon", {"org_name"}, "", {"org_name"}, sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a facet field named `org_name` in the schema.", res_op.error());++ auto results = coll1->search("Amazon", {"org_name"}, "", {"org_year"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());++ res_op = coll1->search("fizzbuzz", {"rand_str"}, "", {"rand_str"}, sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_EQ("Could not find a facet field named `rand_str` in the schema.", res_op.error());++ results = coll1->search("fizzbuzz", {"rand_str"}, "", {"org_year"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());++ collectionManager.drop_collection("coll1");+}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex e51757c0..72fd5d5b 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -27,7 +27,8 @@ protected:field("title", field_types::STRING, false),field("starring", field_types::STRING, false),field("cast", field_types::STRING_ARRAY, true, true),- field("location", field_types::GEOPOINT, false, true, 14),+ field(".*_year", field_types::INT32, true, true),+ field("location", field_types::GEOPOINT, false, true, true, 14),field("points", field_types::INT32, false)};@@ -88,6 +89,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {"\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"},""{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},""{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"+ "{\"facet\":true,\"name\":\".*_year\",\"optional\":true,\"type\":\"int32\"},""{\"facet\":false,\"geo_resolution\":14,\"name\":\"location\",\"optional\":true,\"type\":\"geopoint\"},""{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,""\"name\":\"collection1\",\"num_memory_shards\":4}",