Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow fields to be marked as non-indexable.
kishorenc75caa2ab0113
diff --git a/include/field.h b/include/field.hindex 9ca5bfca..2d33f538 100644--- a/include/field.h+++ b/include/field.h@@ -226,6 +226,14 @@ struct field {if(field.is_dynamic() && !field.optional) {return Option<bool>(400, "Field `" + field.name + "` with wildcard name must be an optional field.");}++ if(!field.index && !field.optional) {+ return Option<bool>(400, "Field `" + field.name + "` must be optional since it is marked as non-indexable.");+ }++ if(!field.index && field.is_auto()) {+ return Option<bool>(400, "Field `" + field.name + "` cannot be marked as non-indexable.");+ }}if(!default_sorting_field.empty() && !found_default_sorting_field) {@@ -239,6 +247,7 @@ struct field {static Option<bool> json_fields_to_fields(nlohmann::json& fields_json,std::string& fallback_field_type,std::vector<field>& fields) {+size_t num_auto_detect_fields = 0;for(nlohmann::json & field_json: fields_json) {@@ -260,6 +269,11 @@ struct field {field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));}+ if(field_json.count(fields::index) != 0 && !field_json.at(fields::index).is_boolean()) {+ return Option<bool>(400, std::string("The `index` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }+if(field_json.count(fields::geo_resolution) != 0) {if(!field_json.at(fields::geo_resolution).is_number_integer()) {return Option<bool>(400, std::string("The `geo_resolution` property of the field `") +@@ -274,22 +288,35 @@ struct field {}if(field_json["name"] == "*") {- if(field_json.count("facet") == 0) {- field_json["facet"] = false;+ if(field_json.count(fields::facet) == 0) {+ field_json[fields::facet] = false;+ }++ if(field_json.count(fields::optional) == 0) {+ field_json[fields::optional] = true;+ }++ if(field_json.count(fields::index) == 0) {+ field_json[fields::index] = true;}- if(field_json.count("optional") == 0) {- field_json["optional"] = true;+ if(field_json.count(fields::geo_resolution) != 0) {+ return Option<bool>(400, "Field `*` cannot contain a geo resolution.");}- if(field_json["optional"] == false) {+ if(field_json[fields::optional] == false) {return Option<bool>(400, "Field `*` must be an optional field.");}- if(field_json["facet"] == true) {+ if(field_json[fields::facet] == true) {return Option<bool>(400, "Field `*` cannot be a facet field.");}+ if(field_json[fields::index] == false) {+ return Option<bool>(400, "Field `*` must be an index field.");+ }++field fallback_field(field_json["name"], field_json["type"], field_json["facet"],field_json["optional"]);@@ -304,21 +331,28 @@ struct field {continue;}- if(field_json.count("facet") == 0) {- field_json["facet"] = false;+ if(field_json.count(fields::facet) == 0) {+ field_json[fields::facet] = false;}- if(field_json.count("optional") == 0) {- // actual value will depend on whether field name is dynamic or not (i.e. contains .*)- if(field_json["name"].get<std::string>().find(".*") != std::string::npos) {- field_json["optional"] = true;- } else {- field_json["optional"] = false;- }+ if(field_json.count(fields::index) == 0) {+ field_json[fields::index] = true;+ }++ bool is_dynamic = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;++ if(field_json.count(fields::optional) == 0) {+ // dynamic fields are always optional+ field_json[fields::optional] = is_dynamic;+ }++ if(field_json.count(fields::geo_resolution) == 0) {+ field_json[fields::geo_resolution] = DEFAULT_GEO_RESOLUTION;}fields.emplace_back(- field(field_json["name"], field_json["type"], field_json["facet"], field_json["optional"])+ field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],+ field_json[fields::optional], fields_json[fields::index], field_json[fields::geo_resolution]));}diff --git a/src/collection.cpp b/src/collection.cppindex f9418328..0e4e1ee4 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -600,6 +600,12 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}field search_field = search_schema.at(field_name);++ if(!search_field.index) {+ std::string error = "Field `" + field_name + "` is marked as a non-indexed field in the schema.";+ return Option<nlohmann::json>(404, error);+ }+if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {std::string error = "Field `" + field_name + "` should be a string or a string array.";return Option<nlohmann::json>(400, error);@@ -2323,6 +2329,11 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constUPDATE_SCHEMA:+ if(!new_field.index) {+ kv++;+ continue;+ }+search_schema.emplace(new_field.name, new_field);fields.emplace_back(new_field);new_fields.emplace_back(new_field);diff --git a/src/index.cpp b/src/index.cppindex 9b5c5975..b2cf8566 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -125,7 +125,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tfor(const auto& field_pair: search_schema) {const std::string & field_name = field_pair.first;- if(document.count(field_name) == 0) {+ if(document.count(field_name) == 0 || !field_pair.second.index) {continue;}@@ -2336,6 +2336,10 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docconst auto& search_field = search_field_it->second;+ if(!search_field.index) {+ continue;+ }+// Go through all the field names and find the keys+values so that they can be removed from in-memory indexif(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {std::vector<std::string> tokens;diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex ade5aaa5..8ad055f4 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -502,6 +502,14 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {ASSERT_EQ("*", fields[0].name);ASSERT_EQ("string*", fields[0].type);+ // reject when you try to set geo property on * field+ fields_json[0][fields::geo_resolution] = 10;+ parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);++ ASSERT_FALSE(parse_op.ok());+ ASSERT_EQ("Field `*` cannot contain a geo resolution.", parse_op.error());+ fields_json[0].erase(fields::geo_resolution);+// reject when you try to set optional to false or facet to truefields_json[0][fields::optional] = false;parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);@@ -793,3 +801,82 @@ TEST_F(CollectionAllFieldsTest, ContainingWildcardOnlyField) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionAllFieldsTest, DoNotIndexFieldMarkedAsNonIndex) {+ Collection *coll1;++ std::vector<field> fields = {field("company_name", field_types::STRING, false),+ field("num_employees", field_types::INT32, false),+ field("post", field_types::STRING, false, true, false),+ field(".*_txt", field_types::STRING, true, true, false),+ field("*", field_types::AUTO, false, true)};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);+ ASSERT_TRUE(op.ok());+ coll1 = op.get();+ }++ nlohmann::json doc;+ doc["company_name"] = "Amazon Inc.";+ doc["num_employees"] = 2000;+ doc["post"] = "Some post.";+ doc["description_txt"] = "Rome was not built in a day.";++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto res_op = coll1->search("Amazon", {"description_txt"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `description_txt` in the schema.", res_op.error());++ res_op = coll1->search("Amazon", {"post"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Field `post` is marked as a non-indexed field in the schema.", res_op.error());++ // try updating a document with non-indexable field+ doc["post"] = "Some post updated.";+ auto update_op = coll1->add(doc.dump(), UPDATE, "0");+ ASSERT_TRUE(add_op.ok());++ auto res = coll1->search("Amazon", {"company_name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ("Some post updated.", res["hits"][0]["document"]["post"].get<std::string>());++ // try to delete doc with non-indexable field+ auto del_op = coll1->remove("0");+ ASSERT_TRUE(del_op.ok());++ fields = {field("company_name", field_types::STRING, false),+ field("num_employees", field_types::INT32, false),+ field("post", field_types::STRING, false, false, false),+ field(".*_txt", field_types::STRING, true, true, false),+ field("*", field_types::AUTO, false, true)};++ auto op = collectionManager.create_collection("coll2", 1, fields, "", 0, field_types::AUTO);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("Field `post` must be optional since it is marked as non-indexable.", op.error());++ fields = {field("company_name", field_types::STRING, false),+ field("num_employees", field_types::INT32, false),+ field("post", field_types::STRING, false, true, false),+ field(".*_txt", field_types::STRING, true, false, false),+ field("*", field_types::AUTO, false, true)};++ op = collectionManager.create_collection("coll2", 1, fields, "", 0, field_types::AUTO);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("Field `.*_txt` with wildcard name must be an optional field.", op.error());++ // don't allow catch all field to contain non-index field++ fields = {field("company_name", field_types::STRING, false),+ field("num_employees", field_types::INT32, false),+ field(".*_txt", field_types::STRING, true, true, false),+ field("*", field_types::AUTO, false, true, false)};++ op = collectionManager.create_collection("coll2", 1, fields, "", 0, field_types::AUTO);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("Field `*` cannot be marked as non-indexable.", op.error());++ collectionManager.drop_collection("coll1");+}