Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow fields to be marked as non-indexable.

kishorenc75caa2ab0113
diff --git a/include/field.h b/include/field.h
index 9ca5bfca..2d33f538 100644
--- a/include/field.h
+++ b/include/field.h
@@ -226,6 +226,14 @@ struct field {
if(field.is_dynamic() && !field.optional) {
return Option<bool>(400, "Field `" + field.name + "` with wildcard name must be an optional field.");
}
+
+ if(!field.index && !field.optional) {
+ return Option<bool>(400, "Field `" + field.name + "` must be optional since it is marked as non-indexable.");
+ }
+
+ if(!field.index && field.is_auto()) {
+ return Option<bool>(400, "Field `" + field.name + "` cannot be marked as non-indexable.");
+ }
}
if(!default_sorting_field.empty() && !found_default_sorting_field) {
@@ -239,6 +247,7 @@ struct field {
static Option<bool> json_fields_to_fields(nlohmann::json& fields_json,
std::string& fallback_field_type,
std::vector<field>& fields) {
+
size_t num_auto_detect_fields = 0;
for(nlohmann::json & field_json: fields_json) {
@@ -260,6 +269,11 @@ struct field {
field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
}
+ if(field_json.count(fields::index) != 0 && !field_json.at(fields::index).is_boolean()) {
+ return Option<bool>(400, std::string("The `index` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
if(field_json.count(fields::geo_resolution) != 0) {
if(!field_json.at(fields::geo_resolution).is_number_integer()) {
return Option<bool>(400, std::string("The `geo_resolution` property of the field `") +
@@ -274,22 +288,35 @@ struct field {
}
if(field_json["name"] == "*") {
- if(field_json.count("facet") == 0) {
- field_json["facet"] = false;
+ if(field_json.count(fields::facet) == 0) {
+ field_json[fields::facet] = false;
+ }
+
+ if(field_json.count(fields::optional) == 0) {
+ field_json[fields::optional] = true;
+ }
+
+ if(field_json.count(fields::index) == 0) {
+ field_json[fields::index] = true;
}
- if(field_json.count("optional") == 0) {
- field_json["optional"] = true;
+ if(field_json.count(fields::geo_resolution) != 0) {
+ return Option<bool>(400, "Field `*` cannot contain a geo resolution.");
}
- if(field_json["optional"] == false) {
+ if(field_json[fields::optional] == false) {
return Option<bool>(400, "Field `*` must be an optional field.");
}
- if(field_json["facet"] == true) {
+ if(field_json[fields::facet] == true) {
return Option<bool>(400, "Field `*` cannot be a facet field.");
}
+ if(field_json[fields::index] == false) {
+ return Option<bool>(400, "Field `*` must be an index field.");
+ }
+
+
field fallback_field(field_json["name"], field_json["type"], field_json["facet"],
field_json["optional"]);
@@ -304,21 +331,28 @@ struct field {
continue;
}
- if(field_json.count("facet") == 0) {
- field_json["facet"] = false;
+ if(field_json.count(fields::facet) == 0) {
+ field_json[fields::facet] = false;
}
- if(field_json.count("optional") == 0) {
- // actual value will depend on whether field name is dynamic or not (i.e. contains .*)
- if(field_json["name"].get<std::string>().find(".*") != std::string::npos) {
- field_json["optional"] = true;
- } else {
- field_json["optional"] = false;
- }
+ if(field_json.count(fields::index) == 0) {
+ field_json[fields::index] = true;
+ }
+
+ bool is_dynamic = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;
+
+ if(field_json.count(fields::optional) == 0) {
+ // dynamic fields are always optional
+ field_json[fields::optional] = is_dynamic;
+ }
+
+ if(field_json.count(fields::geo_resolution) == 0) {
+ field_json[fields::geo_resolution] = DEFAULT_GEO_RESOLUTION;
}
fields.emplace_back(
- field(field_json["name"], field_json["type"], field_json["facet"], field_json["optional"])
+ field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
+ field_json[fields::optional], fields_json[fields::index], field_json[fields::geo_resolution])
);
}
diff --git a/src/collection.cpp b/src/collection.cpp
index f9418328..0e4e1ee4 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -600,6 +600,12 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
field search_field = search_schema.at(field_name);
+
+ if(!search_field.index) {
+ std::string error = "Field `" + field_name + "` is marked as a non-indexed field in the schema.";
+ return Option<nlohmann::json>(404, error);
+ }
+
if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {
std::string error = "Field `" + field_name + "` should be a string or a string array.";
return Option<nlohmann::json>(400, error);
@@ -2323,6 +2329,11 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
UPDATE_SCHEMA:
+ if(!new_field.index) {
+ kv++;
+ continue;
+ }
+
search_schema.emplace(new_field.name, new_field);
fields.emplace_back(new_field);
new_fields.emplace_back(new_field);
diff --git a/src/index.cpp b/src/index.cpp
index 9b5c5975..b2cf8566 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -125,7 +125,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
for(const auto& field_pair: search_schema) {
const std::string & field_name = field_pair.first;
- if(document.count(field_name) == 0) {
+ if(document.count(field_name) == 0 || !field_pair.second.index) {
continue;
}
@@ -2336,6 +2336,10 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
const auto& search_field = search_field_it->second;
+ if(!search_field.index) {
+ continue;
+ }
+
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
std::vector<std::string> tokens;
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index ade5aaa5..8ad055f4 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -502,6 +502,14 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
ASSERT_EQ("*", fields[0].name);
ASSERT_EQ("string*", fields[0].type);
+ // reject when you try to set geo property on * field
+ fields_json[0][fields::geo_resolution] = 10;
+ parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+
+ ASSERT_FALSE(parse_op.ok());
+ ASSERT_EQ("Field `*` cannot contain a geo resolution.", parse_op.error());
+ fields_json[0].erase(fields::geo_resolution);
+
// reject when you try to set optional to false or facet to true
fields_json[0][fields::optional] = false;
parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
@@ -793,3 +801,82 @@ TEST_F(CollectionAllFieldsTest, ContainingWildcardOnlyField) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionAllFieldsTest, DoNotIndexFieldMarkedAsNonIndex) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("company_name", field_types::STRING, false),
+ field("num_employees", field_types::INT32, false),
+ field("post", field_types::STRING, false, true, false),
+ field(".*_txt", field_types::STRING, true, true, false),
+ field("*", field_types::AUTO, false, true)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);
+ ASSERT_TRUE(op.ok());
+ coll1 = op.get();
+ }
+
+ nlohmann::json doc;
+ doc["company_name"] = "Amazon Inc.";
+ doc["num_employees"] = 2000;
+ doc["post"] = "Some post.";
+ doc["description_txt"] = "Rome was not built in a day.";
+
+ auto add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto res_op = coll1->search("Amazon", {"description_txt"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Could not find a field named `description_txt` in the schema.", res_op.error());
+
+ res_op = coll1->search("Amazon", {"post"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Field `post` is marked as a non-indexed field in the schema.", res_op.error());
+
+ // try updating a document with non-indexable field
+ doc["post"] = "Some post updated.";
+ auto update_op = coll1->add(doc.dump(), UPDATE, "0");
+ ASSERT_TRUE(add_op.ok());
+
+ auto res = coll1->search("Amazon", {"company_name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ("Some post updated.", res["hits"][0]["document"]["post"].get<std::string>());
+
+ // try to delete doc with non-indexable field
+ auto del_op = coll1->remove("0");
+ ASSERT_TRUE(del_op.ok());
+
+ fields = {field("company_name", field_types::STRING, false),
+ field("num_employees", field_types::INT32, false),
+ field("post", field_types::STRING, false, false, false),
+ field(".*_txt", field_types::STRING, true, true, false),
+ field("*", field_types::AUTO, false, true)};
+
+ auto op = collectionManager.create_collection("coll2", 1, fields, "", 0, field_types::AUTO);
+ ASSERT_FALSE(op.ok());
+ ASSERT_EQ("Field `post` must be optional since it is marked as non-indexable.", op.error());
+
+ fields = {field("company_name", field_types::STRING, false),
+ field("num_employees", field_types::INT32, false),
+ field("post", field_types::STRING, false, true, false),
+ field(".*_txt", field_types::STRING, true, false, false),
+ field("*", field_types::AUTO, false, true)};
+
+ op = collectionManager.create_collection("coll2", 1, fields, "", 0, field_types::AUTO);
+ ASSERT_FALSE(op.ok());
+ ASSERT_EQ("Field `.*_txt` with wildcard name must be an optional field.", op.error());
+
+ // don't allow catch all field to contain non-index field
+
+ fields = {field("company_name", field_types::STRING, false),
+ field("num_employees", field_types::INT32, false),
+ field(".*_txt", field_types::STRING, true, true, false),
+ field("*", field_types::AUTO, false, true, false)};
+
+ op = collectionManager.create_collection("coll2", 1, fields, "", 0, field_types::AUTO);
+ ASSERT_FALSE(op.ok());
+ ASSERT_EQ("Field `*` cannot be marked as non-indexable.", op.error());
+
+ collectionManager.drop_collection("coll1");
+}