Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Non index fields should not create index structures.

Kishore Nallana6b7b2a3a7a1
diff --git a/src/index.cpp b/src/index.cpp
index 31902801..7b81963b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -47,12 +47,14 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*
seq_ids(new id_list_t(256)), symbols_to_index(symbols_to_index), token_separators(token_separators) {
for(const auto & fname_field: search_schema) {
+ if(!fname_field.second.index) {
+ continue;
+ }
+
if(fname_field.second.is_string()) {
- if(fname_field.second.index) {
- art_tree *t = new art_tree;
- art_tree_init(t);
- search_index.emplace(fname_field.first, t);
- }
+ art_tree *t = new art_tree;
+ art_tree_init(t);
+ search_index.emplace(fname_field.first, t);
} else if(fname_field.second.is_geopoint()) {
auto field_geo_index = new spp::sparse_hash_map<std::string, std::vector<uint32_t>>();
geopoint_index.emplace(fname_field.first, field_geo_index);
@@ -633,6 +635,10 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
return;
}
+ if(!afield.index) {
+ return;
+ }
+
// We have to handle both these edge cases:
// a) `afield` might not exist in the document (optional field)
// b) `afield` value could be empty
@@ -653,7 +659,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
const auto& document = record.doc;
const auto seq_id = record.seq_id;
- if(document.count(afield.name) == 0 || !afield.index || !record.indexed.ok()) {
+ if(document.count(afield.name) == 0 || !record.indexed.ok()) {
continue;
}
@@ -3930,6 +3936,10 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
for(const auto & new_field: new_fields) {
search_schema.emplace(new_field.name, new_field);
+ if(!new_field.index) {
+ continue;
+ }
+
if(new_field.is_sortable()) {
if(new_field.is_num_sortable()) {
spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index c302449b..c2eba296 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -191,11 +191,20 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
ASSERT_FALSE(alter_op.ok());
ASSERT_EQ("There can be only one field named `.*`.", alter_op.error());
- auto search_schema = coll1->get_schema();
- ASSERT_EQ(7, search_schema.size());
+ // add non-index field
+ schema_changes = R"({
+ "fields": [
+ {"name": "raw", "type": "int32", "index": false, "optional": true}
+ ]
+ })"_json;
- auto coll_fields = coll1->get_fields();
- ASSERT_EQ(8, coll_fields.size());
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ ASSERT_EQ(8, coll1->get_schema().size());
+ ASSERT_EQ(9, coll1->get_fields().size());
+
+ ASSERT_EQ(4, coll1->_get_index()->_get_numerical_index().size());
// try restoring collection from disk: all fields should be preserved
collectionManager.dispose();
@@ -206,11 +215,10 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
collectionManager.load(8, 1000);
coll1 = collectionManager.get_collection("coll1").get();
- search_schema = coll1->get_schema();
- ASSERT_EQ(7, search_schema.size());
+ ASSERT_EQ(8, coll1->get_schema().size());
+ ASSERT_EQ(9, coll1->get_fields().size());
- coll_fields = coll1->get_fields();
- ASSERT_EQ(8, coll_fields.size());
+ ASSERT_EQ(4, coll1->_get_index()->_get_numerical_index().size());
collectionManager.drop_collection("coll1");
}
@@ -314,14 +322,13 @@ TEST_F(CollectionSchemaChangeTest, DropFieldsFromCollection) {
}
TEST_F(CollectionSchemaChangeTest, AlterValidations) {
- std::vector<field> fields = {field(".*", field_types::AUTO, false),
- field("title", field_types::STRING, false, false, true, "", 1, 1),
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", 1, 1),
field("location", field_types::GEOPOINT, false),
field("locations", field_types::GEOPOINT_ARRAY, false),
field("tags", field_types::STRING_ARRAY, true),
field("points", field_types::INT32, true),};
- Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "auto").get();
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "").get();
std::vector<std::vector<double>> lat_lngs;
lat_lngs.push_back({48.85821022164442, 2.294239067890161});
@@ -332,6 +339,7 @@ TEST_F(CollectionSchemaChangeTest, AlterValidations) {
doc["location"] = {48.85821022164442, 2.294239067890161};
doc["locations"] = lat_lngs;
doc["tags"] = {"experimental", "news"};
+ doc["desc"] = "Story about fox.";
doc["points"] = 100;
ASSERT_TRUE(coll1->add(doc.dump()).ok());
@@ -361,6 +369,56 @@ TEST_F(CollectionSchemaChangeTest, AlterValidations) {
ASSERT_EQ("Wrong format for `fields`. It should be an array of objects containing `name`, `type`, "
"`optional` and `facet` properties.",alter_op.error());
+ // 3. Try to drop non-existing field
+ schema_changes = R"({
+ "fields": [
+ {"name": "age", "drop": true}
+ ]
+ })"_json;
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Field `age` is not part of collection schema.",alter_op.error());
+
+ // 4. Bad value for `drop` parameter
+ schema_changes = R"({
+ "fields": [
+ {"name": "title", "drop": 123}
+ ]
+ })"_json;
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Field `title` must have a drop value of `true`.", alter_op.error());
+
+ // 5. New field schema should match on-disk data
+ schema_changes = R"({
+ "fields": [
+ {"name": "desc", "type": "int32"}
+ ]
+ })"_json;
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Schema change does not match on-disk data, error: Field `desc` must be an int32.", alter_op.error());
+
+ // 6. Prevent non-optional field when on-disk data has missing values
+ doc.clear();
+ doc["id"] = "1";
+ doc["title"] = "The brown lion was too slow.";
+ doc["location"] = {68.85821022164442, 4.294239067890161};
+ doc["locations"] = lat_lngs;
+ doc["tags"] = {"lion", "zoo"};
+ doc["points"] = 200;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ schema_changes = R"({
+ "fields": [
+ {"name": "desc", "type": "string", "optional": false}
+ ]
+ })"_json;
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Schema change does not match on-disk data, error: Field `desc` has been declared in the "
+ "schema, but is not found in the document.", alter_op.error());
+
collectionManager.drop_collection("coll1");
}
-
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 32cf3193..28155f18 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -2710,26 +2710,24 @@ TEST_F(CollectionSpecificTest, RepeatedTokensInArray) {
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionSpecificTest, HighlightOnlyIncludedFields) {
+TEST_F(CollectionSpecificTest, NonIndexField) {
std::vector<field> fields = {field("title", field_types::STRING, false),
- field("description", field_types::STRING, false),};
+ field("description", field_types::STRING, false, true, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["title"] = "Harry";
- doc1["description"] = "Harry Roscow";
+ doc1["description"] = "A book.";
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
- spp::sparse_hash_set<std::string> include_fields = {"title"};
- auto results = coll1->search("harry", {"title", "description"},
- "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 10, include_fields).get();
+ auto results = coll1->search("harry", {"title"},
+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();
ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ(1, results["hits"][0]["highlights"].size());
- ASSERT_EQ("title", results["hits"][0]["highlights"][0]["field"].get<std::string>());
+ ASSERT_EQ(1, coll1->_get_index()->_get_search_index().size());
collectionManager.drop_collection("coll1");
}