Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Non index fields should not create index structures.
Kishore Nallana6b7b2a3a7a1
diff --git a/src/index.cpp b/src/index.cppindex 31902801..7b81963b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -47,12 +47,14 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*seq_ids(new id_list_t(256)), symbols_to_index(symbols_to_index), token_separators(token_separators) {for(const auto & fname_field: search_schema) {+ if(!fname_field.second.index) {+ continue;+ }+if(fname_field.second.is_string()) {- if(fname_field.second.index) {- art_tree *t = new art_tree;- art_tree_init(t);- search_index.emplace(fname_field.first, t);- }+ art_tree *t = new art_tree;+ art_tree_init(t);+ search_index.emplace(fname_field.first, t);} else if(fname_field.second.is_geopoint()) {auto field_geo_index = new spp::sparse_hash_map<std::string, std::vector<uint32_t>>();geopoint_index.emplace(fname_field.first, field_geo_index);@@ -633,6 +635,10 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>return;}+ if(!afield.index) {+ return;+ }+// We have to handle both these edge cases:// a) `afield` might not exist in the document (optional field)// b) `afield` value could be empty@@ -653,7 +659,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>const auto& document = record.doc;const auto seq_id = record.seq_id;- if(document.count(afield.name) == 0 || !afield.index || !record.indexed.ok()) {+ if(document.count(afield.name) == 0 || !record.indexed.ok()) {continue;}@@ -3930,6 +3936,10 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecfor(const auto & new_field: new_fields) {search_schema.emplace(new_field.name, new_field);+ if(!new_field.index) {+ continue;+ }+if(new_field.is_sortable()) {if(new_field.is_num_sortable()) {spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex c302449b..c2eba296 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -191,11 +191,20 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {ASSERT_FALSE(alter_op.ok());ASSERT_EQ("There can be only one field named `.*`.", alter_op.error());- auto search_schema = coll1->get_schema();- ASSERT_EQ(7, search_schema.size());+ // add non-index field+ schema_changes = R"({+ "fields": [+ {"name": "raw", "type": "int32", "index": false, "optional": true}+ ]+ })"_json;- auto coll_fields = coll1->get_fields();- ASSERT_EQ(8, coll_fields.size());+ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ ASSERT_EQ(8, coll1->get_schema().size());+ ASSERT_EQ(9, coll1->get_fields().size());++ ASSERT_EQ(4, coll1->_get_index()->_get_numerical_index().size());// try restoring collection from disk: all fields should be preservedcollectionManager.dispose();@@ -206,11 +215,10 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {collectionManager.load(8, 1000);coll1 = collectionManager.get_collection("coll1").get();- search_schema = coll1->get_schema();- ASSERT_EQ(7, search_schema.size());+ ASSERT_EQ(8, coll1->get_schema().size());+ ASSERT_EQ(9, coll1->get_fields().size());- coll_fields = coll1->get_fields();- ASSERT_EQ(8, coll_fields.size());+ ASSERT_EQ(4, coll1->_get_index()->_get_numerical_index().size());collectionManager.drop_collection("coll1");}@@ -314,14 +322,13 @@ TEST_F(CollectionSchemaChangeTest, DropFieldsFromCollection) {}TEST_F(CollectionSchemaChangeTest, AlterValidations) {- std::vector<field> fields = {field(".*", field_types::AUTO, false),- field("title", field_types::STRING, false, false, true, "", 1, 1),+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", 1, 1),field("location", field_types::GEOPOINT, false),field("locations", field_types::GEOPOINT_ARRAY, false),field("tags", field_types::STRING_ARRAY, true),field("points", field_types::INT32, true),};- Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "auto").get();+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "").get();std::vector<std::vector<double>> lat_lngs;lat_lngs.push_back({48.85821022164442, 2.294239067890161});@@ -332,6 +339,7 @@ TEST_F(CollectionSchemaChangeTest, AlterValidations) {doc["location"] = {48.85821022164442, 2.294239067890161};doc["locations"] = lat_lngs;doc["tags"] = {"experimental", "news"};+ doc["desc"] = "Story about fox.";doc["points"] = 100;ASSERT_TRUE(coll1->add(doc.dump()).ok());@@ -361,6 +369,56 @@ TEST_F(CollectionSchemaChangeTest, AlterValidations) {ASSERT_EQ("Wrong format for `fields`. It should be an array of objects containing `name`, `type`, ""`optional` and `facet` properties.",alter_op.error());+ // 3. Try to drop non-existing field+ schema_changes = R"({+ "fields": [+ {"name": "age", "drop": true}+ ]+ })"_json;+ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Field `age` is not part of collection schema.",alter_op.error());++ // 4. Bad value for `drop` parameter+ schema_changes = R"({+ "fields": [+ {"name": "title", "drop": 123}+ ]+ })"_json;+ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Field `title` must have a drop value of `true`.", alter_op.error());++ // 5. New field schema should match on-disk data+ schema_changes = R"({+ "fields": [+ {"name": "desc", "type": "int32"}+ ]+ })"_json;+ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Schema change does not match on-disk data, error: Field `desc` must be an int32.", alter_op.error());++ // 6. Prevent non-optional field when on-disk data has missing values+ doc.clear();+ doc["id"] = "1";+ doc["title"] = "The brown lion was too slow.";+ doc["location"] = {68.85821022164442, 4.294239067890161};+ doc["locations"] = lat_lngs;+ doc["tags"] = {"lion", "zoo"};+ doc["points"] = 200;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ schema_changes = R"({+ "fields": [+ {"name": "desc", "type": "string", "optional": false}+ ]+ })"_json;+ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Schema change does not match on-disk data, error: Field `desc` has been declared in the "+ "schema, but is not found in the document.", alter_op.error());+collectionManager.drop_collection("coll1");}-diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 32cf3193..28155f18 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -2710,26 +2710,24 @@ TEST_F(CollectionSpecificTest, RepeatedTokensInArray) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionSpecificTest, HighlightOnlyIncludedFields) {+TEST_F(CollectionSpecificTest, NonIndexField) {std::vector<field> fields = {field("title", field_types::STRING, false),- field("description", field_types::STRING, false),};+ field("description", field_types::STRING, false, true, false),};Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();nlohmann::json doc1;doc1["id"] = "0";doc1["title"] = "Harry";- doc1["description"] = "Harry Roscow";+ doc1["description"] = "A book.";ASSERT_TRUE(coll1->add(doc1.dump()).ok());- spp::sparse_hash_set<std::string> include_fields = {"title"};- auto results = coll1->search("harry", {"title", "description"},- "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 10, include_fields).get();+ auto results = coll1->search("harry", {"title"},+ "", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ(1, results["hits"][0]["highlights"].size());- ASSERT_EQ("title", results["hits"][0]["highlights"][0]["field"].get<std::string>());+ ASSERT_EQ(1, coll1->_get_index()->_get_search_index().size());collectionManager.drop_collection("coll1");}