Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Schema change basics.
Kishore Nallana5b06a0e0db6
diff --git a/include/collection.h b/include/collection.hindex 93fd5911..fca21351 100644--- a/include/collection.h+++ b/include/collection.h@@ -81,11 +81,11 @@ private:std::map<std::string, override_t> overrides;- const std::string default_sorting_field;+ std::string default_sorting_field;const float max_memory_ratio;- const std::string fallback_field_type;+ std::string fallback_field_type;std::vector<field> dynamic_fields;@@ -153,6 +153,13 @@ private:Option<bool> validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,std::vector<sort_by>& sort_fields_std) const;+ Option<bool> persist_collection_meta();++ Option<bool> validate_alter_payload(nlohmann::json& schema_changes,+ std::unordered_map<std::string, field>& schema_additions,+ std::vector<field>& del_fields,+ std::string& fallback_field_type);+public:enum {MAX_ARRAY_MATCHES = 5};@@ -315,9 +322,6 @@ public:Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);- bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,- std::string &value);-size_t get_num_documents() const;DIRTY_VALUES parse_dirty_values_option(std::string& dirty_values) const;@@ -326,6 +330,8 @@ public:std::vector<char> get_token_separators();+ std::string get_fallback_field_type();+// Override operationsOption<uint32_t> add_override(const override_t & override);@@ -365,5 +371,7 @@ public:const std::string& highlight_full_fields,const std::vector<infix_t>& infixes,std::vector<highlight_field_t>& highlight_items) const;++ Option<bool> alter(nlohmann::json& alter_payload);};diff --git a/include/collection_manager.h b/include/collection_manager.hindex c0e1dbd0..c77a067c 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -152,6 +152,8 @@ public:const std::vector<std::string>& symbols_to_index = {},const std::vector<std::string>& token_separators = {});+ Option<bool> update_collection(const std::string& name, nlohmann::json& req_json);+locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;diff --git a/include/core_api.h b/include/core_api.hindex 62a9a0f4..a321d986 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -14,6 +14,8 @@ bool get_collections(const std::shared_ptr<http_req>& req, const std::shared_ptrbool post_create_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool del_drop_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);bool get_collection_summary(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);diff --git a/include/field.h b/include/field.hindex ae37c0c0..d948c644 100644--- a/include/field.h+++ b/include/field.h@@ -290,7 +290,7 @@ struct field {}}- if(!default_sorting_field.empty() && !found_default_sorting_field) {+ if(!default_sorting_field.empty() && !found_default_sorting_field && !fields.empty()) {return Option<bool>(400, "Default sorting field is defined as `" + default_sorting_field +"` but is not found in the schema.");}@@ -325,155 +325,20 @@ struct field {return Option<bool>(true);}+ static Option<bool> json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,+ string& fallback_field_type, size_t& num_auto_detect_fields);+static Option<bool> json_fields_to_fields(nlohmann::json& fields_json,std::string& fallback_field_type,- std::vector<field>& fields) {+ std::vector<field>& the_fields) {size_t num_auto_detect_fields = 0;for(nlohmann::json & field_json: fields_json) {- if(field_json["name"] == "id") {- // No field should exist with the name "id" as it is reserved for internal use- // We cannot throw an error here anymore since that will break backward compatibility!- LOG(WARNING) << "Collection schema cannot contain a field with name `id`. Ignoring field.";- continue;- }-- if(!field_json.is_object() ||- field_json.count(fields::name) == 0 || field_json.count(fields::type) == 0 ||- !field_json.at(fields::name).is_string() || !field_json.at(fields::type).is_string()) {-- return Option<bool>(400, "Wrong format for `fields`. It should be an array of objects containing "- "`name`, `type`, `optional` and `facet` properties.");- }-- if(field_json.count(fields::facet) != 0 && !field_json.at(fields::facet).is_boolean()) {- return Option<bool>(400, std::string("The `facet` property of the field `") +- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));- }-- if(field_json.count(fields::optional) != 0 && !field_json.at(fields::optional).is_boolean()) {- return Option<bool>(400, std::string("The `optional` property of the field `") +- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));- }-- if(field_json.count(fields::index) != 0 && !field_json.at(fields::index).is_boolean()) {- return Option<bool>(400, std::string("The `index` property of the field `") +- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));- }-- if(field_json.count(fields::sort) != 0 && !field_json.at(fields::sort).is_boolean()) {- return Option<bool>(400, std::string("The `sort` property of the field `") +- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));- }-- if(field_json.count(fields::infix) != 0 && !field_json.at(fields::infix).is_boolean()) {- return Option<bool>(400, std::string("The `infix` property of the field `") +- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));- }-- if(field_json.count(fields::locale) != 0){- if(!field_json.at(fields::locale).is_string()) {- return Option<bool>(400, std::string("The `locale` property of the field `") +- field_json[fields::name].get<std::string>() + std::string("` should be a string."));- }-- if(!field_json[fields::locale].get<std::string>().empty() &&- field_json[fields::locale].get<std::string>().size() != 2) {- return Option<bool>(400, std::string("The `locale` value of the field `") +- field_json[fields::name].get<std::string>() + std::string("` is not valid."));- }- }-- if(field_json["name"] == ".*") {- if(field_json.count(fields::facet) == 0) {- field_json[fields::facet] = false;- }-- if(field_json.count(fields::optional) == 0) {- field_json[fields::optional] = true;- }-- if(field_json.count(fields::index) == 0) {- field_json[fields::index] = true;- }-- if(field_json.count(fields::locale) == 0) {- field_json[fields::locale] = "";- }-- if(field_json.count(fields::sort) == 0) {- field_json[fields::sort] = false;- }-- if(field_json.count(fields::infix) == 0) {- field_json[fields::infix] = false;- }-- if(field_json[fields::optional] == false) {- return Option<bool>(400, "Field `.*` must be an optional field.");- }-- if(field_json[fields::facet] == true) {- return Option<bool>(400, "Field `.*` cannot be a facet field.");- }-- if(field_json[fields::index] == false) {- return Option<bool>(400, "Field `.*` must be an index field.");- }-- field fallback_field(field_json["name"], field_json["type"], field_json["facet"],- field_json["optional"], field_json[fields::index], field_json[fields::locale],- field_json[fields::sort], field_json[fields::infix]);-- if(fallback_field.has_valid_type()) {- fallback_field_type = fallback_field.type;- num_auto_detect_fields++;- } else {- return Option<bool>(400, "The `type` of field `.*` is invalid.");- }-- fields.emplace_back(fallback_field);- continue;+ auto op = json_field_to_field(field_json, the_fields, fallback_field_type, num_auto_detect_fields);+ if(!op.ok()) {+ return op;}-- if(field_json.count(fields::facet) == 0) {- field_json[fields::facet] = false;- }-- if(field_json.count(fields::index) == 0) {- field_json[fields::index] = true;- }-- if(field_json.count(fields::locale) == 0) {- field_json[fields::locale] = "";- }-- if(field_json.count(fields::sort) == 0) {- if(field_json["type"] == field_types::INT32 || field_json["type"] == field_types::INT64 ||- field_json["type"] == field_types::FLOAT || field_json["type"] == field_types::BOOL ||- field_json["type"] == field_types::GEOPOINT || field_json["type"] == field_types::GEOPOINT_ARRAY) {- field_json[fields::sort] = true;- } else {- field_json[fields::sort] = false;- }- }-- if(field_json.count(fields::infix) == 0) {- field_json[fields::infix] = false;- }-- if(field_json.count(fields::optional) == 0) {- // dynamic fields are always optional- bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);- field_json[fields::optional] = is_dynamic;- }-- fields.emplace_back(- field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],- field_json[fields::optional], field_json[fields::index], field_json[fields::locale],- field_json[fields::sort], field_json[fields::infix])- );}if(num_auto_detect_fields > 1) {diff --git a/include/index.h b/include/index.hindex 863834dd..9f29a6cd 100644--- a/include/index.h+++ b/include/index.h@@ -268,9 +268,10 @@ struct override_t {if(!filter_by.empty()) {override["filter_by"] = filter_by;- override["remove_matched_tokens"] = remove_matched_tokens;}+ override["remove_matched_tokens"] = remove_matched_tokens;+return override;}};@@ -446,6 +447,9 @@ private:spp::sparse_hash_map<std::string, spp::sparse_hash_map<std::string, std::vector<uint32_t>>*> geopoint_index;+ // geo_array_field => (seq_id => values) used for exact filtering of geo array records+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;+// facet_field => (seq_id => values)spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;@@ -458,9 +462,6 @@ private:// infix field => valuespp::sparse_hash_map<std::string, array_mapped_infix_t> infix_index;- // geo_array_field => (seq_id => values) used for exact filtering of geo array records- spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;-// this is used for wildcard queriesid_list_t* seq_ids;@@ -711,7 +712,10 @@ public:const size_t max_extra_suffix, const size_t facet_query_num_typos,const bool filter_curated_hits, bool split_join_tokens) const;- Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);+ void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name);++ Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document,+ const std::vector<field>& del_fields, const bool is_update);static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,const size_t batch_start_index, const size_t batch_size,@@ -719,15 +723,17 @@ public:const std::unordered_map<std::string, field> & search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,- const std::vector<char>& symbols_to_index);+ const std::vector<char>& symbols_to_index,+ const bool do_validation);static size_t batch_memory_index(Index *index,- std::vector<index_record> & iter_batch,- const std::string & default_sorting_field,- const std::unordered_map<std::string, field> & search_schema,+ std::vector<index_record>& iter_batch,+ const std::string& default_sorting_field,+ const std::unordered_map<std::string, field>& search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,- const std::vector<char>& symbols_to_index);+ const std::vector<char>& symbols_to_index,+ const bool do_validation);void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);@@ -745,7 +751,7 @@ public:void do_filtering_with_lock(uint32_t*& filter_ids, uint32_t& filter_ids_length,const std::vector<filter>& filters) const;- void refresh_schemas(const std::vector<field>& new_fields);+ void refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields);// the following methods are not synchronized because their parent calls are synchronized or they are const/staticdiff --git a/src/collection.cpp b/src/collection.cppindex efb97b0b..481640af 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -361,12 +361,11 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t}index_record rec(0, seq_id, document, op, dirty_values);- Index::compute_token_offsets_facets(rec, search_schema, token_separators, symbols_to_index);std::vector<index_record> index_batch;index_batch.emplace_back(std::move(rec));Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema,- fallback_field_type, token_separators, symbols_to_index);+ fallback_field_type, token_separators, symbols_to_index, true);num_documents += 1;return Option<>(200);@@ -376,7 +375,7 @@ size_t Collection::batch_index_in_memory(std::vector<index_record>& index_recordstd::unique_lock lock(mutex);size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,search_schema, fallback_field_type,- token_separators, symbols_to_index);+ token_separators, symbols_to_index, true);num_documents += num_indexed;return num_indexed;}@@ -2152,7 +2151,7 @@ void Collection::remove_document(const nlohmann::json & document, const uint32_t{std::unique_lock lock(mutex);- index->remove(seq_id, document, false);+ index->remove(seq_id, document, {}, false);num_documents -= 1;}@@ -2461,6 +2460,300 @@ spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {return synonym_index->get_synonyms();}+Option<bool> Collection::persist_collection_meta() {+ std::string coll_meta_json;+ StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);++ if(status != StoreStatus::FOUND) {+ return Option<bool>(500, "Could not fetch collection meta from store.");+ }++ nlohmann::json collection_meta;++ try {+ collection_meta = nlohmann::json::parse(coll_meta_json);+ } catch(...) {+ return Option<bool>(500, "Unable to parse collection meta.");+ }++ nlohmann::json fields_json = nlohmann::json::array();+ Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json);++ if(!fields_json_op.ok()) {+ return Option<bool>(fields_json_op.code(), fields_json_op.error());+ }++ collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;+ collection_meta[Collection::COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;+ collection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE] = fallback_field_type;++ bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());+ if(!persisted) {+ return Option<bool>(500, "Could not persist collection meta to store.");+ }++ return Option<bool>(true);+}++Option<bool> Collection::alter(nlohmann::json& alter_payload) {+ std::unique_lock lock(mutex);++ // Validate that all stored documents are compatible with the proposed schema changes.+ std::unordered_map<std::string, field> schema_additions;+ std::vector<field> del_fields;+ std::string this_fallback_field_type;++ auto validate_op = validate_alter_payload(alter_payload, schema_additions, del_fields, this_fallback_field_type);+ if(!validate_op.ok()) {+ return validate_op;+ }++ if(!this_fallback_field_type.empty() && !fallback_field_type.empty()) {+ return Option<bool>(400, "The schema already contains a `.*` field.");+ }++ if(!this_fallback_field_type.empty() && fallback_field_type.empty()) {+ fallback_field_type = this_fallback_field_type;+ }++ // Update schema with additions (deletions can only be made later)+ std::vector<field> new_fields;++ for(auto& kv: schema_additions) {+ const auto& f = kv.second;++ if(f.is_dynamic()) {+ // regexp fields and fields with auto type are treated as dynamic fields+ dynamic_fields.push_back(f);+ continue;+ }++ if(f.name == ".*") {+ fields.push_back(f);+ continue;+ }++ search_schema.emplace(kv.first, f);+ fields.push_back(f);+ new_fields.push_back(f);+ }++ index->refresh_schemas(new_fields, {});++ // Now, we can index existing data onto the updated schema+ const std::string seq_id_prefix = get_seq_id_collection_prefix();+ rocksdb::Iterator* iter = store->scan(seq_id_prefix);+ std::unique_ptr<rocksdb::Iterator> iter_guard(iter);++ size_t num_found_docs = 0;+ std::vector<index_record> iter_batch;+ const size_t index_batch_size = 1000;++ auto begin = std::chrono::high_resolution_clock::now();++ while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {+ num_found_docs++;+ const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());++ nlohmann::json document;++ try {+ document = nlohmann::json::parse(iter->value().ToString());+ } catch(const std::exception& e) {+ return Option<bool>(false, "Bad JSON in document: " + document.dump(-1, ' ', false,+ nlohmann::detail::error_handler_t::ignore));+ }++ index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::DROP);+ iter_batch.emplace_back(std::move(record));++ // Peek and check for last record right here so that we handle batched indexing correctly+ // Without doing this, the "last batch" would have to be indexed outside the loop.+ iter->Next();+ bool last_record = !(iter->Valid() && iter->key().starts_with(seq_id_prefix));++ if(num_found_docs % index_batch_size == 0 || last_record) {+ Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, fallback_field_type,+ token_separators, symbols_to_index, false);++ if(!del_fields.empty()) {+ for(auto& rec: iter_batch) {+ index->remove(seq_id, rec.doc, del_fields, true);+ }+ }++ iter_batch.clear();+ }++ if(num_found_docs % ((1 << 14)) == 0) {+ // having a cheaper higher layer check to prevent checking clock too often+ auto time_elapsed = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::high_resolution_clock::now() - begin).count();++ if(time_elapsed > 30) {+ begin = std::chrono::high_resolution_clock::now();+ LOG(INFO) << "Altered " << num_found_docs << " so far.";+ }+ }+ }++ LOG(INFO) << "Finished altering " << num_found_docs << " document(s).";++ for(auto& del_field: del_fields) {+ search_schema.erase(del_field.name);+ auto new_end = std::remove_if(fields.begin(), fields.end(), [&del_field](const field& f) {+ return f.name == del_field.name;+ });++ fields.erase(new_end, fields.end());++ if(del_field.name == ".*") {+ fallback_field_type = "";+ }++ if(del_field.name == default_sorting_field) {+ default_sorting_field = "";+ }+ }++ index->refresh_schemas({}, del_fields);++ auto persist_op = persist_collection_meta();+ if(!persist_op.ok()) {+ return persist_op;+ }++ return Option<bool>(true);+}++Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,+ std::unordered_map<std::string, field>& schema_additions,+ std::vector<field>& del_fields,+ std::string& fallback_field_type) {+ if(!schema_changes.is_object()) {+ return Option<bool>(400, "Bad JSON.");+ }++ if(schema_changes.size() != 1) {+ return Option<bool>(400, "Only `fields` can be updated at the moment.");+ }++ const std::string err_msg = "The `fields` value should be an array of objects containing "+ "the field `name` and other properties.";++ if(!schema_changes["fields"].is_array() || schema_changes["fields"].empty()) {+ return Option<bool>(400, err_msg);+ }++ // basic validation of fields+ std::vector<field> diff_fields;+ std::unordered_map<std::string, field> updated_search_schema = search_schema;+ size_t num_auto_detect_fields = 0;++ for(const auto& kv: schema_changes["fields"].items()) {+ if(!kv.value().is_object()) {+ return Option<bool>(400, err_msg);+ }++ if(!kv.value().contains("name")) {+ return Option<bool>(400, err_msg);+ }++ const std::string& field_name = kv.value()["name"].get<std::string>();+ const auto& field_it = search_schema.find(field_name);+ auto found_field = (field_it != search_schema.end());++ if(kv.value().contains("drop")) {+ if(field_name == ".*") {+ del_fields.emplace_back(".*", field_types::AUTO, false);+ continue;+ }++ if(!found_field) {+ return Option<bool>(400, "Field `" + field_name + "` is not part of collection schema.");+ }++ if(!kv.value()["drop"].is_boolean() || !kv.value()["drop"].get<bool>()) {+ return Option<bool>(400, "Field `" + field_name + "` must have a drop value of `true`.");+ }++ del_fields.push_back(field_it->second);++ } else {+ // add or update existing field+ auto is_addition = (!found_field);+ if(is_addition) {+ // addition: must validate fields+ auto parse_op = field::json_field_to_field(kv.value(), diff_fields, fallback_field_type,+ num_auto_detect_fields);+ if (!parse_op.ok()) {+ return parse_op;+ }++ const auto& f = diff_fields.back();+ updated_search_schema.emplace(f.name, f);+ schema_additions.emplace(f.name, f);+ } else {+ // update, not supported for now+ return Option<bool>(400, "Field `" + field_name + "` is already part of schema: only "+ "field additions and deletions are supported for now.");+ }+ }+ }++ if(num_auto_detect_fields > 1) {+ return Option<bool>(400, "There can be only one field named `.*`.");+ }++ // data validations: here we ensure that already stored data is compatible with requested schema changes+ const std::string seq_id_prefix = get_seq_id_collection_prefix();++ rocksdb::Iterator* iter = store->scan(seq_id_prefix);+ std::unique_ptr<rocksdb::Iterator> iter_guard(iter);++ size_t num_found_docs = 0;+ auto begin = std::chrono::high_resolution_clock::now();++ while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {+ num_found_docs++;+ const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());+ nlohmann::json document;++ try {+ document = nlohmann::json::parse(iter->value().ToString());+ } catch(const std::exception& e) {+ return Option<bool>(false, "Bad JSON in document: " + document.dump(-1, ' ', false,+ nlohmann::detail::error_handler_t::ignore));+ }++ // validate existing data on disk for compatibility via updated_search_schema+ auto validate_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,+ updated_search_schema,+ index_operation_t::CREATE,+ fallback_field_type,+ DIRTY_VALUES::COERCE_OR_REJECT);+ if(!validate_op.ok()) {+ std::string schema_err = "Schema change does not match on-disk data, error: " + validate_op.error();+ return Option<bool>(validate_op.code(), schema_err);+ }++ if(num_found_docs % ((1 << 14)) == 0) {+ // having a cheaper higher layer check to prevent checking clock too often+ auto time_elapsed = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::high_resolution_clock::now() - begin).count();++ if(time_elapsed > 30) {+ begin = std::chrono::high_resolution_clock::now();+ LOG(INFO) << "Verified " << num_found_docs << " so far.";+ }+ }++ iter->Next();+ }++ return Option<bool>(true);+}+Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values) {std::unique_lock lock(mutex);@@ -2570,38 +2863,12 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const}if(!new_fields.empty()) {- // we should persist changes to fields in store- std::string coll_meta_json;- StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);-- if(status != StoreStatus::FOUND) {- return Option<bool>(500, "Could not fetch collection meta from store.");+ auto persist_op = persist_collection_meta();+ if(!persist_op.ok()) {+ return persist_op;}- nlohmann::json collection_meta;-- try {- collection_meta = nlohmann::json::parse(coll_meta_json);- nlohmann::json fields_json = nlohmann::json::array();;-- Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json);-- if(!fields_json_op.ok()) {- return Option<bool>(fields_json_op.code(), fields_json_op.error());- }-- collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;- bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());-- if(!persisted) {- return Option<bool>(500, "Could not persist collection meta to store.");- }-- index->refresh_schemas(new_fields);-- } catch(...) {- return Option<bool>(500, "Unable to parse collection meta.");- }+ index->refresh_schemas(new_fields, {});}return Option<bool>(true);@@ -2668,3 +2935,7 @@ std::vector<char> Collection::get_symbols_to_index() {std::vector<char> Collection::get_token_separators() {return token_separators;}++std::string Collection::get_fallback_field_type() {+ return fallback_field_type;+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex a12353c9..47a25e57 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1119,7 +1119,6 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection}// Fetch records from the store and re-create memory index- std::vector<std::string> documents;const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();rocksdb::Iterator* iter = cm.store->scan(seq_id_prefix);@@ -1241,3 +1240,21 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {preset_configs.erase(preset_name);return Option<bool>(true);}++Option<bool> CollectionManager::update_collection(const std::string& name, nlohmann::json& req_json) {+ if(!req_json["fields"].is_array() || req_json["fields"].empty()) {+ return Option<bool>(400, "The `fields` value should be an array of objects containing "+ "the field properties.");+ }++ auto collection = get_collection(name);+ if(collection == nullptr) {+ return Option<bool>(404, "Not found.");+ }++ // Supported operations:+ // - Adding a new field+ // - Dropping a field++ return collection->alter(req_json);+}diff --git a/src/core_api.cpp b/src/core_api.cppindex e9346fe9..d1341e4a 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -160,6 +160,35 @@ bool post_create_collection(const std::shared_ptr<http_req>& req, const std::shareturn false;}+bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ //LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ CollectionManager & collectionManager = CollectionManager::get_instance();+ auto collection = collectionManager.get_collection(req->params["collection"]);++ if(collection == nullptr) {+ res->set_404();+ return false;+ }++ auto alter_op = collection->alter(req_json);+ if(!alter_op.ok()) {+ res->set(alter_op.code(), alter_op.error());+ return false;+ }++ res->set_200(req->body);+ return true;+}+bool del_drop_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {std::string doc_id = req->params["id"];CollectionManager & collectionManager = CollectionManager::get_instance();diff --git a/src/field.cpp b/src/field.cppindex 60831ef1..5d4730c2 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -361,3 +361,152 @@ Option<bool> filter::parse_filter_query(const string& simple_filter_query,return Option<bool>(true);}++Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,+ string& fallback_field_type, size_t& num_auto_detect_fields) {++ if(field_json["name"] == "id") {+ // No field should exist with the name "id" as it is reserved for internal use+ // We cannot throw an error here anymore since that will break backward compatibility!+ LOG(WARNING) << "Collection schema cannot contain a field with name `id`. Ignoring field.";+ return Option<bool>(true);+ }++ if(!field_json.is_object() ||+ field_json.count(fields::name) == 0 || field_json.count(fields::type) == 0 ||+ !field_json.at(fields::name).is_string() || !field_json.at(fields::type).is_string()) {++ return Option<bool>(400, "Wrong format for `fields`. It should be an array of objects containing "+ "`name`, `type`, `optional` and `facet` properties.");+ }++ if(field_json.count(fields::facet) != 0 && !field_json.at(fields::facet).is_boolean()) {+ return Option<bool>(400, std::string("The `facet` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }++ if(field_json.count(fields::optional) != 0 && !field_json.at(fields::optional).is_boolean()) {+ return Option<bool>(400, std::string("The `optional` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }++ if(field_json.count(fields::index) != 0 && !field_json.at(fields::index).is_boolean()) {+ return Option<bool>(400, std::string("The `index` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }++ if(field_json.count(fields::sort) != 0 && !field_json.at(fields::sort).is_boolean()) {+ return Option<bool>(400, std::string("The `sort` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }++ if(field_json.count(fields::infix) != 0 && !field_json.at(fields::infix).is_boolean()) {+ return Option<bool>(400, std::string("The `infix` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }++ if(field_json.count(fields::locale) != 0){+ if(!field_json.at(fields::locale).is_string()) {+ return Option<bool>(400, std::string("The `locale` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a string."));+ }++ if(!field_json[fields::locale].get<std::string>().empty() &&+ field_json[fields::locale].get<std::string>().size() != 2) {+ return Option<bool>(400, std::string("The `locale` value of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` is not valid."));+ }+ }++ if(field_json["name"] == ".*") {+ if(field_json.count(fields::facet) == 0) {+ field_json[fields::facet] = false;+ }++ if(field_json.count(fields::optional) == 0) {+ field_json[fields::optional] = true;+ }++ if(field_json.count(fields::index) == 0) {+ field_json[fields::index] = true;+ }++ if(field_json.count(fields::locale) == 0) {+ field_json[fields::locale] = "";+ }++ if(field_json.count(fields::sort) == 0) {+ field_json[fields::sort] = false;+ }++ if(field_json.count(fields::infix) == 0) {+ field_json[fields::infix] = false;+ }++ if(field_json[fields::optional] == false) {+ return Option<bool>(400, "Field `.*` must be an optional field.");+ }++ if(field_json[fields::facet] == true) {+ return Option<bool>(400, "Field `.*` cannot be a facet field.");+ }++ if(field_json[fields::index] == false) {+ return Option<bool>(400, "Field `.*` must be an index field.");+ }++ field fallback_field(field_json["name"], field_json["type"], field_json["facet"],+ field_json["optional"], field_json[fields::index], field_json[fields::locale],+ field_json[fields::sort], field_json[fields::infix]);++ if(fallback_field.has_valid_type()) {+ fallback_field_type = fallback_field.type;+ num_auto_detect_fields++;+ } else {+ return Option<bool>(400, "The `type` of field `.*` is invalid.");+ }++ the_fields.emplace_back(fallback_field);+ return Option<bool>(true);+ }++ if(field_json.count(fields::facet) == 0) {+ field_json[fields::facet] = false;+ }++ if(field_json.count(fields::index) == 0) {+ field_json[fields::index] = true;+ }++ if(field_json.count(fields::locale) == 0) {+ field_json[fields::locale] = "";+ }++ if(field_json.count(fields::sort) == 0) {+ if(field_json["type"] == field_types::INT32 || field_json["type"] == field_types::INT64 ||+ field_json["type"] == field_types::FLOAT || field_json["type"] == field_types::BOOL ||+ field_json["type"] == field_types::GEOPOINT || field_json["type"] == field_types::GEOPOINT_ARRAY) {+ field_json[fields::sort] = true;+ } else {+ field_json[fields::sort] = false;+ }+ }++ if(field_json.count(fields::infix) == 0) {+ field_json[fields::infix] = false;+ }++ if(field_json.count(fields::optional) == 0) {+ // dynamic fields are always optional+ bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);+ field_json[fields::optional] = is_dynamic;+ }++ the_fields.emplace_back(+ field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],+ field_json[fields::optional], field_json[fields::index], field_json[fields::locale],+ field_json[fields::sort], field_json[fields::infix])+ );++ return Option<bool>(true);+}diff --git a/src/index.cpp b/src/index.cppindex f2699f3d..31902801 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -443,7 +443,8 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteconst std::unordered_map<std::string, field>& search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,- const std::vector<char>& symbols_to_index) {+ const std::vector<char>& symbols_to_index,+ const bool do_validation) {// runs in a partitioned thread@@ -460,16 +461,18 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& itecontinue;}- Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,- default_sorting_field,- search_schema,- index_rec.operation,- fallback_field_type,- index_rec.dirty_values);+ if(do_validation) {+ Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,+ default_sorting_field,+ search_schema,+ index_rec.operation,+ fallback_field_type,+ index_rec.dirty_values);- if(!validation_op.ok()) {- index_rec.index_failure(validation_op.code(), validation_op.error());- continue;+ if(!validation_op.ok()) {+ index_rec.index_failure(validation_op.code(), validation_op.error());+ continue;+ }}if(index_rec.is_update) {@@ -513,7 +516,8 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bconst std::unordered_map<std::string, field> & search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,- const std::vector<char>& symbols_to_index) {+ const std::vector<char>& symbols_to_index,+ const bool do_validation) {const size_t concurrency = 4;const size_t num_threads = std::min(concurrency, iter_batch.size());@@ -539,7 +543,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bindex->thread_pool->enqueue([&, batch_index, batch_len]() {validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,- fallback_field_type, token_separators, symbols_to_index);+ fallback_field_type, token_separators, symbols_to_index, do_validation);std::unique_lock<std::mutex> lock(m_process);num_processed++;@@ -565,7 +569,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b}if(index_rec.is_update) {- index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);+ index->remove(index_rec.seq_id, index_rec.del_doc, {}, index_rec.is_update);} else if(index_rec.indexed.ok()) {num_indexed++;}@@ -3738,134 +3742,145 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &tokereturn total_cost;}-Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update) {- std::unique_lock lock(mutex);-- for(auto it = document.begin(); it != document.end(); ++it) {- const std::string& field_name = it.key();- const auto& search_field_it = search_schema.find(field_name);- if(search_field_it == search_schema.end()) {- continue;- }+void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name) {+ const auto& search_field_it = search_schema.find(field_name);+ if(search_field_it == search_schema.end()) {+ return;+ }- const auto& search_field = search_field_it->second;+ const auto& search_field = search_field_it->second;- if(!search_field.index) {- continue;- }+ if(!search_field.index) {+ return;+ }- // Go through all the field names and find the keys+values so that they can be removed from in-memory index- if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {- std::vector<std::string> tokens;- tokenize_string_field(document, search_field, tokens, search_field.locale);+ // Go through all the field names and find the keys+values so that they can be removed from in-memory index+ if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {+ std::vector<std::string> tokens;+ tokenize_string_field(document, search_field, tokens, search_field.locale);- for(size_t i = 0; i < tokens.size(); i++) {- const auto& token = tokens[i];- const unsigned char *key = (const unsigned char *) token.c_str();- int key_len = (int) (token.length() + 1);+ for(size_t i = 0; i < tokens.size(); i++) {+ const auto& token = tokens[i];+ const unsigned char *key = (const unsigned char *) token.c_str();+ int key_len = (int) (token.length() + 1);- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);- if(leaf != nullptr) {- posting_t::erase(leaf->values, seq_id);- if (posting_t::num_ids(leaf->values) == 0) {- void* values = art_delete(search_index.at(field_name), key, key_len);- posting_t::destroy_list(values);- }+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);+ if(leaf != nullptr) {+ posting_t::erase(leaf->values, seq_id);+ if (posting_t::num_ids(leaf->values) == 0) {+ void* values = art_delete(search_index.at(field_name), key, key_len);+ posting_t::destroy_list(values);}+ }- if(search_field.infix) {- auto strhash = StringUtils::hash_wy(key, token.size());- const auto& infix_sets = infix_index.at(search_field.name);- infix_sets[strhash % 4]->erase(token);+ if(search_field.infix) {+ auto strhash = StringUtils::hash_wy(key, token.size());+ const auto& infix_sets = infix_index.at(search_field.name);+ infix_sets[strhash % 4]->erase(token);+ }+ }+ } else if(search_field.is_int32()) {+ const std::vector<int32_t>& values = search_field.is_single_integer() ?+ std::vector<int32_t>{document[field_name].get<int32_t>()} :+ document[field_name].get<std::vector<int32_t>>();+ for(int32_t value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ num_tree->remove(value, seq_id);+ }+ } else if(search_field.is_int64()) {+ const std::vector<int64_t>& values = search_field.is_single_integer() ?+ std::vector<int64_t>{document[field_name].get<int64_t>()} :+ document[field_name].get<std::vector<int64_t>>();+ for(int64_t value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ num_tree->remove(value, seq_id);+ }+ } else if(search_field.is_float()) {+ const std::vector<float>& values = search_field.is_single_float() ?+ std::vector<float>{document[field_name].get<float>()} :+ document[field_name].get<std::vector<float>>();+ for(float value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ int64_t fintval = float_to_in64_t(value);+ num_tree->remove(fintval, seq_id);+ }+ } else if(search_field.is_bool()) {++ const std::vector<bool>& values = search_field.is_single_bool() ?+ std::vector<bool>{document[field_name].get<bool>()} :+ document[field_name].get<std::vector<bool>>();+ for(bool value: values) {+ num_tree_t* num_tree = numerical_index.at(field_name);+ int64_t bool_int64 = value ? 1 : 0;+ num_tree->remove(bool_int64, seq_id);+ }+ } else if(search_field.is_geopoint()) {+ auto geo_index = geopoint_index[field_name];+ S2RegionTermIndexer::Options options;+ options.set_index_contains_points_only(true);+ S2RegionTermIndexer indexer(options);++ const std::vector<std::vector<double>>& latlongs = search_field.is_single_geopoint() ?+ std::vector<std::vector<double>>{document[field_name].get<std::vector<double>>()} :+ document[field_name].get<std::vector<std::vector<double>>>();++ for(const std::vector<double>& latlong: latlongs) {+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();+ for(const auto& term: indexer.GetIndexTerms(point, "")) {+ auto term_it = geo_index->find(term);+ if(term_it == geo_index->end()) {+ continue;}- }- } else if(search_field.is_int32()) {- const std::vector<int32_t>& values = search_field.is_single_integer() ?- std::vector<int32_t>{document[field_name].get<int32_t>()} :- document[field_name].get<std::vector<int32_t>>();- for(int32_t value: values) {- num_tree_t* num_tree = numerical_index.at(field_name);- num_tree->remove(value, seq_id);- }- } else if(search_field.is_int64()) {- const std::vector<int64_t>& values = search_field.is_single_integer() ?- std::vector<int64_t>{document[field_name].get<int64_t>()} :- document[field_name].get<std::vector<int64_t>>();- for(int64_t value: values) {- num_tree_t* num_tree = numerical_index.at(field_name);- num_tree->remove(value, seq_id);- }- } else if(search_field.is_float()) {- const std::vector<float>& values = search_field.is_single_float() ?- std::vector<float>{document[field_name].get<float>()} :- document[field_name].get<std::vector<float>>();- for(float value: values) {- num_tree_t* num_tree = numerical_index.at(field_name);- int64_t fintval = float_to_in64_t(value);- num_tree->remove(fintval, seq_id);- }- } else if(search_field.is_bool()) {-- const std::vector<bool>& values = search_field.is_single_bool() ?- std::vector<bool>{document[field_name].get<bool>()} :- document[field_name].get<std::vector<bool>>();- for(bool value: values) {- num_tree_t* num_tree = numerical_index.at(field_name);- int64_t bool_int64 = value ? 1 : 0;- num_tree->remove(bool_int64, seq_id);- }- } else if(search_field.is_geopoint()) {- auto geo_index = geopoint_index[field_name];- S2RegionTermIndexer::Options options;- options.set_index_contains_points_only(true);- S2RegionTermIndexer indexer(options);-- const std::vector<std::vector<double>>& latlongs = search_field.is_single_geopoint() ?- std::vector<std::vector<double>>{document[field_name].get<std::vector<double>>()} :- document[field_name].get<std::vector<std::vector<double>>>();-- for(const std::vector<double>& latlong: latlongs) {- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();- for(const auto& term: indexer.GetIndexTerms(point, "")) {- auto term_it = geo_index->find(term);- if(term_it == geo_index->end()) {- continue;- }- std::vector<uint32_t>& ids = term_it->second;- ids.erase(std::remove(ids.begin(), ids.end(), seq_id), ids.end());- if(ids.empty()) {- geo_index->erase(term);- }+ std::vector<uint32_t>& ids = term_it->second;+ ids.erase(std::remove(ids.begin(), ids.end(), seq_id), ids.end());+ if(ids.empty()) {+ geo_index->erase(term);}}+ }- if(!search_field.is_single_geopoint()) {- spp::sparse_hash_map<uint32_t, int64_t*>*& field_geo_array_map = geo_array_index.at(field_name);- auto geo_array_it = field_geo_array_map->find(seq_id);- if(geo_array_it != field_geo_array_map->end()) {- delete [] geo_array_it->second;- field_geo_array_map->erase(seq_id);- }+ if(!search_field.is_single_geopoint()) {+ spp::sparse_hash_map<uint32_t, int64_t*>*& field_geo_array_map = geo_array_index.at(field_name);+ auto geo_array_it = field_geo_array_map->find(seq_id);+ if(geo_array_it != field_geo_array_map->end()) {+ delete [] geo_array_it->second;+ field_geo_array_map->erase(seq_id);}}+ }- // remove facets- const auto& field_facets_it = facet_index_v3.find(field_name);+ // remove facets+ const auto& field_facets_it = facet_index_v3.find(field_name);- if(field_facets_it != facet_index_v3.end()) {- const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);- }+ if(field_facets_it != facet_index_v3.end()) {+ const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);}+ }- // remove sort field- if(sort_index.count(field_name) != 0) {- sort_index[field_name]->erase(seq_id);- }+ // remove sort field+ if(sort_index.count(field_name) != 0) {+ sort_index[field_name]->erase(seq_id);+ }++ if(str_sort_index.count(field_name) != 0) {+ str_sort_index[field_name]->remove(seq_id);+ }+}- if(str_sort_index.count(field_name) != 0) {- str_sort_index[field_name]->remove(seq_id);+Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document,+ const std::vector<field>& del_fields, const bool is_update) {+ std::unique_lock lock(mutex);++ if(!del_fields.empty()) {+ for(auto& the_field: del_fields) {+ remove_field(seq_id, document, the_field.name);+ }+ } else {+ for(auto it = document.begin(); it != document.end(); ++it) {+ const std::string& field_name = it.key();+ remove_field(seq_id, document, field_name);}}@@ -3909,7 +3924,7 @@ const spp::sparse_hash_map<std::string, array_mapped_infix_t>& Index::_get_infixreturn infix_index;};-void Index::refresh_schemas(const std::vector<field>& new_fields) {+void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields) {std::unique_lock lock(mutex);for(const auto & new_field: new_fields) {@@ -3967,6 +3982,65 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {infix_index.emplace(new_field.name, infix_sets);}}++ for(const auto & del_field: del_fields) {+ search_schema.erase(del_field.name);++ if(del_field.is_string() || field_types::is_string_or_array(del_field.type)) {+ art_tree_destroy(search_index[del_field.name]);+ delete search_index[del_field.name];+ search_index.erase(del_field.name);+ } else if(del_field.is_geopoint()) {+ delete geopoint_index[del_field.name];+ geopoint_index.erase(del_field.name);++ if(!del_field.is_single_geopoint()) {+ spp::sparse_hash_map<uint32_t, int64_t*>* geo_array_map = geo_array_index[del_field.name];+ for(auto& kv: *geo_array_map) {+ delete kv.second;+ }+ delete geo_array_map;+ geo_array_index.erase(del_field.name);+ }+ } else {+ delete numerical_index[del_field.name];+ numerical_index.erase(del_field.name);+ }++ if(del_field.is_sortable()) {+ if(del_field.is_num_sortable()) {+ delete sort_index[del_field.name];+ sort_index.erase(del_field.name);+ } else if(del_field.is_str_sortable()) {+ delete str_sort_index[del_field.name];+ str_sort_index.erase(del_field.name);+ }+ }++ if(del_field.is_facet()) {+ auto& arr = facet_index_v3[del_field.name];+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ delete arr[i];+ }++ facet_index_v3.erase(del_field.name);++ if(!del_field.is_string()) {+ art_tree_destroy(search_index[del_field.faceted_name()]);+ delete search_index[del_field.faceted_name()];+ search_index.erase(del_field.faceted_name());+ }+ }++ if(del_field.infix) {+ auto& infix_set = infix_index[del_field.name];+ for(size_t i = 0; i < infix_set.size(); i++) {+ delete infix_set[i];+ }++ infix_index.erase(del_field.name);+ }+ }}Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const std::string& fallback_field_type,diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex d22ae714..16e96100 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -44,6 +44,7 @@ void master_server_routes() {// collection managementserver->post("/collections", post_create_collection);+ server->patch("/collections/:collection", patch_update_collection);server->get("/collections", get_collections);server->del("/collections/:collection", del_drop_collection);server->get("/collections/:collection", get_collection_summary);diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 759d3765..b5610181 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -1368,7 +1368,6 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {while(curr_block != end_block && curr_block->ids.last() < id) {curr_block = curr_block->next;- // FIXME: remove duplicationdelete [] ids;delete [] offset_index;delete [] offsets;diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex f822c8d1..aafd889d 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -431,7 +431,8 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {ASSERT_EQ(1, overrides.size());auto override_json = overrides["include-rule"].to_json();ASSERT_FALSE(override_json.contains("filter_by"));- ASSERT_FALSE(override_json.contains("remove_matched_tokens"));+ ASSERT_TRUE(override_json.contains("remove_matched_tokens"));+ ASSERT_FALSE(override_json["remove_matched_tokens"].get<bool>());auto results = coll_mul_fields->search("not-found", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,{false}, Index::DROP_TOKENS_THRESHOLD,diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppnew file mode 100644index 00000000..c302449b--- /dev/null+++ b/test/collection_schema_change_test.cpp@@ -0,0 +1,366 @@+#include <gtest/gtest.h>+#include <string>+#include <vector>+#include <fstream>+#include <algorithm>+#include <collection_manager.h>+#include "collection.h"++class CollectionSchemaChangeTest : public ::testing::Test {+protected:+ Store *store;+ CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;++ std::vector<std::string> query_fields;+ std::vector<sort_by> sort_fields;++ void setupCollection() {+ std::string state_dir_path = "/tmp/typesense_test/collection_schema_change";+ LOG(INFO) << "Truncating and creating: " << state_dir_path;+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);+ }++ virtual void SetUp() {+ setupCollection();+ }++ virtual void TearDown() {+ collectionManager.dispose();+ delete store;+ }+};++TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "The quick brown fox was too fast.";+ doc["tags"] = {"experimental", "news"};+ doc["category"] = "animals";+ doc["quantity"] = 100;+ doc["points"] = 100;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("fox",+ {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ auto schema_changes = R"({+ "fields": [+ {"name": "tags", "type": "string[]", "infix": true},+ {"name": "category", "type": "string", "sort": true}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ results = coll1->search("eriment",+ {"tags"}, "", {}, {}, {0}, 3, 1, FREQUENCY,+ {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {always}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());++ sort_fields = { sort_by("category", "DESC") };+ results = coll1->search("*",+ {}, "", {}, sort_fields, {0}, 3, 1, FREQUENCY,+ {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {always}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());++ schema_changes = R"({+ "fields": [+ {"name": "quantity", "type": "int32", "facet": true}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ results = coll1->search("*",+ {}, "quantity: 100", {"quantity"}, {}, {0}, 3, 1, FREQUENCY,+ {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {always}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_STREQ("100", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);++ // add a dynamic field+ schema_changes = R"({+ "fields": [+ {"name": ".*_bool", "type": "bool"}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ doc["id"] = "1";+ doc["title"] = "The one";+ doc["tags"] = {"sports", "news"};+ doc["category"] = "things";+ doc["quantity"] = 200;+ doc["points"] = 100;+ doc["on_sale_bool"] = true;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ results = coll1->search("*",+ {}, "on_sale_bool: true", {}, {}, {0}, 3, 1, FREQUENCY,+ {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {always}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ // add auto field+ schema_changes = R"({+ "fields": [+ {"name": ".*", "type": "auto"}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ doc["id"] = "2";+ doc["title"] = "The two";+ doc["tags"] = {"sports", "news"};+ doc["category"] = "things";+ doc["quantity"] = 200;+ doc["points"] = 100;+ doc["on_sale_bool"] = false;+ doc["foobar"] = 123;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ results = coll1->search("*",+ {}, "foobar: 123", {}, {}, {0}, 3, 1, FREQUENCY,+ {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ 4, {always}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_STREQ("2", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ // try to add auto field again+ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("The schema already contains a `.*` field.", alter_op.error());++ // try to add a regular field with 2 auto fields+ schema_changes = R"({+ "fields": [+ {"name": "bar", "type": "auto"},+ {"name": ".*", "type": "auto"},+ {"name": ".*", "type": "auto"}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("There can be only one field named `.*`.", alter_op.error());++ auto search_schema = coll1->get_schema();+ ASSERT_EQ(7, search_schema.size());++ auto coll_fields = coll1->get_fields();+ ASSERT_EQ(8, coll_fields.size());++ // try restoring collection from disk: all fields should be preserved+ collectionManager.dispose();+ delete store;++ store = new Store("/tmp/typesense_test/collection_schema_change");+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);+ coll1 = collectionManager.get_collection("coll1").get();++ search_schema = coll1->get_schema();+ ASSERT_EQ(7, search_schema.size());++ coll_fields = coll1->get_fields();+ ASSERT_EQ(8, coll_fields.size());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSchemaChangeTest, DropFieldsFromCollection) {+ std::vector<field> fields = {field(".*", field_types::AUTO, false),+ field("title", field_types::STRING, false, false, true, "", 1, 1),+ field("location", field_types::GEOPOINT, false),+ field("locations", field_types::GEOPOINT_ARRAY, false),+ field("tags", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, true),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "auto").get();++ std::vector<std::vector<double>> lat_lngs;+ lat_lngs.push_back({48.85821022164442, 2.294239067890161});++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "The quick brown fox was too fast.";+ doc["location"] = {48.85821022164442, 2.294239067890161};+ doc["locations"] = lat_lngs;+ doc["tags"] = {"experimental", "news"};+ doc["points"] = 100;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("*",+ {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ auto schema_changes = R"({+ "fields": [+ {"name": ".*", "drop": true},+ {"name": "title", "drop": true},+ {"name": "location", "drop": true},+ {"name": "locations", "drop": true},+ {"name": "tags", "drop": true},+ {"name": "points", "drop": true}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_TRUE(alter_op.ok());++ results = coll1->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ auto res_op = coll1->search("quick", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `title` in the schema.", res_op.error());++ auto search_schema = coll1->get_schema();+ ASSERT_EQ(0, search_schema.size());++ auto coll_fields = coll1->get_fields();+ ASSERT_EQ(0, coll_fields.size());++ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().size());+ ASSERT_EQ(0, coll1->_get_index()->_get_infix_index().size());+ ASSERT_EQ(1, coll1->_get_index()->num_seq_ids());+ ASSERT_EQ("", coll1->get_fallback_field_type());+ ASSERT_EQ("", coll1->get_default_sorting_field());++ // try restoring collection from disk: all fields should be deleted+ collectionManager.dispose();+ delete store;++ store = new Store("/tmp/typesense_test/collection_schema_change");+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);+ coll1 = collectionManager.get_collection("coll1").get();++ search_schema = coll1->get_schema();+ ASSERT_EQ(0, search_schema.size());++ coll_fields = coll1->get_fields();+ ASSERT_EQ(0, coll_fields.size());++ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().size());+ ASSERT_EQ(0, coll1->_get_index()->_get_infix_index().size());+ ASSERT_EQ(1, coll1->_get_index()->num_seq_ids());+ ASSERT_EQ("", coll1->get_default_sorting_field());+ ASSERT_EQ("", coll1->get_fallback_field_type());++ results = coll1->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();+ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ res_op = coll1->search("quick", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `title` in the schema.", res_op.error());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSchemaChangeTest, AlterValidations) {+ std::vector<field> fields = {field(".*", field_types::AUTO, false),+ field("title", field_types::STRING, false, false, true, "", 1, 1),+ field("location", field_types::GEOPOINT, false),+ field("locations", field_types::GEOPOINT_ARRAY, false),+ field("tags", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, true),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "auto").get();++ std::vector<std::vector<double>> lat_lngs;+ lat_lngs.push_back({48.85821022164442, 2.294239067890161});++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "The quick brown fox was too fast.";+ doc["location"] = {48.85821022164442, 2.294239067890161};+ doc["locations"] = lat_lngs;+ doc["tags"] = {"experimental", "news"};+ doc["points"] = 100;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // 1. Modify existing field, which is not supported++ auto schema_changes = R"({+ "fields": [+ {"name": "title", "type": "string[]"}+ ]+ })"_json;++ auto alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Field `title` is already part of schema: only field additions and deletions are supported for now.",+ alter_op.error());++ // 2. Bad field format+ schema_changes = R"({+ "fields": [+ {"name": "age", "typezzz": "int32"}+ ]+ })"_json;++ alter_op = coll1->alter(schema_changes);+ ASSERT_FALSE(alter_op.ok());+ ASSERT_EQ("Wrong format for `fields`. It should be an array of objects containing `name`, `type`, "+ "`optional` and `facet` properties.",alter_op.error());++ collectionManager.drop_collection("coll1");+}+