Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Schema change basics.

Kishore Nallana5b06a0e0db6
diff --git a/include/collection.h b/include/collection.h
index 93fd5911..fca21351 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -81,11 +81,11 @@ private:
std::map<std::string, override_t> overrides;
- const std::string default_sorting_field;
+ std::string default_sorting_field;
const float max_memory_ratio;
- const std::string fallback_field_type;
+ std::string fallback_field_type;
std::vector<field> dynamic_fields;
@@ -153,6 +153,13 @@ private:
Option<bool> validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,
std::vector<sort_by>& sort_fields_std) const;
+ Option<bool> persist_collection_meta();
+
+ Option<bool> validate_alter_payload(nlohmann::json& schema_changes,
+ std::unordered_map<std::string, field>& schema_additions,
+ std::vector<field>& del_fields,
+ std::string& fallback_field_type);
+
public:
enum {MAX_ARRAY_MATCHES = 5};
@@ -315,9 +322,6 @@ public:
Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);
- bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,
- std::string &value);
-
size_t get_num_documents() const;
DIRTY_VALUES parse_dirty_values_option(std::string& dirty_values) const;
@@ -326,6 +330,8 @@ public:
std::vector<char> get_token_separators();
+ std::string get_fallback_field_type();
+
// Override operations
Option<uint32_t> add_override(const override_t & override);
@@ -365,5 +371,7 @@ public:
const std::string& highlight_full_fields,
const std::vector<infix_t>& infixes,
std::vector<highlight_field_t>& highlight_items) const;
+
+ Option<bool> alter(nlohmann::json& alter_payload);
};
diff --git a/include/collection_manager.h b/include/collection_manager.h
index c0e1dbd0..c77a067c 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -152,6 +152,8 @@ public:
const std::vector<std::string>& symbols_to_index = {},
const std::vector<std::string>& token_separators = {});
+ Option<bool> update_collection(const std::string& name, nlohmann::json& req_json);
+
locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;
locked_resource_view_t<Collection> get_collection_with_id(uint32_t collection_id) const;
diff --git a/include/core_api.h b/include/core_api.h
index 62a9a0f4..a321d986 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -14,6 +14,8 @@ bool get_collections(const std::shared_ptr<http_req>& req, const std::shared_ptr
bool post_create_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
bool del_drop_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
bool get_collection_summary(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
diff --git a/include/field.h b/include/field.h
index ae37c0c0..d948c644 100644
--- a/include/field.h
+++ b/include/field.h
@@ -290,7 +290,7 @@ struct field {
}
}
- if(!default_sorting_field.empty() && !found_default_sorting_field) {
+ if(!default_sorting_field.empty() && !found_default_sorting_field && !fields.empty()) {
return Option<bool>(400, "Default sorting field is defined as `" + default_sorting_field +
"` but is not found in the schema.");
}
@@ -325,155 +325,20 @@ struct field {
return Option<bool>(true);
}
+ static Option<bool> json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,
+ string& fallback_field_type, size_t& num_auto_detect_fields);
+
static Option<bool> json_fields_to_fields(nlohmann::json& fields_json,
std::string& fallback_field_type,
- std::vector<field>& fields) {
+ std::vector<field>& the_fields) {
size_t num_auto_detect_fields = 0;
for(nlohmann::json & field_json: fields_json) {
- if(field_json["name"] == "id") {
- // No field should exist with the name "id" as it is reserved for internal use
- // We cannot throw an error here anymore since that will break backward compatibility!
- LOG(WARNING) << "Collection schema cannot contain a field with name `id`. Ignoring field.";
- continue;
- }
-
- if(!field_json.is_object() ||
- field_json.count(fields::name) == 0 || field_json.count(fields::type) == 0 ||
- !field_json.at(fields::name).is_string() || !field_json.at(fields::type).is_string()) {
-
- return Option<bool>(400, "Wrong format for `fields`. It should be an array of objects containing "
- "`name`, `type`, `optional` and `facet` properties.");
- }
-
- if(field_json.count(fields::facet) != 0 && !field_json.at(fields::facet).is_boolean()) {
- return Option<bool>(400, std::string("The `facet` property of the field `") +
- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
- }
-
- if(field_json.count(fields::optional) != 0 && !field_json.at(fields::optional).is_boolean()) {
- return Option<bool>(400, std::string("The `optional` property of the field `") +
- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
- }
-
- if(field_json.count(fields::index) != 0 && !field_json.at(fields::index).is_boolean()) {
- return Option<bool>(400, std::string("The `index` property of the field `") +
- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
- }
-
- if(field_json.count(fields::sort) != 0 && !field_json.at(fields::sort).is_boolean()) {
- return Option<bool>(400, std::string("The `sort` property of the field `") +
- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
- }
-
- if(field_json.count(fields::infix) != 0 && !field_json.at(fields::infix).is_boolean()) {
- return Option<bool>(400, std::string("The `infix` property of the field `") +
- field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
- }
-
- if(field_json.count(fields::locale) != 0){
- if(!field_json.at(fields::locale).is_string()) {
- return Option<bool>(400, std::string("The `locale` property of the field `") +
- field_json[fields::name].get<std::string>() + std::string("` should be a string."));
- }
-
- if(!field_json[fields::locale].get<std::string>().empty() &&
- field_json[fields::locale].get<std::string>().size() != 2) {
- return Option<bool>(400, std::string("The `locale` value of the field `") +
- field_json[fields::name].get<std::string>() + std::string("` is not valid."));
- }
- }
-
- if(field_json["name"] == ".*") {
- if(field_json.count(fields::facet) == 0) {
- field_json[fields::facet] = false;
- }
-
- if(field_json.count(fields::optional) == 0) {
- field_json[fields::optional] = true;
- }
-
- if(field_json.count(fields::index) == 0) {
- field_json[fields::index] = true;
- }
-
- if(field_json.count(fields::locale) == 0) {
- field_json[fields::locale] = "";
- }
-
- if(field_json.count(fields::sort) == 0) {
- field_json[fields::sort] = false;
- }
-
- if(field_json.count(fields::infix) == 0) {
- field_json[fields::infix] = false;
- }
-
- if(field_json[fields::optional] == false) {
- return Option<bool>(400, "Field `.*` must be an optional field.");
- }
-
- if(field_json[fields::facet] == true) {
- return Option<bool>(400, "Field `.*` cannot be a facet field.");
- }
-
- if(field_json[fields::index] == false) {
- return Option<bool>(400, "Field `.*` must be an index field.");
- }
-
- field fallback_field(field_json["name"], field_json["type"], field_json["facet"],
- field_json["optional"], field_json[fields::index], field_json[fields::locale],
- field_json[fields::sort], field_json[fields::infix]);
-
- if(fallback_field.has_valid_type()) {
- fallback_field_type = fallback_field.type;
- num_auto_detect_fields++;
- } else {
- return Option<bool>(400, "The `type` of field `.*` is invalid.");
- }
-
- fields.emplace_back(fallback_field);
- continue;
+ auto op = json_field_to_field(field_json, the_fields, fallback_field_type, num_auto_detect_fields);
+ if(!op.ok()) {
+ return op;
}
-
- if(field_json.count(fields::facet) == 0) {
- field_json[fields::facet] = false;
- }
-
- if(field_json.count(fields::index) == 0) {
- field_json[fields::index] = true;
- }
-
- if(field_json.count(fields::locale) == 0) {
- field_json[fields::locale] = "";
- }
-
- if(field_json.count(fields::sort) == 0) {
- if(field_json["type"] == field_types::INT32 || field_json["type"] == field_types::INT64 ||
- field_json["type"] == field_types::FLOAT || field_json["type"] == field_types::BOOL ||
- field_json["type"] == field_types::GEOPOINT || field_json["type"] == field_types::GEOPOINT_ARRAY) {
- field_json[fields::sort] = true;
- } else {
- field_json[fields::sort] = false;
- }
- }
-
- if(field_json.count(fields::infix) == 0) {
- field_json[fields::infix] = false;
- }
-
- if(field_json.count(fields::optional) == 0) {
- // dynamic fields are always optional
- bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);
- field_json[fields::optional] = is_dynamic;
- }
-
- fields.emplace_back(
- field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
- field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
- field_json[fields::sort], field_json[fields::infix])
- );
}
if(num_auto_detect_fields > 1) {
diff --git a/include/index.h b/include/index.h
index 863834dd..9f29a6cd 100644
--- a/include/index.h
+++ b/include/index.h
@@ -268,9 +268,10 @@ struct override_t {
if(!filter_by.empty()) {
override["filter_by"] = filter_by;
- override["remove_matched_tokens"] = remove_matched_tokens;
}
+ override["remove_matched_tokens"] = remove_matched_tokens;
+
return override;
}
};
@@ -446,6 +447,9 @@ private:
spp::sparse_hash_map<std::string, spp::sparse_hash_map<std::string, std::vector<uint32_t>>*> geopoint_index;
+ // geo_array_field => (seq_id => values) used for exact filtering of geo array records
+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;
+
// facet_field => (seq_id => values)
spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;
@@ -458,9 +462,6 @@ private:
// infix field => value
spp::sparse_hash_map<std::string, array_mapped_infix_t> infix_index;
- // geo_array_field => (seq_id => values) used for exact filtering of geo array records
- spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;
-
// this is used for wildcard queries
id_list_t* seq_ids;
@@ -711,7 +712,10 @@ public:
const size_t max_extra_suffix, const size_t facet_query_num_typos,
const bool filter_curated_hits, bool split_join_tokens) const;
- Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);
+ void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name);
+
+ Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document,
+ const std::vector<field>& del_fields, const bool is_update);
static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,
const size_t batch_start_index, const size_t batch_size,
@@ -719,15 +723,17 @@ public:
const std::unordered_map<std::string, field> & search_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
- const std::vector<char>& symbols_to_index);
+ const std::vector<char>& symbols_to_index,
+ const bool do_validation);
static size_t batch_memory_index(Index *index,
- std::vector<index_record> & iter_batch,
- const std::string & default_sorting_field,
- const std::unordered_map<std::string, field> & search_schema,
+ std::vector<index_record>& iter_batch,
+ const std::string& default_sorting_field,
+ const std::unordered_map<std::string, field>& search_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
- const std::vector<char>& symbols_to_index);
+ const std::vector<char>& symbols_to_index,
+ const bool do_validation);
void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);
@@ -745,7 +751,7 @@ public:
void do_filtering_with_lock(uint32_t*& filter_ids, uint32_t& filter_ids_length,
const std::vector<filter>& filters) const;
- void refresh_schemas(const std::vector<field>& new_fields);
+ void refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields);
// the following methods are not synchronized because their parent calls are synchronized or they are const/static
diff --git a/src/collection.cpp b/src/collection.cpp
index efb97b0b..481640af 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -361,12 +361,11 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_t
}
index_record rec(0, seq_id, document, op, dirty_values);
- Index::compute_token_offsets_facets(rec, search_schema, token_separators, symbols_to_index);
std::vector<index_record> index_batch;
index_batch.emplace_back(std::move(rec));
Index::batch_memory_index(index, index_batch, default_sorting_field, search_schema,
- fallback_field_type, token_separators, symbols_to_index);
+ fallback_field_type, token_separators, symbols_to_index, true);
num_documents += 1;
return Option<>(200);
@@ -376,7 +375,7 @@ size_t Collection::batch_index_in_memory(std::vector<index_record>& index_record
std::unique_lock lock(mutex);
size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,
search_schema, fallback_field_type,
- token_separators, symbols_to_index);
+ token_separators, symbols_to_index, true);
num_documents += num_indexed;
return num_indexed;
}
@@ -2152,7 +2151,7 @@ void Collection::remove_document(const nlohmann::json & document, const uint32_t
{
std::unique_lock lock(mutex);
- index->remove(seq_id, document, false);
+ index->remove(seq_id, document, {}, false);
num_documents -= 1;
}
@@ -2461,6 +2460,300 @@ spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {
return synonym_index->get_synonyms();
}
+Option<bool> Collection::persist_collection_meta() {
+ std::string coll_meta_json;
+ StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);
+
+ if(status != StoreStatus::FOUND) {
+ return Option<bool>(500, "Could not fetch collection meta from store.");
+ }
+
+ nlohmann::json collection_meta;
+
+ try {
+ collection_meta = nlohmann::json::parse(coll_meta_json);
+ } catch(...) {
+ return Option<bool>(500, "Unable to parse collection meta.");
+ }
+
+ nlohmann::json fields_json = nlohmann::json::array();
+ Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json);
+
+ if(!fields_json_op.ok()) {
+ return Option<bool>(fields_json_op.code(), fields_json_op.error());
+ }
+
+ collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
+ collection_meta[Collection::COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;
+ collection_meta[Collection::COLLECTION_FALLBACK_FIELD_TYPE] = fallback_field_type;
+
+ bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());
+ if(!persisted) {
+ return Option<bool>(500, "Could not persist collection meta to store.");
+ }
+
+ return Option<bool>(true);
+}
+
+Option<bool> Collection::alter(nlohmann::json& alter_payload) {
+ std::unique_lock lock(mutex);
+
+ // Validate that all stored documents are compatible with the proposed schema changes.
+ std::unordered_map<std::string, field> schema_additions;
+ std::vector<field> del_fields;
+ std::string this_fallback_field_type;
+
+ auto validate_op = validate_alter_payload(alter_payload, schema_additions, del_fields, this_fallback_field_type);
+ if(!validate_op.ok()) {
+ return validate_op;
+ }
+
+ if(!this_fallback_field_type.empty() && !fallback_field_type.empty()) {
+ return Option<bool>(400, "The schema already contains a `.*` field.");
+ }
+
+ if(!this_fallback_field_type.empty() && fallback_field_type.empty()) {
+ fallback_field_type = this_fallback_field_type;
+ }
+
+ // Update schema with additions (deletions can only be made later)
+ std::vector<field> new_fields;
+
+ for(auto& kv: schema_additions) {
+ const auto& f = kv.second;
+
+ if(f.is_dynamic()) {
+ // regexp fields and fields with auto type are treated as dynamic fields
+ dynamic_fields.push_back(f);
+ continue;
+ }
+
+ if(f.name == ".*") {
+ fields.push_back(f);
+ continue;
+ }
+
+ search_schema.emplace(kv.first, f);
+ fields.push_back(f);
+ new_fields.push_back(f);
+ }
+
+ index->refresh_schemas(new_fields, {});
+
+ // Now, we can index existing data onto the updated schema
+ const std::string seq_id_prefix = get_seq_id_collection_prefix();
+ rocksdb::Iterator* iter = store->scan(seq_id_prefix);
+ std::unique_ptr<rocksdb::Iterator> iter_guard(iter);
+
+ size_t num_found_docs = 0;
+ std::vector<index_record> iter_batch;
+ const size_t index_batch_size = 1000;
+
+ auto begin = std::chrono::high_resolution_clock::now();
+
+ while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
+ num_found_docs++;
+ const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());
+
+ nlohmann::json document;
+
+ try {
+ document = nlohmann::json::parse(iter->value().ToString());
+ } catch(const std::exception& e) {
+ return Option<bool>(false, "Bad JSON in document: " + document.dump(-1, ' ', false,
+ nlohmann::detail::error_handler_t::ignore));
+ }
+
+ index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::DROP);
+ iter_batch.emplace_back(std::move(record));
+
+ // Peek and check for last record right here so that we handle batched indexing correctly
+ // Without doing this, the "last batch" would have to be indexed outside the loop.
+ iter->Next();
+ bool last_record = !(iter->Valid() && iter->key().starts_with(seq_id_prefix));
+
+ if(num_found_docs % index_batch_size == 0 || last_record) {
+ Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, fallback_field_type,
+ token_separators, symbols_to_index, false);
+
+ if(!del_fields.empty()) {
+ for(auto& rec: iter_batch) {
+ index->remove(seq_id, rec.doc, del_fields, true);
+ }
+ }
+
+ iter_batch.clear();
+ }
+
+ if(num_found_docs % ((1 << 14)) == 0) {
+ // having a cheaper higher layer check to prevent checking clock too often
+ auto time_elapsed = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::high_resolution_clock::now() - begin).count();
+
+ if(time_elapsed > 30) {
+ begin = std::chrono::high_resolution_clock::now();
+ LOG(INFO) << "Altered " << num_found_docs << " so far.";
+ }
+ }
+ }
+
+ LOG(INFO) << "Finished altering " << num_found_docs << " document(s).";
+
+ for(auto& del_field: del_fields) {
+ search_schema.erase(del_field.name);
+ auto new_end = std::remove_if(fields.begin(), fields.end(), [&del_field](const field& f) {
+ return f.name == del_field.name;
+ });
+
+ fields.erase(new_end, fields.end());
+
+ if(del_field.name == ".*") {
+ fallback_field_type = "";
+ }
+
+ if(del_field.name == default_sorting_field) {
+ default_sorting_field = "";
+ }
+ }
+
+ index->refresh_schemas({}, del_fields);
+
+ auto persist_op = persist_collection_meta();
+ if(!persist_op.ok()) {
+ return persist_op;
+ }
+
+ return Option<bool>(true);
+}
+
+Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
+ std::unordered_map<std::string, field>& schema_additions,
+ std::vector<field>& del_fields,
+ std::string& fallback_field_type) {
+ if(!schema_changes.is_object()) {
+ return Option<bool>(400, "Bad JSON.");
+ }
+
+ if(schema_changes.size() != 1) {
+ return Option<bool>(400, "Only `fields` can be updated at the moment.");
+ }
+
+ const std::string err_msg = "The `fields` value should be an array of objects containing "
+ "the field `name` and other properties.";
+
+ if(!schema_changes["fields"].is_array() || schema_changes["fields"].empty()) {
+ return Option<bool>(400, err_msg);
+ }
+
+ // basic validation of fields
+ std::vector<field> diff_fields;
+ std::unordered_map<std::string, field> updated_search_schema = search_schema;
+ size_t num_auto_detect_fields = 0;
+
+ for(const auto& kv: schema_changes["fields"].items()) {
+ if(!kv.value().is_object()) {
+ return Option<bool>(400, err_msg);
+ }
+
+ if(!kv.value().contains("name")) {
+ return Option<bool>(400, err_msg);
+ }
+
+ const std::string& field_name = kv.value()["name"].get<std::string>();
+ const auto& field_it = search_schema.find(field_name);
+ auto found_field = (field_it != search_schema.end());
+
+ if(kv.value().contains("drop")) {
+ if(field_name == ".*") {
+ del_fields.emplace_back(".*", field_types::AUTO, false);
+ continue;
+ }
+
+ if(!found_field) {
+ return Option<bool>(400, "Field `" + field_name + "` is not part of collection schema.");
+ }
+
+ if(!kv.value()["drop"].is_boolean() || !kv.value()["drop"].get<bool>()) {
+ return Option<bool>(400, "Field `" + field_name + "` must have a drop value of `true`.");
+ }
+
+ del_fields.push_back(field_it->second);
+
+ } else {
+ // add or update existing field
+ auto is_addition = (!found_field);
+ if(is_addition) {
+ // addition: must validate fields
+ auto parse_op = field::json_field_to_field(kv.value(), diff_fields, fallback_field_type,
+ num_auto_detect_fields);
+ if (!parse_op.ok()) {
+ return parse_op;
+ }
+
+ const auto& f = diff_fields.back();
+ updated_search_schema.emplace(f.name, f);
+ schema_additions.emplace(f.name, f);
+ } else {
+ // update, not supported for now
+ return Option<bool>(400, "Field `" + field_name + "` is already part of schema: only "
+ "field additions and deletions are supported for now.");
+ }
+ }
+ }
+
+ if(num_auto_detect_fields > 1) {
+ return Option<bool>(400, "There can be only one field named `.*`.");
+ }
+
+ // data validations: here we ensure that already stored data is compatible with requested schema changes
+ const std::string seq_id_prefix = get_seq_id_collection_prefix();
+
+ rocksdb::Iterator* iter = store->scan(seq_id_prefix);
+ std::unique_ptr<rocksdb::Iterator> iter_guard(iter);
+
+ size_t num_found_docs = 0;
+ auto begin = std::chrono::high_resolution_clock::now();
+
+ while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
+ num_found_docs++;
+ const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());
+ nlohmann::json document;
+
+ try {
+ document = nlohmann::json::parse(iter->value().ToString());
+ } catch(const std::exception& e) {
+ return Option<bool>(false, "Bad JSON in document: " + document.dump(-1, ' ', false,
+ nlohmann::detail::error_handler_t::ignore));
+ }
+
+ // validate existing data on disk for compatibility via updated_search_schema
+ auto validate_op = Index::validate_index_in_memory(document, seq_id, default_sorting_field,
+ updated_search_schema,
+ index_operation_t::CREATE,
+ fallback_field_type,
+ DIRTY_VALUES::COERCE_OR_REJECT);
+ if(!validate_op.ok()) {
+ std::string schema_err = "Schema change does not match on-disk data, error: " + validate_op.error();
+ return Option<bool>(validate_op.code(), schema_err);
+ }
+
+ if(num_found_docs % ((1 << 14)) == 0) {
+ // having a cheaper higher layer check to prevent checking clock too often
+ auto time_elapsed = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::high_resolution_clock::now() - begin).count();
+
+ if(time_elapsed > 30) {
+ begin = std::chrono::high_resolution_clock::now();
+ LOG(INFO) << "Verified " << num_found_docs << " so far.";
+ }
+ }
+
+ iter->Next();
+ }
+
+ return Option<bool>(true);
+}
+
Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const DIRTY_VALUES& dirty_values) {
std::unique_lock lock(mutex);
@@ -2570,38 +2863,12 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
}
if(!new_fields.empty()) {
- // we should persist changes to fields in store
- std::string coll_meta_json;
- StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);
-
- if(status != StoreStatus::FOUND) {
- return Option<bool>(500, "Could not fetch collection meta from store.");
+ auto persist_op = persist_collection_meta();
+ if(!persist_op.ok()) {
+ return persist_op;
}
- nlohmann::json collection_meta;
-
- try {
- collection_meta = nlohmann::json::parse(coll_meta_json);
- nlohmann::json fields_json = nlohmann::json::array();;
-
- Option<bool> fields_json_op = field::fields_to_json_fields(fields, default_sorting_field, fields_json);
-
- if(!fields_json_op.ok()) {
- return Option<bool>(fields_json_op.code(), fields_json_op.error());
- }
-
- collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
- bool persisted = store->insert(Collection::get_meta_key(name), collection_meta.dump());
-
- if(!persisted) {
- return Option<bool>(500, "Could not persist collection meta to store.");
- }
-
- index->refresh_schemas(new_fields);
-
- } catch(...) {
- return Option<bool>(500, "Unable to parse collection meta.");
- }
+ index->refresh_schemas(new_fields, {});
}
return Option<bool>(true);
@@ -2668,3 +2935,7 @@ std::vector<char> Collection::get_symbols_to_index() {
std::vector<char> Collection::get_token_separators() {
return token_separators;
}
+
+std::string Collection::get_fallback_field_type() {
+ return fallback_field_type;
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index a12353c9..47a25e57 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1119,7 +1119,6 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
}
// Fetch records from the store and re-create memory index
- std::vector<std::string> documents;
const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();
rocksdb::Iterator* iter = cm.store->scan(seq_id_prefix);
@@ -1241,3 +1240,21 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {
preset_configs.erase(preset_name);
return Option<bool>(true);
}
+
+Option<bool> CollectionManager::update_collection(const std::string& name, nlohmann::json& req_json) {
+ if(!req_json["fields"].is_array() || req_json["fields"].empty()) {
+ return Option<bool>(400, "The `fields` value should be an array of objects containing "
+ "the field properties.");
+ }
+
+ auto collection = get_collection(name);
+ if(collection == nullptr) {
+ return Option<bool>(404, "Not found.");
+ }
+
+ // Supported operations:
+ // - Adding a new field
+ // - Dropping a field
+
+ return collection->alter(req_json);
+}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index e9346fe9..d1341e4a 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -160,6 +160,35 @@ bool post_create_collection(const std::shared_ptr<http_req>& req, const std::sha
return false;
}
+bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const std::exception& e) {
+ //LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ auto collection = collectionManager.get_collection(req->params["collection"]);
+
+ if(collection == nullptr) {
+ res->set_404();
+ return false;
+ }
+
+ auto alter_op = collection->alter(req_json);
+ if(!alter_op.ok()) {
+ res->set(alter_op.code(), alter_op.error());
+ return false;
+ }
+
+ res->set_200(req->body);
+ return true;
+}
+
bool del_drop_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
std::string doc_id = req->params["id"];
CollectionManager & collectionManager = CollectionManager::get_instance();
diff --git a/src/field.cpp b/src/field.cpp
index 60831ef1..5d4730c2 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -361,3 +361,152 @@ Option<bool> filter::parse_filter_query(const string& simple_filter_query,
return Option<bool>(true);
}
+
+Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,
+ string& fallback_field_type, size_t& num_auto_detect_fields) {
+
+ if(field_json["name"] == "id") {
+ // No field should exist with the name "id" as it is reserved for internal use
+ // We cannot throw an error here anymore since that will break backward compatibility!
+ LOG(WARNING) << "Collection schema cannot contain a field with name `id`. Ignoring field.";
+ return Option<bool>(true);
+ }
+
+ if(!field_json.is_object() ||
+ field_json.count(fields::name) == 0 || field_json.count(fields::type) == 0 ||
+ !field_json.at(fields::name).is_string() || !field_json.at(fields::type).is_string()) {
+
+ return Option<bool>(400, "Wrong format for `fields`. It should be an array of objects containing "
+ "`name`, `type`, `optional` and `facet` properties.");
+ }
+
+ if(field_json.count(fields::facet) != 0 && !field_json.at(fields::facet).is_boolean()) {
+ return Option<bool>(400, std::string("The `facet` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
+ if(field_json.count(fields::optional) != 0 && !field_json.at(fields::optional).is_boolean()) {
+ return Option<bool>(400, std::string("The `optional` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
+ if(field_json.count(fields::index) != 0 && !field_json.at(fields::index).is_boolean()) {
+ return Option<bool>(400, std::string("The `index` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
+ if(field_json.count(fields::sort) != 0 && !field_json.at(fields::sort).is_boolean()) {
+ return Option<bool>(400, std::string("The `sort` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
+ if(field_json.count(fields::infix) != 0 && !field_json.at(fields::infix).is_boolean()) {
+ return Option<bool>(400, std::string("The `infix` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
+ if(field_json.count(fields::locale) != 0){
+ if(!field_json.at(fields::locale).is_string()) {
+ return Option<bool>(400, std::string("The `locale` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a string."));
+ }
+
+ if(!field_json[fields::locale].get<std::string>().empty() &&
+ field_json[fields::locale].get<std::string>().size() != 2) {
+ return Option<bool>(400, std::string("The `locale` value of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` is not valid."));
+ }
+ }
+
+ if(field_json["name"] == ".*") {
+ if(field_json.count(fields::facet) == 0) {
+ field_json[fields::facet] = false;
+ }
+
+ if(field_json.count(fields::optional) == 0) {
+ field_json[fields::optional] = true;
+ }
+
+ if(field_json.count(fields::index) == 0) {
+ field_json[fields::index] = true;
+ }
+
+ if(field_json.count(fields::locale) == 0) {
+ field_json[fields::locale] = "";
+ }
+
+ if(field_json.count(fields::sort) == 0) {
+ field_json[fields::sort] = false;
+ }
+
+ if(field_json.count(fields::infix) == 0) {
+ field_json[fields::infix] = false;
+ }
+
+ if(field_json[fields::optional] == false) {
+ return Option<bool>(400, "Field `.*` must be an optional field.");
+ }
+
+ if(field_json[fields::facet] == true) {
+ return Option<bool>(400, "Field `.*` cannot be a facet field.");
+ }
+
+ if(field_json[fields::index] == false) {
+ return Option<bool>(400, "Field `.*` must be an index field.");
+ }
+
+ field fallback_field(field_json["name"], field_json["type"], field_json["facet"],
+ field_json["optional"], field_json[fields::index], field_json[fields::locale],
+ field_json[fields::sort], field_json[fields::infix]);
+
+ if(fallback_field.has_valid_type()) {
+ fallback_field_type = fallback_field.type;
+ num_auto_detect_fields++;
+ } else {
+ return Option<bool>(400, "The `type` of field `.*` is invalid.");
+ }
+
+ the_fields.emplace_back(fallback_field);
+ return Option<bool>(true);
+ }
+
+ if(field_json.count(fields::facet) == 0) {
+ field_json[fields::facet] = false;
+ }
+
+ if(field_json.count(fields::index) == 0) {
+ field_json[fields::index] = true;
+ }
+
+ if(field_json.count(fields::locale) == 0) {
+ field_json[fields::locale] = "";
+ }
+
+ if(field_json.count(fields::sort) == 0) {
+ if(field_json["type"] == field_types::INT32 || field_json["type"] == field_types::INT64 ||
+ field_json["type"] == field_types::FLOAT || field_json["type"] == field_types::BOOL ||
+ field_json["type"] == field_types::GEOPOINT || field_json["type"] == field_types::GEOPOINT_ARRAY) {
+ field_json[fields::sort] = true;
+ } else {
+ field_json[fields::sort] = false;
+ }
+ }
+
+ if(field_json.count(fields::infix) == 0) {
+ field_json[fields::infix] = false;
+ }
+
+ if(field_json.count(fields::optional) == 0) {
+ // dynamic fields are always optional
+ bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);
+ field_json[fields::optional] = is_dynamic;
+ }
+
+ the_fields.emplace_back(
+ field(field_json[fields::name], field_json[fields::type], field_json[fields::facet],
+ field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
+ field_json[fields::sort], field_json[fields::infix])
+ );
+
+ return Option<bool>(true);
+}
diff --git a/src/index.cpp b/src/index.cpp
index f2699f3d..31902801 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -443,7 +443,8 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
const std::unordered_map<std::string, field>& search_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
- const std::vector<char>& symbols_to_index) {
+ const std::vector<char>& symbols_to_index,
+ const bool do_validation) {
// runs in a partitioned thread
@@ -460,16 +461,18 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
continue;
}
- Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,
- default_sorting_field,
- search_schema,
- index_rec.operation,
- fallback_field_type,
- index_rec.dirty_values);
+ if(do_validation) {
+ Option<uint32_t> validation_op = validate_index_in_memory(index_rec.doc, index_rec.seq_id,
+ default_sorting_field,
+ search_schema,
+ index_rec.operation,
+ fallback_field_type,
+ index_rec.dirty_values);
- if(!validation_op.ok()) {
- index_rec.index_failure(validation_op.code(), validation_op.error());
- continue;
+ if(!validation_op.ok()) {
+ index_rec.index_failure(validation_op.code(), validation_op.error());
+ continue;
+ }
}
if(index_rec.is_update) {
@@ -513,7 +516,8 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
const std::unordered_map<std::string, field> & search_schema,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
- const std::vector<char>& symbols_to_index) {
+ const std::vector<char>& symbols_to_index,
+ const bool do_validation) {
const size_t concurrency = 4;
const size_t num_threads = std::min(concurrency, iter_batch.size());
@@ -539,7 +543,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
index->thread_pool->enqueue([&, batch_index, batch_len]() {
validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,
- fallback_field_type, token_separators, symbols_to_index);
+ fallback_field_type, token_separators, symbols_to_index, do_validation);
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
@@ -565,7 +569,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
}
if(index_rec.is_update) {
- index->remove(index_rec.seq_id, index_rec.del_doc, index_rec.is_update);
+ index->remove(index_rec.seq_id, index_rec.del_doc, {}, index_rec.is_update);
} else if(index_rec.indexed.ok()) {
num_indexed++;
}
@@ -3738,134 +3742,145 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &toke
return total_cost;
}
-Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update) {
- std::unique_lock lock(mutex);
-
- for(auto it = document.begin(); it != document.end(); ++it) {
- const std::string& field_name = it.key();
- const auto& search_field_it = search_schema.find(field_name);
- if(search_field_it == search_schema.end()) {
- continue;
- }
+void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name) {
+ const auto& search_field_it = search_schema.find(field_name);
+ if(search_field_it == search_schema.end()) {
+ return;
+ }
- const auto& search_field = search_field_it->second;
+ const auto& search_field = search_field_it->second;
- if(!search_field.index) {
- continue;
- }
+ if(!search_field.index) {
+ return;
+ }
- // Go through all the field names and find the keys+values so that they can be removed from in-memory index
- if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
- std::vector<std::string> tokens;
- tokenize_string_field(document, search_field, tokens, search_field.locale);
+ // Go through all the field names and find the keys+values so that they can be removed from in-memory index
+ if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
+ std::vector<std::string> tokens;
+ tokenize_string_field(document, search_field, tokens, search_field.locale);
- for(size_t i = 0; i < tokens.size(); i++) {
- const auto& token = tokens[i];
- const unsigned char *key = (const unsigned char *) token.c_str();
- int key_len = (int) (token.length() + 1);
+ for(size_t i = 0; i < tokens.size(); i++) {
+ const auto& token = tokens[i];
+ const unsigned char *key = (const unsigned char *) token.c_str();
+ int key_len = (int) (token.length() + 1);
- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);
- if(leaf != nullptr) {
- posting_t::erase(leaf->values, seq_id);
- if (posting_t::num_ids(leaf->values) == 0) {
- void* values = art_delete(search_index.at(field_name), key, key_len);
- posting_t::destroy_list(values);
- }
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name), key, key_len);
+ if(leaf != nullptr) {
+ posting_t::erase(leaf->values, seq_id);
+ if (posting_t::num_ids(leaf->values) == 0) {
+ void* values = art_delete(search_index.at(field_name), key, key_len);
+ posting_t::destroy_list(values);
}
+ }
- if(search_field.infix) {
- auto strhash = StringUtils::hash_wy(key, token.size());
- const auto& infix_sets = infix_index.at(search_field.name);
- infix_sets[strhash % 4]->erase(token);
+ if(search_field.infix) {
+ auto strhash = StringUtils::hash_wy(key, token.size());
+ const auto& infix_sets = infix_index.at(search_field.name);
+ infix_sets[strhash % 4]->erase(token);
+ }
+ }
+ } else if(search_field.is_int32()) {
+ const std::vector<int32_t>& values = search_field.is_single_integer() ?
+ std::vector<int32_t>{document[field_name].get<int32_t>()} :
+ document[field_name].get<std::vector<int32_t>>();
+ for(int32_t value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ num_tree->remove(value, seq_id);
+ }
+ } else if(search_field.is_int64()) {
+ const std::vector<int64_t>& values = search_field.is_single_integer() ?
+ std::vector<int64_t>{document[field_name].get<int64_t>()} :
+ document[field_name].get<std::vector<int64_t>>();
+ for(int64_t value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ num_tree->remove(value, seq_id);
+ }
+ } else if(search_field.is_float()) {
+ const std::vector<float>& values = search_field.is_single_float() ?
+ std::vector<float>{document[field_name].get<float>()} :
+ document[field_name].get<std::vector<float>>();
+ for(float value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ int64_t fintval = float_to_in64_t(value);
+ num_tree->remove(fintval, seq_id);
+ }
+ } else if(search_field.is_bool()) {
+
+ const std::vector<bool>& values = search_field.is_single_bool() ?
+ std::vector<bool>{document[field_name].get<bool>()} :
+ document[field_name].get<std::vector<bool>>();
+ for(bool value: values) {
+ num_tree_t* num_tree = numerical_index.at(field_name);
+ int64_t bool_int64 = value ? 1 : 0;
+ num_tree->remove(bool_int64, seq_id);
+ }
+ } else if(search_field.is_geopoint()) {
+ auto geo_index = geopoint_index[field_name];
+ S2RegionTermIndexer::Options options;
+ options.set_index_contains_points_only(true);
+ S2RegionTermIndexer indexer(options);
+
+ const std::vector<std::vector<double>>& latlongs = search_field.is_single_geopoint() ?
+ std::vector<std::vector<double>>{document[field_name].get<std::vector<double>>()} :
+ document[field_name].get<std::vector<std::vector<double>>>();
+
+ for(const std::vector<double>& latlong: latlongs) {
+ S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
+ for(const auto& term: indexer.GetIndexTerms(point, "")) {
+ auto term_it = geo_index->find(term);
+ if(term_it == geo_index->end()) {
+ continue;
}
- }
- } else if(search_field.is_int32()) {
- const std::vector<int32_t>& values = search_field.is_single_integer() ?
- std::vector<int32_t>{document[field_name].get<int32_t>()} :
- document[field_name].get<std::vector<int32_t>>();
- for(int32_t value: values) {
- num_tree_t* num_tree = numerical_index.at(field_name);
- num_tree->remove(value, seq_id);
- }
- } else if(search_field.is_int64()) {
- const std::vector<int64_t>& values = search_field.is_single_integer() ?
- std::vector<int64_t>{document[field_name].get<int64_t>()} :
- document[field_name].get<std::vector<int64_t>>();
- for(int64_t value: values) {
- num_tree_t* num_tree = numerical_index.at(field_name);
- num_tree->remove(value, seq_id);
- }
- } else if(search_field.is_float()) {
- const std::vector<float>& values = search_field.is_single_float() ?
- std::vector<float>{document[field_name].get<float>()} :
- document[field_name].get<std::vector<float>>();
- for(float value: values) {
- num_tree_t* num_tree = numerical_index.at(field_name);
- int64_t fintval = float_to_in64_t(value);
- num_tree->remove(fintval, seq_id);
- }
- } else if(search_field.is_bool()) {
-
- const std::vector<bool>& values = search_field.is_single_bool() ?
- std::vector<bool>{document[field_name].get<bool>()} :
- document[field_name].get<std::vector<bool>>();
- for(bool value: values) {
- num_tree_t* num_tree = numerical_index.at(field_name);
- int64_t bool_int64 = value ? 1 : 0;
- num_tree->remove(bool_int64, seq_id);
- }
- } else if(search_field.is_geopoint()) {
- auto geo_index = geopoint_index[field_name];
- S2RegionTermIndexer::Options options;
- options.set_index_contains_points_only(true);
- S2RegionTermIndexer indexer(options);
-
- const std::vector<std::vector<double>>& latlongs = search_field.is_single_geopoint() ?
- std::vector<std::vector<double>>{document[field_name].get<std::vector<double>>()} :
- document[field_name].get<std::vector<std::vector<double>>>();
-
- for(const std::vector<double>& latlong: latlongs) {
- S2Point point = S2LatLng::FromDegrees(latlong[0], latlong[1]).ToPoint();
- for(const auto& term: indexer.GetIndexTerms(point, "")) {
- auto term_it = geo_index->find(term);
- if(term_it == geo_index->end()) {
- continue;
- }
- std::vector<uint32_t>& ids = term_it->second;
- ids.erase(std::remove(ids.begin(), ids.end(), seq_id), ids.end());
- if(ids.empty()) {
- geo_index->erase(term);
- }
+ std::vector<uint32_t>& ids = term_it->second;
+ ids.erase(std::remove(ids.begin(), ids.end(), seq_id), ids.end());
+ if(ids.empty()) {
+ geo_index->erase(term);
}
}
+ }
- if(!search_field.is_single_geopoint()) {
- spp::sparse_hash_map<uint32_t, int64_t*>*& field_geo_array_map = geo_array_index.at(field_name);
- auto geo_array_it = field_geo_array_map->find(seq_id);
- if(geo_array_it != field_geo_array_map->end()) {
- delete [] geo_array_it->second;
- field_geo_array_map->erase(seq_id);
- }
+ if(!search_field.is_single_geopoint()) {
+ spp::sparse_hash_map<uint32_t, int64_t*>*& field_geo_array_map = geo_array_index.at(field_name);
+ auto geo_array_it = field_geo_array_map->find(seq_id);
+ if(geo_array_it != field_geo_array_map->end()) {
+ delete [] geo_array_it->second;
+ field_geo_array_map->erase(seq_id);
}
}
+ }
- // remove facets
- const auto& field_facets_it = facet_index_v3.find(field_name);
+ // remove facets
+ const auto& field_facets_it = facet_index_v3.find(field_name);
- if(field_facets_it != facet_index_v3.end()) {
- const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
- }
+ if(field_facets_it != facet_index_v3.end()) {
+ const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
+ if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
+ field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
}
+ }
- // remove sort field
- if(sort_index.count(field_name) != 0) {
- sort_index[field_name]->erase(seq_id);
- }
+ // remove sort field
+ if(sort_index.count(field_name) != 0) {
+ sort_index[field_name]->erase(seq_id);
+ }
+
+ if(str_sort_index.count(field_name) != 0) {
+ str_sort_index[field_name]->remove(seq_id);
+ }
+}
- if(str_sort_index.count(field_name) != 0) {
- str_sort_index[field_name]->remove(seq_id);
+Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document,
+ const std::vector<field>& del_fields, const bool is_update) {
+ std::unique_lock lock(mutex);
+
+ if(!del_fields.empty()) {
+ for(auto& the_field: del_fields) {
+ remove_field(seq_id, document, the_field.name);
+ }
+ } else {
+ for(auto it = document.begin(); it != document.end(); ++it) {
+ const std::string& field_name = it.key();
+ remove_field(seq_id, document, field_name);
}
}
@@ -3909,7 +3924,7 @@ const spp::sparse_hash_map<std::string, array_mapped_infix_t>& Index::_get_infix
return infix_index;
};
-void Index::refresh_schemas(const std::vector<field>& new_fields) {
+void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vector<field>& del_fields) {
std::unique_lock lock(mutex);
for(const auto & new_field: new_fields) {
@@ -3967,6 +3982,65 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {
infix_index.emplace(new_field.name, infix_sets);
}
}
+
+ for(const auto & del_field: del_fields) {
+ search_schema.erase(del_field.name);
+
+ if(del_field.is_string() || field_types::is_string_or_array(del_field.type)) {
+ art_tree_destroy(search_index[del_field.name]);
+ delete search_index[del_field.name];
+ search_index.erase(del_field.name);
+ } else if(del_field.is_geopoint()) {
+ delete geopoint_index[del_field.name];
+ geopoint_index.erase(del_field.name);
+
+ if(!del_field.is_single_geopoint()) {
+ spp::sparse_hash_map<uint32_t, int64_t*>* geo_array_map = geo_array_index[del_field.name];
+ for(auto& kv: *geo_array_map) {
+ delete kv.second;
+ }
+ delete geo_array_map;
+ geo_array_index.erase(del_field.name);
+ }
+ } else {
+ delete numerical_index[del_field.name];
+ numerical_index.erase(del_field.name);
+ }
+
+ if(del_field.is_sortable()) {
+ if(del_field.is_num_sortable()) {
+ delete sort_index[del_field.name];
+ sort_index.erase(del_field.name);
+ } else if(del_field.is_str_sortable()) {
+ delete str_sort_index[del_field.name];
+ str_sort_index.erase(del_field.name);
+ }
+ }
+
+ if(del_field.is_facet()) {
+ auto& arr = facet_index_v3[del_field.name];
+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
+ delete arr[i];
+ }
+
+ facet_index_v3.erase(del_field.name);
+
+ if(!del_field.is_string()) {
+ art_tree_destroy(search_index[del_field.faceted_name()]);
+ delete search_index[del_field.faceted_name()];
+ search_index.erase(del_field.faceted_name());
+ }
+ }
+
+ if(del_field.infix) {
+ auto& infix_set = infix_index[del_field.name];
+ for(size_t i = 0; i < infix_set.size(); i++) {
+ delete infix_set[i];
+ }
+
+ infix_index.erase(del_field.name);
+ }
+ }
}
Option<uint32_t> Index::coerce_string(const DIRTY_VALUES& dirty_values, const std::string& fallback_field_type,
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index d22ae714..16e96100 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -44,6 +44,7 @@ void master_server_routes() {
// collection management
server->post("/collections", post_create_collection);
+ server->patch("/collections/:collection", patch_update_collection);
server->get("/collections", get_collections);
server->del("/collections/:collection", del_drop_collection);
server->get("/collections/:collection", get_collection_summary);
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 759d3765..b5610181 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -1368,7 +1368,6 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {
while(curr_block != end_block && curr_block->ids.last() < id) {
curr_block = curr_block->next;
- // FIXME: remove duplication
delete [] ids;
delete [] offset_index;
delete [] offsets;
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index f822c8d1..aafd889d 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -431,7 +431,8 @@ TEST_F(CollectionOverrideTest, ExcludeIncludeFacetFilterQuery) {
ASSERT_EQ(1, overrides.size());
auto override_json = overrides["include-rule"].to_json();
ASSERT_FALSE(override_json.contains("filter_by"));
- ASSERT_FALSE(override_json.contains("remove_matched_tokens"));
+ ASSERT_TRUE(override_json.contains("remove_matched_tokens"));
+ ASSERT_FALSE(override_json["remove_matched_tokens"].get<bool>());
auto results = coll_mul_fields->search("not-found", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
new file mode 100644
index 00000000..c302449b
--- /dev/null
+++ b/test/collection_schema_change_test.cpp
@@ -0,0 +1,366 @@
+#include <gtest/gtest.h>
+#include <string>
+#include <vector>
+#include <fstream>
+#include <algorithm>
+#include <collection_manager.h>
+#include "collection.h"
+
+class CollectionSchemaChangeTest : public ::testing::Test {
+protected:
+ Store *store;
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
+
+ std::vector<std::string> query_fields;
+ std::vector<sort_by> sort_fields;
+
+ void setupCollection() {
+ std::string state_dir_path = "/tmp/typesense_test/collection_schema_change";
+ LOG(INFO) << "Truncating and creating: " << state_dir_path;
+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
+
+ store = new Store(state_dir_path);
+ collectionManager.init(store, 1.0, "auth_key", quit);
+ collectionManager.load(8, 1000);
+ }
+
+ virtual void SetUp() {
+ setupCollection();
+ }
+
+ virtual void TearDown() {
+ collectionManager.dispose();
+ delete store;
+ }
+};
+
+TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "The quick brown fox was too fast.";
+ doc["tags"] = {"experimental", "news"};
+ doc["category"] = "animals";
+ doc["quantity"] = 100;
+ doc["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("fox",
+ {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ auto schema_changes = R"({
+ "fields": [
+ {"name": "tags", "type": "string[]", "infix": true},
+ {"name": "category", "type": "string", "sort": true}
+ ]
+ })"_json;
+
+ auto alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ results = coll1->search("eriment",
+ {"tags"}, "", {}, {}, {0}, 3, 1, FREQUENCY,
+ {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ sort_fields = { sort_by("category", "DESC") };
+ results = coll1->search("*",
+ {}, "", {}, sort_fields, {0}, 3, 1, FREQUENCY,
+ {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ schema_changes = R"({
+ "fields": [
+ {"name": "quantity", "type": "int32", "facet": true}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ results = coll1->search("*",
+ {}, "quantity: 100", {"quantity"}, {}, {0}, 3, 1, FREQUENCY,
+ {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_STREQ("100", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
+
+ // add a dynamic field
+ schema_changes = R"({
+ "fields": [
+ {"name": ".*_bool", "type": "bool"}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ doc["id"] = "1";
+ doc["title"] = "The one";
+ doc["tags"] = {"sports", "news"};
+ doc["category"] = "things";
+ doc["quantity"] = 200;
+ doc["points"] = 100;
+ doc["on_sale_bool"] = true;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ results = coll1->search("*",
+ {}, "on_sale_bool: true", {}, {}, {0}, 3, 1, FREQUENCY,
+ {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ // add auto field
+ schema_changes = R"({
+ "fields": [
+ {"name": ".*", "type": "auto"}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ doc["id"] = "2";
+ doc["title"] = "The two";
+ doc["tags"] = {"sports", "news"};
+ doc["category"] = "things";
+ doc["quantity"] = 200;
+ doc["points"] = 100;
+ doc["on_sale_bool"] = false;
+ doc["foobar"] = 123;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ results = coll1->search("*",
+ {}, "foobar: 123", {}, {}, {0}, 3, 1, FREQUENCY,
+ {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_STREQ("2", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ // try to add auto field again
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("The schema already contains a `.*` field.", alter_op.error());
+
+ // try to add a regular field with 2 auto fields
+ schema_changes = R"({
+ "fields": [
+ {"name": "bar", "type": "auto"},
+ {"name": ".*", "type": "auto"},
+ {"name": ".*", "type": "auto"}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("There can be only one field named `.*`.", alter_op.error());
+
+ auto search_schema = coll1->get_schema();
+ ASSERT_EQ(7, search_schema.size());
+
+ auto coll_fields = coll1->get_fields();
+ ASSERT_EQ(8, coll_fields.size());
+
+ // try restoring collection from disk: all fields should be preserved
+ collectionManager.dispose();
+ delete store;
+
+ store = new Store("/tmp/typesense_test/collection_schema_change");
+ collectionManager.init(store, 1.0, "auth_key", quit);
+ collectionManager.load(8, 1000);
+ coll1 = collectionManager.get_collection("coll1").get();
+
+ search_schema = coll1->get_schema();
+ ASSERT_EQ(7, search_schema.size());
+
+ coll_fields = coll1->get_fields();
+ ASSERT_EQ(8, coll_fields.size());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSchemaChangeTest, DropFieldsFromCollection) {
+ std::vector<field> fields = {field(".*", field_types::AUTO, false),
+ field("title", field_types::STRING, false, false, true, "", 1, 1),
+ field("location", field_types::GEOPOINT, false),
+ field("locations", field_types::GEOPOINT_ARRAY, false),
+ field("tags", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, true),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "auto").get();
+
+ std::vector<std::vector<double>> lat_lngs;
+ lat_lngs.push_back({48.85821022164442, 2.294239067890161});
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "The quick brown fox was too fast.";
+ doc["location"] = {48.85821022164442, 2.294239067890161};
+ doc["locations"] = lat_lngs;
+ doc["tags"] = {"experimental", "news"};
+ doc["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("*",
+ {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ auto schema_changes = R"({
+ "fields": [
+ {"name": ".*", "drop": true},
+ {"name": "title", "drop": true},
+ {"name": "location", "drop": true},
+ {"name": "locations", "drop": true},
+ {"name": "tags", "drop": true},
+ {"name": "points", "drop": true}
+ ]
+ })"_json;
+
+ auto alter_op = coll1->alter(schema_changes);
+ ASSERT_TRUE(alter_op.ok());
+
+ results = coll1->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ auto res_op = coll1->search("quick", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Could not find a field named `title` in the schema.", res_op.error());
+
+ auto search_schema = coll1->get_schema();
+ ASSERT_EQ(0, search_schema.size());
+
+ auto coll_fields = coll1->get_fields();
+ ASSERT_EQ(0, coll_fields.size());
+
+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_infix_index().size());
+ ASSERT_EQ(1, coll1->_get_index()->num_seq_ids());
+ ASSERT_EQ("", coll1->get_fallback_field_type());
+ ASSERT_EQ("", coll1->get_default_sorting_field());
+
+ // try restoring collection from disk: all fields should be deleted
+ collectionManager.dispose();
+ delete store;
+
+ store = new Store("/tmp/typesense_test/collection_schema_change");
+ collectionManager.init(store, 1.0, "auth_key", quit);
+ collectionManager.load(8, 1000);
+ coll1 = collectionManager.get_collection("coll1").get();
+
+ search_schema = coll1->get_schema();
+ ASSERT_EQ(0, search_schema.size());
+
+ coll_fields = coll1->get_fields();
+ ASSERT_EQ(0, coll_fields.size());
+
+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_infix_index().size());
+ ASSERT_EQ(1, coll1->_get_index()->num_seq_ids());
+ ASSERT_EQ("", coll1->get_default_sorting_field());
+ ASSERT_EQ("", coll1->get_fallback_field_type());
+
+ results = coll1->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ res_op = coll1->search("quick", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Could not find a field named `title` in the schema.", res_op.error());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSchemaChangeTest, AlterValidations) {
+ std::vector<field> fields = {field(".*", field_types::AUTO, false),
+ field("title", field_types::STRING, false, false, true, "", 1, 1),
+ field("location", field_types::GEOPOINT, false),
+ field("locations", field_types::GEOPOINT_ARRAY, false),
+ field("tags", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, true),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points", 0, "auto").get();
+
+ std::vector<std::vector<double>> lat_lngs;
+ lat_lngs.push_back({48.85821022164442, 2.294239067890161});
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "The quick brown fox was too fast.";
+ doc["location"] = {48.85821022164442, 2.294239067890161};
+ doc["locations"] = lat_lngs;
+ doc["tags"] = {"experimental", "news"};
+ doc["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // 1. Modify existing field, which is not supported
+
+ auto schema_changes = R"({
+ "fields": [
+ {"name": "title", "type": "string[]"}
+ ]
+ })"_json;
+
+ auto alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Field `title` is already part of schema: only field additions and deletions are supported for now.",
+ alter_op.error());
+
+ // 2. Bad field format
+ schema_changes = R"({
+ "fields": [
+ {"name": "age", "typezzz": "int32"}
+ ]
+ })"_json;
+
+ alter_op = coll1->alter(schema_changes);
+ ASSERT_FALSE(alter_op.ok());
+ ASSERT_EQ("Wrong format for `fields`. It should be an array of objects containing `name`, `type`, "
+ "`optional` and `facet` properties.",alter_op.error());
+
+ collectionManager.drop_collection("coll1");
+}
+