Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Do value coercion before removing value from index.
Kishore Nallan37d7ecabe737
diff --git a/include/collection.h b/include/collection.hindex ca14a4d1..32ece94d 100644--- a/include/collection.h+++ b/include/collection.h@@ -183,7 +183,7 @@ private:bool& found_highlight,bool& found_full_highlight) const;- void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);+ void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);void process_remove_field_for_embedding_fields(const field& del_field, std::vector<field>& garbage_embed_fields);diff --git a/include/index.h b/include/index.hindex 5c9d9be3..1b730535 100644--- a/include/index.h+++ b/include/index.h@@ -706,10 +706,10 @@ public:bool enable_typos_for_alpha_numerical_tokens = true) const;- void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,+ void remove_field(uint32_t seq_id, nlohmann::json& document, const std::string& field_name,const bool is_update);- Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document,+ Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document,const std::vector<field>& del_fields, const bool is_update);static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,diff --git a/src/collection.cpp b/src/collection.cppindex 3c282e67..381239c7 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4460,7 +4460,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) const {return Option<nlohmann::json>(document);}-void Collection::remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {+void Collection::remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {const std::string& id = document["id"];{diff --git a/src/index.cpp b/src/index.cppindex e1751987..074d7084 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -555,16 +555,17 @@ void Index::validate_and_preprocess(Index *index,size_t Index::batch_memory_index(Index *index,- std::vector<index_record>& iter_batch,- const std::string & default_sorting_field,- const tsl::htrie_map<char, field> & actual_search_schema,- const tsl::htrie_map<char, field> & embedding_fields,- const std::string& fallback_field_type,- const std::vector<char>& token_separators,- const std::vector<char>& symbols_to_index,- const bool do_validation, const size_t remote_embedding_batch_size,- const size_t remote_embedding_timeout_ms, const size_t remote_embedding_num_tries, const bool generate_embeddings,- const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields) {+ std::vector<index_record>& iter_batch,+ const std::string & default_sorting_field,+ const tsl::htrie_map<char, field> & actual_search_schema,+ const tsl::htrie_map<char, field> & embedding_fields,+ const std::string& fallback_field_type,+ const std::vector<char>& token_separators,+ const std::vector<char>& symbols_to_index,+ const bool do_validation, const size_t remote_embedding_batch_size,+ const size_t remote_embedding_timeout_ms, const size_t remote_embedding_num_tries,+ const bool generate_embeddings,+ const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields) {const size_t concurrency = 4;const size_t num_threads = std::min(concurrency, iter_batch.size());const size_t window_size = (num_threads == 0) ? 0 :@@ -6733,7 +6734,7 @@ void Index::remove_facet_token(const field& search_field, spp::sparse_hash_map<s}}-void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,+void Index::remove_field(uint32_t seq_id, nlohmann::json& document, const std::string& field_name,const bool is_update) {const auto& search_field_it = search_schema.find(field_name);if(search_field_it == search_schema.end()) {@@ -6750,6 +6751,13 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, constreturn ;}+ auto coerce_op = validator_t::coerce_element(search_field, document, document[field_name],+ "", DIRTY_VALUES::COERCE_OR_REJECT);+ if(!coerce_op.ok()) {+ LOG(ERROR) << "Bad type for field " << field_name;+ return ;+ }+// Go through all the field names and find the keys+values so that they can be removed from in-memory indexif(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {std::vector<std::string> tokens;@@ -6892,7 +6900,7 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const}}-Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document,+Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document,const std::vector<field>& del_fields, const bool is_update) {std::unique_lock lock(mutex);