Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Do value coercion before removing value from index.

Kishore Nallan37d7ecabe737
diff --git a/include/collection.h b/include/collection.h
index ca14a4d1..32ece94d 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -183,7 +183,7 @@ private:
bool& found_highlight,
bool& found_full_highlight) const;
- void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
+ void remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);
void process_remove_field_for_embedding_fields(const field& del_field, std::vector<field>& garbage_embed_fields);
diff --git a/include/index.h b/include/index.h
index 5c9d9be3..1b730535 100644
--- a/include/index.h
+++ b/include/index.h
@@ -706,10 +706,10 @@ public:
bool enable_typos_for_alpha_numerical_tokens = true
) const;
- void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
+ void remove_field(uint32_t seq_id, nlohmann::json& document, const std::string& field_name,
const bool is_update);
- Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document,
+ Option<uint32_t> remove(const uint32_t seq_id, nlohmann::json & document,
const std::vector<field>& del_fields, const bool is_update);
static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,
diff --git a/src/collection.cpp b/src/collection.cpp
index 3c282e67..381239c7 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4460,7 +4460,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) const {
return Option<nlohmann::json>(document);
}
-void Collection::remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {
+void Collection::remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {
const std::string& id = document["id"];
{
diff --git a/src/index.cpp b/src/index.cpp
index e1751987..074d7084 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -555,16 +555,17 @@ void Index::validate_and_preprocess(Index *index,
size_t Index::
batch_memory_index(Index *index,
- std::vector<index_record>& iter_batch,
- const std::string & default_sorting_field,
- const tsl::htrie_map<char, field> & actual_search_schema,
- const tsl::htrie_map<char, field> & embedding_fields,
- const std::string& fallback_field_type,
- const std::vector<char>& token_separators,
- const std::vector<char>& symbols_to_index,
- const bool do_validation, const size_t remote_embedding_batch_size,
- const size_t remote_embedding_timeout_ms, const size_t remote_embedding_num_tries, const bool generate_embeddings,
- const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields) {
+ std::vector<index_record>& iter_batch,
+ const std::string & default_sorting_field,
+ const tsl::htrie_map<char, field> & actual_search_schema,
+ const tsl::htrie_map<char, field> & embedding_fields,
+ const std::string& fallback_field_type,
+ const std::vector<char>& token_separators,
+ const std::vector<char>& symbols_to_index,
+ const bool do_validation, const size_t remote_embedding_batch_size,
+ const size_t remote_embedding_timeout_ms, const size_t remote_embedding_num_tries,
+ const bool generate_embeddings,
+ const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields) {
const size_t concurrency = 4;
const size_t num_threads = std::min(concurrency, iter_batch.size());
const size_t window_size = (num_threads == 0) ? 0 :
@@ -6733,7 +6734,7 @@ void Index::remove_facet_token(const field& search_field, spp::sparse_hash_map<s
}
}
-void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
+void Index::remove_field(uint32_t seq_id, nlohmann::json& document, const std::string& field_name,
const bool is_update) {
const auto& search_field_it = search_schema.find(field_name);
if(search_field_it == search_schema.end()) {
@@ -6750,6 +6751,13 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const
return ;
}
+ auto coerce_op = validator_t::coerce_element(search_field, document, document[field_name],
+ "", DIRTY_VALUES::COERCE_OR_REJECT);
+ if(!coerce_op.ok()) {
+ LOG(ERROR) << "Bad type for field " << field_name;
+ return ;
+ }
+
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
std::vector<std::string> tokens;
@@ -6892,7 +6900,7 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const
}
}
-Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & document,
+Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document,
const std::vector<field>& del_fields, const bool is_update) {
std::unique_lock lock(mutex);