Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix reindexing all fields on alter

ozanarmagan48b2c6bba70b
diff --git a/include/index.h b/include/index.h
index 66cf3365..e994b6da 100644
--- a/include/index.h
+++ b/include/index.h
@@ -689,7 +689,8 @@ public:
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index,
- const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true);
+ const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true,
+ const bool use_addition_fields = false, const tsl::htrie_map<char, field>& addition_fields = {});
void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);
diff --git a/src/collection.cpp b/src/collection.cpp
index 19463522..fd87b61c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -3843,7 +3843,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
}
Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,
- fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field);
+ fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field, true, schema_additions);
if(found_embedding_field) {
for(auto& index_record : iter_batch) {
if(index_record.indexed.ok()) {
diff --git a/src/index.cpp b/src/index.cpp
index d545a927..f649cf4b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -513,17 +513,20 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,
const std::string & default_sorting_field,
- const tsl::htrie_map<char, field> & search_schema,
+ const tsl::htrie_map<char, field> & actual_search_schema,
const tsl::htrie_map<char, field> & embedding_fields,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index,
- const bool do_validation, const size_t remote_embedding_batch_size, const bool generate_embeddings) {
+ const bool do_validation, const size_t remote_embedding_batch_size, const bool generate_embeddings,
+ const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields) {
const size_t concurrency = 4;
const size_t num_threads = std::min(concurrency, iter_batch.size());
const size_t window_size = (num_threads == 0) ? 0 :
(iter_batch.size() + num_threads - 1) / num_threads; // rounds up
+ const auto& search_schema = use_addition_fields ? addition_fields : actual_search_schema;
+
size_t num_indexed = 0;
size_t num_processed = 0;
@@ -547,7 +550,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
index->thread_pool->enqueue([&, batch_index, batch_len]() {
write_log_index = local_write_log_index;
- validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,
+ validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, actual_search_schema,
embedding_fields, fallback_field_type, token_separators, symbols_to_index, do_validation, remote_embedding_batch_size, generate_embeddings);
std::unique_lock<std::mutex> lock(m_process);