Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix reindexing all fields on alter
ozanarmagan48b2c6bba70b
diff --git a/include/index.h b/include/index.hindex 66cf3365..e994b6da 100644--- a/include/index.h+++ b/include/index.h@@ -689,7 +689,8 @@ public:const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,- const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true);+ const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true,+ const bool use_addition_fields = false, const tsl::htrie_map<char, field>& addition_fields = {});void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);diff --git a/src/collection.cpp b/src/collection.cppindex 19463522..fd87b61c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -3843,7 +3843,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields}Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,- fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field);+ fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field, true, schema_additions);if(found_embedding_field) {for(auto& index_record : iter_batch) {if(index_record.indexed.ok()) {diff --git a/src/index.cpp b/src/index.cppindex d545a927..f649cf4b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -513,17 +513,20 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& itesize_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,const std::string & default_sorting_field,- const tsl::htrie_map<char, field> & search_schema,+ const tsl::htrie_map<char, field> & actual_search_schema,const tsl::htrie_map<char, field> & embedding_fields,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,- const bool do_validation, const size_t remote_embedding_batch_size, const bool generate_embeddings) {+ const bool do_validation, const size_t remote_embedding_batch_size, const bool generate_embeddings,+ const bool use_addition_fields, const tsl::htrie_map<char, field>& addition_fields) {const size_t concurrency = 4;const size_t num_threads = std::min(concurrency, iter_batch.size());const size_t window_size = (num_threads == 0) ? 0 :(iter_batch.size() + num_threads - 1) / num_threads; // rounds up+ const auto& search_schema = use_addition_fields ? addition_fields : actual_search_schema;+size_t num_indexed = 0;size_t num_processed = 0;@@ -547,7 +550,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bindex->thread_pool->enqueue([&, batch_index, batch_len]() {write_log_index = local_write_log_index;- validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, search_schema,+ validate_and_preprocess(index, iter_batch, batch_index, batch_len, default_sorting_field, actual_search_schema,embedding_fields, fallback_field_type, token_separators, symbols_to_index, do_validation, remote_embedding_batch_size, generate_embeddings);std::unique_lock<std::mutex> lock(m_process);