Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle unhandled errors during indexing.
Kishore Nallan34a698f43280
diff --git a/src/collection.cpp b/src/collection.cppindex 966955e6..1cd40e34 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2447,21 +2447,28 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constif (new_field.is_num_sort_field()) {// only numerical fields are added to sort index in dynamic type detectionnew_field.sort = true;- sort_schema.emplace(new_field.name, new_field);}- search_schema.emplace(new_field.name, new_field);- fields.emplace_back(new_field);new_fields.emplace_back(new_field);-- if(new_field.is_facet()) {- facet_schema.emplace(new_field.name, new_field);- }}kv++;}+ for(auto& new_field: new_fields) {+ if (new_field.is_num_sort_field()) {+ // only numerical fields are added to sort index in dynamic type detection+ sort_schema.emplace(new_field.name, new_field);+ }++ search_schema.emplace(new_field.name, new_field);+ if(new_field.is_facet()) {+ facet_schema.emplace(new_field.name, new_field);+ }++ fields.emplace_back(new_field);+ }+if(!new_fields.empty()) {// we should persist changes to fields in storestd::string coll_meta_json;diff --git a/src/index.cpp b/src/index.cppindex ee0f395f..8f857611 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -569,7 +569,15 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bindex->thread_pool->enqueue([&]() {const field& f = (field_name == "id") ?field("id", field_types::STRING, false) : search_schema.at(field_name);- index->index_field_in_memory(f, iter_batch);+ try {+ index->index_field_in_memory(f, iter_batch);+ } catch(std::exception& e) {+ LOG(ERROR) << "Unhandled Typesense error: " << e.what();+ for(auto& record: iter_batch) {+ record.index_failure(500, "Unhandled Typesense error in index batch, check logs for details.");+ }+ }+std::unique_lock<std::mutex> lock(m_process);num_processed++;cv_process.notify_one();diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex afaa94f2..0844fd98 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1459,3 +1459,47 @@ TEST_F(CollectionAllFieldsTest, EmptyArrayShouldBeAcceptedAsFirstValueOfAutoFielcollectionManager.drop_collection("coll1");}++TEST_F(CollectionAllFieldsTest, SchemaUpdateShouldBeAtomicForAllFields) {+ // when a given field in a document is "bad", other fields should not be partially added to schema+ Collection *coll1;++ std::vector<field> fields = {field(".*", field_types::AUTO, false, true),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ auto op = collectionManager.create_collection("coll1", 1, fields, "",+ 0, field_types::AUTO);+ ASSERT_TRUE(op.ok());+ coll1 = op.get();+ }++ // insert a document with bad data for that key, but surrounded by "good" keys+ // this should NOT end up creating schema changes+ nlohmann::json doc;+ doc["int_1"] = 100;+ doc["int_2"] = nlohmann::json::array();+ doc["int_2"].push_back(nlohmann::json::object());+ doc["int_3"] = 300;++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_FALSE(add_op.ok());++ ASSERT_EQ(1, coll1->get_fields().size());+ ASSERT_EQ(0, coll1->get_sort_fields().size());+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().size());++ // now insert document with just "int_1" key+ nlohmann::json doc2;+ doc2["int_1"] = 200;+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ ASSERT_EQ(2, coll1->get_fields().size());+ ASSERT_EQ(1, coll1->get_sort_fields().size());+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().size());++ collectionManager.drop_collection("coll1");+}