Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle unhandled errors during indexing.

Kishore Nallan34a698f43280
diff --git a/src/collection.cpp b/src/collection.cpp
index 966955e6..1cd40e34 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2447,21 +2447,28 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
if (new_field.is_num_sort_field()) {
// only numerical fields are added to sort index in dynamic type detection
new_field.sort = true;
- sort_schema.emplace(new_field.name, new_field);
}
- search_schema.emplace(new_field.name, new_field);
- fields.emplace_back(new_field);
new_fields.emplace_back(new_field);
-
- if(new_field.is_facet()) {
- facet_schema.emplace(new_field.name, new_field);
- }
}
kv++;
}
+ for(auto& new_field: new_fields) {
+ if (new_field.is_num_sort_field()) {
+ // only numerical fields are added to sort index in dynamic type detection
+ sort_schema.emplace(new_field.name, new_field);
+ }
+
+ search_schema.emplace(new_field.name, new_field);
+ if(new_field.is_facet()) {
+ facet_schema.emplace(new_field.name, new_field);
+ }
+
+ fields.emplace_back(new_field);
+ }
+
if(!new_fields.empty()) {
// we should persist changes to fields in store
std::string coll_meta_json;
diff --git a/src/index.cpp b/src/index.cpp
index ee0f395f..8f857611 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -569,7 +569,15 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
index->thread_pool->enqueue([&]() {
const field& f = (field_name == "id") ?
field("id", field_types::STRING, false) : search_schema.at(field_name);
- index->index_field_in_memory(f, iter_batch);
+ try {
+ index->index_field_in_memory(f, iter_batch);
+ } catch(std::exception& e) {
+ LOG(ERROR) << "Unhandled Typesense error: " << e.what();
+ for(auto& record: iter_batch) {
+ record.index_failure(500, "Unhandled Typesense error in index batch, check logs for details.");
+ }
+ }
+
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
cv_process.notify_one();
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index afaa94f2..0844fd98 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1459,3 +1459,47 @@ TEST_F(CollectionAllFieldsTest, EmptyArrayShouldBeAcceptedAsFirstValueOfAutoFiel
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionAllFieldsTest, SchemaUpdateShouldBeAtomicForAllFields) {
+ // when a given field in a document is "bad", other fields should not be partially added to schema
+ Collection *coll1;
+
+ std::vector<field> fields = {field(".*", field_types::AUTO, false, true),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ auto op = collectionManager.create_collection("coll1", 1, fields, "",
+ 0, field_types::AUTO);
+ ASSERT_TRUE(op.ok());
+ coll1 = op.get();
+ }
+
+ // insert a document with bad data for that key, but surrounded by "good" keys
+ // this should NOT end up creating schema changes
+ nlohmann::json doc;
+ doc["int_1"] = 100;
+ doc["int_2"] = nlohmann::json::array();
+ doc["int_2"].push_back(nlohmann::json::object());
+ doc["int_3"] = 300;
+
+ auto add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_FALSE(add_op.ok());
+
+ ASSERT_EQ(1, coll1->get_fields().size());
+ ASSERT_EQ(0, coll1->get_sort_fields().size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_numerical_index().size());
+
+ // now insert document with just "int_1" key
+ nlohmann::json doc2;
+ doc2["int_1"] = 200;
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ ASSERT_EQ(2, coll1->get_fields().size());
+ ASSERT_EQ(1, coll1->get_sort_fields().size());
+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().size());
+ ASSERT_EQ(1, coll1->_get_index()->_get_numerical_index().size());
+
+ collectionManager.drop_collection("coll1");
+}