Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
[Bug] String array field update + faceting.
Kishore Nallan908a8b0026a5
diff --git a/include/collection.h b/include/collection.hindex ebfacb57..065ba6f8 100644--- a/include/collection.h+++ b/include/collection.h@@ -296,7 +296,7 @@ public:static constexpr const char* SEQ_ID_PREFIX = "$SI";static constexpr const char* DOC_ID_PREFIX = "$DI";- void facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,+ bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,std::string &value);void aggregate_topster(size_t query_index, Topster &topster, Topster *index_topster) const;diff --git a/include/topster.h b/include/topster.hindex 2e585a6b..bad3ea30 100644--- a/include/topster.h+++ b/include/topster.h@@ -130,8 +130,6 @@ struct Topster {// erase current min heap key from kv_mapkv_map.erase(old_min_heap_kv.distinct_key);- // kv will be copied into the pointer at heap_op_index- kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);} else {// kv is guaranteed to be > current min heap: kvs[0]// create fresh topster for this distinct group key since it does not exist@@ -159,11 +157,11 @@ struct Topster {group_kv_map.erase(kvs[heap_op_index]->distinct_key);kv_map.erase(kvs[heap_op_index]->distinct_key);}-- // kv will be copied into the pointer at heap_op_index- kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);}+ // kv will be copied into the pointer at heap_op_index+ kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);+} else { // not distinct//LOG(INFO) << "Searching for key: " << kv->key;@@ -191,9 +189,6 @@ struct Topster {// replace existing kv and sift downheap_op_index = existing_kv->array_index;kv_map.erase(kvs[heap_op_index]->key);-- // kv will be copied into the pointer at heap_op_index- kv_map.emplace(kv->key, kvs[heap_op_index]);} else { // not duplicateif(size < MAX_SIZE) {@@ -208,10 +203,10 @@ struct Topster {heap_op_index = 0;kv_map.erase(kvs[heap_op_index]->key);}-- // kv will be copied into the pointer at heap_op_index- kv_map.emplace(kv->key, kvs[heap_op_index]);}++ // kv will be copied into the pointer at heap_op_index+ kv_map.emplace(kv->key, kvs[heap_op_index]);}// we have to replace the existing element in the heap and sift downdiff --git a/src/collection.cpp b/src/collection.cppindex aaa51843..78ff3069 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -224,7 +224,10 @@ void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::jsonfor(auto it = document.begin(); it != document.end(); ++it) {new_doc[it.key()] = it.value();- if(old_doc.count(it.key()) != 0) {++ // if the update document contains a field that exists in old, we record that (for delete + reindex)+ bool field_exists_in_old_doc = (old_doc.count(it.key()) != 0);+ if(field_exists_in_old_doc) {// key exists in the stored doc, so it must be reindexed// we need to check for this because a field can be optionaldel_doc[it.key()] = old_doc[it.key()];@@ -1173,7 +1176,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}std::string value;- facet_value_to_string(a_facet, facet_count, document, value);+ bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);++ if(!facet_found) {+ continue;+ }std::vector<std::string> tokens;StringUtils::split(value, tokens, " ");@@ -1281,9 +1288,27 @@ void Collection::aggregate_topster(size_t query_index, Topster &topster, Topster}}-void Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,+bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,const nlohmann::json &document, std::string &value) {+ if(document.count(a_facet.field_name) == 0) {+ LOG(ERROR) << "Could not find field " << a_facet.field_name << " in document during faceting.";+ LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;+ LOG(ERROR) << "Actual document: " << document;+ return false;+ }++ if(facet_schema.at(a_facet.field_name).is_array()) {+ size_t array_sz = document[a_facet.field_name].size();+ if(facet_count.array_pos >= array_sz) {+ LOG(ERROR) << "Facet field array size " << array_sz << " lesser than array pos " << facet_count.array_pos+ << " for facet field " << a_facet.field_name;+ LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;+ LOG(ERROR) << "Actual document: " << document;+ return false;+ }+ }+if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {value = document[a_facet.field_name];} else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {@@ -1315,6 +1340,8 @@ void Collection::facet_value_to_string(const facet &a_facet, const facet_count_tvalue = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());value = (value == "1") ? "true" : "false";}++ return true;}void Collection::highlight_result(const field &search_field,diff --git a/src/index.cpp b/src/index.cppindex 7de60861..4efa5575 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -104,11 +104,12 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t// initialize facet index since it will be updated as well during search indexing// even if a field is optional, a facet position will be available in the vector for that field+ // NOTE: Use of `emplace()` means that we will not replace existing facet values.std::vector<std::vector<uint64_t>> values(facet_schema.size());facet_index_v2.emplace(seq_id, values);// assumes that validation has already been done- for(const std::pair<std::string, field> & field_pair: search_schema) {+ for(const auto& field_pair: search_schema) {const std::string & field_name = field_pair.first;if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {@@ -239,8 +240,8 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,return Option<>(400, "Default sorting field `" + default_sorting_field + "` exceeds maximum value of a float.");}- for(const std::pair<std::string, field> & field_pair: search_schema) {- const std::string & field_name = field_pair.first;+ for(const auto& field_pair: search_schema) {+ const std::string& field_name = field_pair.first;if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {continue;@@ -357,6 +358,7 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_doif(exact_match) {it = del_doc.erase(it);+ update_doc.erase(field_name);} else {++it;}@@ -836,7 +838,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,if(query_token_positions.find(qtoken_pos.pos) == query_token_positions.end() ||query_token_positions[qtoken_pos.pos].cost >= qtoken_pos.cost ) {token_pos_cost_t ftoken_pos_cost = {field_token_index, qtoken_pos.cost};- query_token_positions.emplace(qtoken_pos.pos, ftoken_pos_cost);+ query_token_positions[qtoken_pos.pos] = ftoken_pos_cost;}}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex efdc33b8..4d9e6f97 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1330,7 +1330,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {std::vector<field> fields = {field("title", field_types::STRING, false),- field("starring", field_types::STRING, false),+ field("starring", field_types::STRING, true),field("cast", field_types::STRING_ARRAY, false),field("points", field_types::INT32, false)};@@ -1365,17 +1365,17 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {ASSERT_EQ(1, import_results[i].size());}- auto results = coll_mul_fields->search("*", query_fields, "", {}, sort_fields, 0, 30, 1, FREQUENCY, false).get();+ auto results = coll_mul_fields->search("*", query_fields, "", {"starring"}, sort_fields, 0, 30, 1, FREQUENCY, false).get();ASSERT_EQ(19, results["hits"].size());ASSERT_EQ(19, coll_mul_fields->get_num_documents());- results = coll_mul_fields->search("back again forest", query_fields, "", {}, sort_fields, 0, 30, 1, FREQUENCY, false).get();+ results = coll_mul_fields->search("back again forest", query_fields, "", {"starring"}, sort_fields, 0, 30, 1, FREQUENCY, false).get();ASSERT_EQ(1, results["hits"].size());ASSERT_STREQ("Back Again Forest", coll_mul_fields->get("18").get()["title"].get<std::string>().c_str());- results = coll_mul_fields->search("fifth", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ results = coll_mul_fields->search("fifth", query_fields, "", {"starring"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(2, results["hits"].size());ASSERT_STREQ("The <mark>Fifth</mark> Harry", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());@@ -1432,7 +1432,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {ASSERT_TRUE(import_response["success"].get<bool>());ASSERT_EQ(1, import_response["num_imported"].get<int>());- results = coll_mul_fields->search("Good Will Hunting", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ results = coll_mul_fields->search("Good Will Hunting", query_fields, "", {"starring"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(70, results["hits"][0]["document"]["points"].get<uint32_t>());// updating a document that does not exist should fail, others should succeed@@ -1449,7 +1449,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {ASSERT_STREQ("Could not find a document with id: 20", import_results[0]["error"].get<std::string>().c_str());ASSERT_EQ(404, import_results[0]["code"].get<size_t>());- results = coll_mul_fields->search("wake up harry", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ results = coll_mul_fields->search("wake up harry", query_fields, "", {"starring"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(64, results["hits"][0]["document"]["points"].get<uint32_t>());// trying to create documents with existing IDs should fail@@ -2510,7 +2510,7 @@ TEST_F(CollectionTest, UpdateDocument) {auto add_op = coll1->add(doc.dump());ASSERT_TRUE(add_op.ok());- auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ auto res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();@@ -2518,6 +2518,25 @@ TEST_F(CollectionTest, UpdateDocument) {ASSERT_STREQ("The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.",res["hits"][0]["document"]["title"].get<std::string>().c_str());+ // reindex the document entirely again verbatim and try querying+ add_op = coll1->add(doc.dump(), UPSERT);+ ASSERT_TRUE(add_op.ok());++ res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();++ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(1, res["facet_counts"].size());+ ASSERT_STREQ("tags", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(2, res["facet_counts"][0]["counts"].size());++ ASSERT_STREQ("NEWS", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][0]["count"]);++ ASSERT_STREQ("LAZY", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][1]["count"]);+// try changing the title and searching for an older tokendoc["title"] = "The quick brown fox.";add_op = coll1->add(doc.dump(), UPSERT);@@ -2525,13 +2544,13 @@ TEST_F(CollectionTest, UpdateDocument) {ASSERT_EQ(1, coll1->get_num_documents());- res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();ASSERT_EQ(0, res["hits"].size());- res = coll1->search("quick", {"title"}, "", {}, sort_fields, 0, 10, 1,+ res = coll1->search("quick", {"title"}, "", {"title"}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();@@ -2551,7 +2570,7 @@ TEST_F(CollectionTest, UpdateDocument) {ASSERT_TRUE(add_op.ok());// check for old tag- res = coll1->search("NEWS", {"tags"}, "", {}, sort_fields, 0, 10, 1,+ res = coll1->search("NEWS", {"tags"}, "", {"tags"}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();@@ -2573,7 +2592,7 @@ TEST_F(CollectionTest, UpdateDocument) {add_op = coll1->add(doc3.dump(), UPDATE);ASSERT_TRUE(add_op.ok());- res = coll1->search("*", {"tags"}, "points: > 90", {}, sort_fields, 0, 10, 1,+ res = coll1->search("*", {"tags"}, "points: > 90", {"tags"}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();@@ -2587,7 +2606,7 @@ TEST_F(CollectionTest, UpdateDocument) {add_op = coll1->add(doc4.dump(), UPSERT, "100");ASSERT_TRUE(add_op.ok());- res = coll1->search("*", {"tags"}, "points: > 101", {}, sort_fields, 0, 10, 1,+ res = coll1->search("*", {"tags"}, "points: > 101", {"tags"}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();@@ -2599,7 +2618,7 @@ TEST_F(CollectionTest, UpdateDocument) {add_op = coll1->add(doc4.dump(), UPSERT, "100");ASSERT_FALSE(add_op.ok());- res = coll1->search("*", {"tags"}, "points: > 101", {}, sort_fields, 0, 10, 1,+ res = coll1->search("*", {"tags"}, "points: > 101", {"tags"}, sort_fields, 0, 10, 1,token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();@@ -2629,6 +2648,66 @@ TEST_F(CollectionTest, UpdateDocument) {ASSERT_FALSE(add_op.ok());ASSERT_EQ(400, add_op.code());ASSERT_STREQ("The `id` should not be empty.", add_op.error().c_str());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionTest, UpdateDocumentSorting) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field("tags", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, false)};++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1");+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "100";+ doc1["title"] = "The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.";+ doc1["tags"] = {"NEWS", "LAZY"};+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "101";+ doc2["title"] = "The random sentence.";+ doc2["tags"] = {"RANDOM"};+ doc2["points"] = 101;++ auto add_op = coll1->add(doc1.dump());+ coll1->add(doc2.dump());++ auto res = coll1->search("*", {"tags"}, "", {"tags"}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();++ ASSERT_EQ(2, res["hits"].size());+ ASSERT_EQ(101, res["hits"][0]["document"]["points"].get<size_t>());+ ASSERT_STREQ("101", res["hits"][0]["document"]["id"].get<std::string>().c_str());++ ASSERT_EQ(100, res["hits"][1]["document"]["points"].get<size_t>());+ ASSERT_STREQ("100", res["hits"][1]["document"]["id"].get<std::string>().c_str());++ // now update doc1 points from 100 -> 1000 and it should bubble up+ doc1["points"] = 1000;+ coll1->add(doc1.dump(), UPDATE);++ res = coll1->search("*", {"tags"}, "", {"tags"}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();++ ASSERT_EQ(2, res["hits"].size());+ ASSERT_EQ(1000, res["hits"][0]["document"]["points"].get<size_t>());+ ASSERT_STREQ("100", res["hits"][0]["document"]["id"].get<std::string>().c_str());++ ASSERT_EQ(101, res["hits"][1]["document"]["points"].get<size_t>());+ ASSERT_STREQ("101", res["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");}TEST_F(CollectionTest, SearchHighlightFieldFully) {