Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix synonym upserts.
Kishore Nallanf47c09fd63bf
diff --git a/src/synonym_index.cpp b/src/synonym_index.cppindex 0e8c70b9..d41ddf00 100644--- a/src/synonym_index.cpp+++ b/src/synonym_index.cpp@@ -30,37 +30,55 @@ void SynonymIndex::synonym_reduction_internal(const std::vector<std::string>& toif(processed_tokens.count(merged_tokens_str) == 0) {// tokens in this window match a synonym: reconstruct tokens and rerun synonym mapping against matchesfor (const auto &leaf: leaves) {- for (auto index = 0; index < synonym_index; ++index) {- if (posting_t::contains(leaf->values, index)) {- const auto &syn_def = synonym_definitions.at(index);-- for (const auto &syn_def_tokens: syn_def.synonyms) {- std::vector<std::string> new_tokens;-- for (size_t i = 0; i < start_index; i++) {- new_tokens.push_back(tokens[i]);- }-- for (size_t i = 0; i < syn_def_tokens.size(); i++) {- const auto &syn_def_token = syn_def_tokens[i];- new_tokens.push_back(syn_def_token);- processed_tokens.emplace(syn_def_token);- }-- for (size_t i = start_index + window_len; i < tokens.size(); i++) {- new_tokens.push_back(tokens[i]);- }-- processed_tokens.emplace(merged_tokens_str);- auto syn_def_tokens_str = StringUtils::join(syn_def_tokens, " ");- processed_tokens.emplace(syn_def_tokens_str);-- recursed = true;- synonym_reduction_internal(new_tokens, window_len,- start_index, processed_tokens, results, orig_tokens,- synonym_prefix, synonym_num_typos);+ std::vector<posting_list_t*> expanded_plists;+ posting_list_t::iterator_t it(nullptr, nullptr, nullptr);++ if(IS_COMPACT_POSTING(leaf->values)) {+ auto compact_posting_list = COMPACT_POSTING_PTR(leaf->values);+ posting_list_t* full_posting_list = compact_posting_list->to_full_posting_list();+ expanded_plists.push_back(full_posting_list);+ it = full_posting_list->new_iterator(nullptr, nullptr, 0);+ } else {+ posting_list_t* full_posting_list = (posting_list_t*)(leaf->values);+ it = full_posting_list->new_iterator(nullptr, nullptr, 0);+ }++ while(it.valid()) {+ auto syn_index = it.id();+ const auto &syn_def = synonym_definitions.at(syn_index);++ for (const auto &syn_def_tokens: syn_def.synonyms) {+ std::vector<std::string> new_tokens;++ for (size_t i = 0; i < start_index; i++) {+ new_tokens.push_back(tokens[i]);}++ for (size_t i = 0; i < syn_def_tokens.size(); i++) {+ const auto &syn_def_token = syn_def_tokens[i];+ new_tokens.push_back(syn_def_token);+ processed_tokens.emplace(syn_def_token);+ }++ for (size_t i = start_index + window_len; i < tokens.size(); i++) {+ new_tokens.push_back(tokens[i]);+ }++ processed_tokens.emplace(merged_tokens_str);+ auto syn_def_tokens_str = StringUtils::join(syn_def_tokens, " ");+ processed_tokens.emplace(syn_def_tokens_str);++ recursed = true;+ synonym_reduction_internal(new_tokens, window_len,+ start_index, processed_tokens, results, orig_tokens,+ synonym_prefix, synonym_num_typos);}++ it.next();+ }++ for(posting_list_t* plist: expanded_plists) {+ delete plist;}}}@@ -164,16 +182,18 @@ Option<bool> SynonymIndex::remove_synonym(const std::string & collection_name, cconst auto& synonym = synonym_definitions.at(syn_iter->second);std::vector<std::string> keys;- if(!synonym.root.empty()) {- keys.insert(keys.end(), synonym.root.begin(), synonym.root.end());- } else {- for(const auto & syn_tokens : synonym.synonyms) {- keys.insert(keys.end(), syn_tokens.begin(), syn_tokens.end());- }- }+ keys.insert(keys.end(), synonym.root.begin(), synonym.root.end());+ keys.insert(keys.end(), synonym.raw_synonyms.begin(), synonym.raw_synonyms.end());for(const auto& key : keys) {- art_delete(synonym_index_tree, (unsigned char*)key.c_str(), key.size() + 1);+ art_leaf* found_leaf = (art_leaf *) art_search(synonym_index_tree, (unsigned char *) key.c_str(), key.size() + 1);+ if(found_leaf) {+ auto index = syn_iter->second;+ posting_t::erase(found_leaf->values, index);+ if(posting_t::num_ids(found_leaf->values) == 0) {+ art_delete(synonym_index_tree, (unsigned char*)key.c_str(), key.size() + 1);+ }+ }}auto index = synonym_ids_index_map.at(id);diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex 071c4f09..bca96311 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -725,6 +725,33 @@ TEST_F(CollectionSynonymsTest, DeleteAndUpsertDuplicationOfSynonms) {ASSERT_EQ(0, coll_mul_fields->get_synonyms().get().size());}+TEST_F(CollectionSynonymsTest, UpsertAndSearch) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false)};+ auto coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc;+ doc["title"] = "Rose gold rosenblade, 500 stk";+ doc["points"] = 0;+ coll1->add(doc.dump());++ coll1->add_synonym(R"({"id":"abcde","locale":"da","root":"",+ "synonyms":["rosegold","rosaguld","rosa guld","rose gold","roseguld","rose guld"]})"_json);++ ASSERT_EQ(1, coll1->get_synonyms().get().size());++ // try to upsert synonym with same ID+ auto upsert_op = coll1->add_synonym(R"({"id":"abcde","locale":"da","root":"",+ "synonyms":["rosegold","rosaguld","rosa guld","rose gold","roseguld","rose guld"]})"_json);+ ASSERT_TRUE(upsert_op.ok());+ ASSERT_EQ(1, coll1->get_synonyms().get().size());++ // now try searching+ auto res = coll1->search("rosa guld", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(1, res["found"].get<uint32_t>());+}+TEST_F(CollectionSynonymsTest, SynonymJsonSerialization) {synonym_t synonym1;synonym1.id = "ipod-synonyms";