Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Synonyms basics.

kishorenc88918ef95856
diff --git a/include/collection.h b/include/collection.h
index d381ea9b..f8111013 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -16,6 +16,7 @@
#include <json.hpp>
#include <field.h>
#include <option.h>
+#include "tokenizer.h"
struct override_t {
@@ -97,6 +98,88 @@ struct doc_seq_id_t {
bool is_new;
};
+struct synonym_t {
+ std::string id;
+ std::vector<std::string> root;
+ std::vector<std::vector<std::string>> synonyms;
+
+ synonym_t() = default;
+
+ synonym_t(const std::string& id, const std::vector<std::string>& root,
+ const std::vector<std::vector<std::string>>& synonyms):
+ id(id), root(root), synonyms(synonyms) {
+
+ }
+
+ explicit synonym_t(const nlohmann::json& synonym) {
+ id = synonym["id"].get<std::string>();
+ if(synonym.count("root") != 0) {
+ root = synonym["root"].get<std::vector<std::string>>();
+ }
+ synonyms = synonym["synonyms"].get<std::vector<std::vector<std::string>>>();
+ }
+
+ nlohmann::json to_json() const {
+ nlohmann::json obj;
+ obj["id"] = id;
+ obj["root"] = root;
+ obj["synonyms"] = synonyms;
+ return obj;
+ }
+
+ static Option<bool> parse(const nlohmann::json& synonym_json, synonym_t& syn) {
+ if(synonym_json.count("id") == 0) {
+ return Option<bool>(400, "Missing `id` field.");
+ }
+
+ if(synonym_json.count("synonyms") == 0) {
+ return Option<bool>(400, "Could not find an array of `synonyms`");
+ }
+
+ if(synonym_json.count("root") != 0 && !synonym_json["root"].is_string()) {
+ return Option<bool>(400, "Key `root` should be a string.");
+ }
+
+ if (!synonym_json["synonyms"].is_array() || synonym_json["synonyms"].empty()) {
+ return Option<bool>(400, "Could not find an array of `synonyms`");
+ }
+
+ for(const auto & synonym: synonym_json["synonyms"]) {
+ if(!synonym.is_string()) {
+ return Option<bool>(400, "Could not find a valid string array of `synonyms`");
+ }
+
+ std::vector<std::string> tokens;
+ Tokenizer(synonym, false).tokenize(tokens);
+ syn.synonyms.push_back(tokens);
+ }
+
+ if(synonym_json.count("root") != 0) {
+ std::vector<std::string> tokens;
+ Tokenizer(synonym_json["root"], false).tokenize(tokens);
+ syn.root = tokens;
+ }
+
+ syn.id = synonym_json["id"];
+ return Option<bool>(true);
+ }
+
+ static uint64_t get_hash(const std::vector<std::string>& tokens) {
+ uint64_t hash = 1;
+ for(size_t i=0; i < tokens.size(); i++) {
+ auto& token = tokens[i];
+ uint64_t token_hash = StringUtils::hash_wy(token.c_str(), token.size());
+ if(i == 0) {
+ hash = token_hash;
+ } else {
+ hash = Index::hash_combine(hash, token_hash);
+ }
+ }
+
+ return hash;
+ }
+};
+
class Collection {
private:
@@ -146,6 +229,9 @@ private:
std::unordered_map<std::string, field> sort_schema;
+ spp::sparse_hash_map<std::string, synonym_t> synonym_definitions;
+ spp::sparse_hash_map<uint64_t, std::string> synonym_index;
+
std::string default_sorting_field;
size_t num_memory_shards;
@@ -191,6 +277,12 @@ private:
Option<bool> parse_pinned_hits(const std::string& pinned_hits_str,
std::map<size_t, std::vector<std::string>>& pinned_hits);
+ void synonym_reduction_internal(const std::vector<std::string>& tokens,
+ size_t start_window_size,
+ size_t start_index_pos,
+ std::set<uint64_t>& processed_syn_hashes,
+ std::vector<std::vector<std::string>>& results);
+
public:
Collection() = delete;
@@ -206,6 +298,8 @@ public:
static std::string get_override_key(const std::string & collection_name, const std::string & override_id);
+ static std::string get_synonym_key(const std::string & collection_name, const std::string & synonym_id);
+
std::string get_seq_id_collection_prefix();
std::string get_name();
@@ -281,7 +375,7 @@ public:
Option<uint32_t> remove_override(const std::string & id);
- std::map<std::string, override_t> get_overrides() {
+ std::map<std::string, override_t>& get_overrides() {
return overrides;
};
@@ -310,6 +404,7 @@ public:
static constexpr const char* COLLECTION_META_PREFIX = "$CM";
static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";
+ static constexpr const char* COLLECTION_SYNONYM_PREFIX = "$CY";
static constexpr const char* SEQ_ID_PREFIX = "$SI";
static constexpr const char* DOC_ID_PREFIX = "$DI";
@@ -331,5 +426,13 @@ public:
void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::string>& q_exclude_tokens) const;
+
+ // synonym operations
+ void synonym_reduction(const std::vector<std::string>& tokens, std::vector<std::vector<std::string>>& results);
+
+ spp::sparse_hash_map<std::string, synonym_t>& get_synonyms();
+ bool get_synonym(const std::string& id, synonym_t& synonym);
+ Option<bool> add_synonym(const synonym_t& synonym);
+ Option<bool> remove_synonym(const std::string & id);
};
diff --git a/include/core_api.h b/include/core_api.h
index 41c7ceeb..1851385c 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -53,6 +53,16 @@ bool put_override(http_req& req, http_res& res);
bool del_override(http_req& req, http_res& res);
+// Synonyms
+
+bool get_synonyms(http_req& req, http_res& res);
+
+bool get_synonym(http_req& req, http_res& res);
+
+bool put_synonym(http_req& req, http_res& res);
+
+bool del_synonym(http_req& req, http_res& res);
+
// Keys
bool get_keys(http_req& req, http_res& res);
diff --git a/include/index.h b/include/index.h
index a0d12e7f..ecb1e229 100644
--- a/include/index.h
+++ b/include/index.h
@@ -26,6 +26,7 @@ struct token_candidates {
struct search_args {
std::vector<std::string> q_include_tokens;
std::vector<std::string> q_exclude_tokens;
+ std::vector<std::vector<std::string>> q_synonyms;
std::vector<std::string> search_fields;
std::vector<filter> filters;
std::vector<facet> facets;
@@ -58,13 +59,14 @@ struct search_args {
search_args(std::vector<std::string> q_include_tokens,
std::vector<std::string> q_exclude_tokens,
+ std::vector<std::vector<std::string>> q_synonyms,
std::vector<std::string> search_fields, std::vector<filter> filters,
std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,
std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,
size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,
size_t drop_tokens_threshold, size_t typo_tokens_threshold,
const std::vector<std::string>& group_by_fields, size_t group_limit):
- q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens),
+ q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens), q_synonyms(q_synonyms),
search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),
max_facet_values(max_facet_values), per_page(per_page),
@@ -204,12 +206,6 @@ private:
void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
- // reference: https://stackoverflow.com/a/27952689/131050
- uint64_t hash_combine(uint64_t lhs, uint64_t rhs) const {
- lhs ^= rhs + 0x517cc1b727220a95 + (lhs << 6) + (lhs >> 2);
- return lhs;
- }
-
public:
Index() = delete;
@@ -218,11 +214,18 @@ public:
~Index();
+ // reference: https://stackoverflow.com/a/27952689/131050
+ static uint64_t hash_combine(uint64_t combined, uint64_t hash) {
+ combined ^= hash + 0x517cc1b727220a95 + (combined << 6) + (combined >> 2);
+ return combined;
+ }
+
void run_search();
void search(Option<uint32_t> & outcome,
const std::vector<std::string> & q_include_tokens,
const std::vector<std::string> & q_exclude_tokens,
+ const std::vector<std::vector<std::string>>& q_synonyms,
const std::vector<std::string> & search_fields,
const std::vector<filter> & filters, std::vector<facet> & facets,
facet_query_t & facet_query,
diff --git a/include/option.h b/include/option.h
index 94b2d899..e837920b 100644
--- a/include/option.h
+++ b/include/option.h
@@ -9,11 +9,11 @@ private:
bool is_ok;
std::string error_msg;
- uint32_t error_code;
+ uint32_t error_code{};
public:
- Option(const T & value): value(value), is_ok(true) {
+ explicit Option(const T & value): value(value), is_ok(true) {
}
diff --git a/src/collection.cpp b/src/collection.cpp
index f1f79e97..57c053a0 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -694,7 +694,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
bool found_match_score = false;
- for(const auto & sort_field : sort_fields) {
+ for(const auto & sort_field : sort_fields_std) {
if(sort_field.name == sort_field_const::text_match) {
found_match_score = true;
break;
@@ -738,10 +738,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
parse_search_query(query, q_include_tokens, q_exclude_tokens);
+ // get synonyms
+ std::vector<std::vector<std::string>> q_synonyms;
+ synonym_reduction(q_include_tokens, q_synonyms);
+
// send data to individual index threads
size_t index_id = 0;
for(Index* index: indices) {
- index->search_params = new search_args(q_include_tokens, q_exclude_tokens, search_fields, filters, facets,
+ index->search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, search_fields, filters, facets,
index_to_included_ids[index_id], index_to_excluded_ids[index_id],
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
per_page, page, token_order, prefix,
@@ -1126,7 +1130,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";
//!store->print_memory_usage();
- return result;
+ return Option<nlohmann::json>(result);
}
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
@@ -1678,6 +1682,10 @@ std::string Collection::get_override_key(const std::string & collection_name, co
return std::string(COLLECTION_OVERRIDE_PREFIX) + "_" + collection_name + "_" + override_id;
}
+std::string Collection::get_synonym_key(const std::string & collection_name, const std::string & synonym_id) {
+ return std::string(COLLECTION_SYNONYM_PREFIX) + "_" + collection_name + "_" + synonym_id;
+}
+
std::string Collection::get_seq_id_collection_prefix() {
return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);
}
@@ -1881,13 +1889,11 @@ Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,
if(!StringUtils::is_positive_integer(pinned_pos)) {
return Option<bool>(false, "Pinned hits are not in expected format.");
- return false;
}
int position = std::stoi(pinned_pos);
if(position == 0) {
return Option<bool>(false, "Pinned hits must start from position 1.");
- return false;
}
pinned_hits[position].emplace_back(pinned_id);
@@ -1896,3 +1902,164 @@ Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,
return Option<bool>(true);
}
+
+void Collection::synonym_reduction(const std::vector<std::string>& tokens, std::vector<std::vector<std::string>>& results) {
+ std::set<uint64_t> processed_syn_hashes;
+ synonym_reduction_internal(tokens, tokens.size(), 0, processed_syn_hashes, results);
+}
+
+Option<bool> Collection::add_synonym(const synonym_t& synonym) {
+ if(synonym_definitions.count(synonym.id) != 0) {
+ return Option<bool>(409, "There is already another synonym with that `id`.");
+ }
+
+ synonym_definitions[synonym.id] = synonym;
+
+ if(!synonym.root.empty()) {
+ uint64_t root_hash = synonym_t::get_hash(synonym.root);
+ synonym_index[root_hash] = synonym.id;
+ } else {
+ for(const auto & syn_tokens : synonym.synonyms) {
+ uint64_t syn_hash = synonym_t::get_hash(syn_tokens);
+ synonym_index[syn_hash] = synonym.id;
+ }
+ }
+
+ bool inserted = store->insert(Collection::get_synonym_key(name, synonym.id), synonym.to_json().dump());
+ if(!inserted) {
+ return Option<bool>(500, "Error while storing the synonym on disk.");
+ }
+
+ return Option<bool>(true);
+}
+
+bool Collection::get_synonym(const std::string& id, synonym_t& synonym) {
+ if(synonym_definitions.count(id) != 0) {
+ synonym = synonym_definitions[id];
+ return true;
+ }
+
+ return false;
+}
+
+void Collection::synonym_reduction_internal(const std::vector<std::string>& tokens,
+ size_t start_window_size, size_t start_index_pos,
+ std::set<uint64_t>& processed_syn_hashes,
+ std::vector<std::vector<std::string>>& results) {
+
+ bool recursed = false;
+
+ for(size_t window_len = start_window_size; window_len > 0; window_len--) {
+ for(size_t start_index = start_index_pos; start_index+window_len-1 < tokens.size(); start_index++) {
+ std::vector<uint64_t> syn_hashes;
+ uint64_t syn_hash = 1;
+
+ for(size_t i = start_index; i < start_index+window_len; i++) {
+ uint64_t token_hash = StringUtils::hash_wy(tokens[i].c_str(), tokens[i].size());
+
+ if(i == start_index) {
+ syn_hash = token_hash;
+ } else {
+ syn_hash = Index::hash_combine(syn_hash, token_hash);
+ }
+
+ syn_hashes.push_back(token_hash);
+ }
+
+ const auto& syn_itr = synonym_index.find(syn_hash);
+
+ if(syn_itr != synonym_index.end() && processed_syn_hashes.count(syn_hash) == 0) {
+ // tokens in this window match a synonym: reconstruct tokens and rerun synonym mapping against matches
+ const auto& syn_id = syn_itr->second;
+ const auto& syn_def = synonym_definitions[syn_id];
+
+ for(const auto& syn_def_tokens: syn_def.synonyms) {
+ std::vector<std::string> new_tokens;
+
+ for(size_t i = 0; i < start_index; i++) {
+ new_tokens.push_back(tokens[i]);
+ }
+
+ std::vector<uint64_t> syn_def_hashes;
+ uint64_t syn_def_hash = 1;
+
+ for(size_t i=0; i < syn_def_tokens.size(); i++) {
+ const auto& syn_def_token = syn_def_tokens[i];
+ new_tokens.push_back(syn_def_token);
+ uint64_t token_hash = StringUtils::hash_wy(syn_def_token.c_str(),
+ syn_def_token.size());
+
+ if(i == 0) {
+ syn_def_hash = token_hash;
+ } else {
+ syn_def_hash = Index::hash_combine(syn_def_hash, token_hash);
+ }
+
+ syn_def_hashes.push_back(token_hash);
+ }
+
+ if(syn_def_hash == syn_hash) {
+ // skip over token matching itself in the group
+ continue;
+ }
+
+ for(size_t i = start_index+window_len; i < tokens.size(); i++) {
+ new_tokens.push_back(tokens[i]);
+ }
+
+ processed_syn_hashes.emplace(syn_def_hash);
+ processed_syn_hashes.emplace(syn_hash);
+
+ for(uint64_t h: syn_def_hashes) {
+ processed_syn_hashes.emplace(h);
+ }
+
+ for(uint64_t h: syn_hashes) {
+ processed_syn_hashes.emplace(h);
+ }
+
+ recursed = true;
+ synonym_reduction_internal(new_tokens, window_len, start_index, processed_syn_hashes, results);
+ }
+ }
+ }
+
+ // reset it because for the next window we have to start from scratch
+ start_index_pos = 0;
+ }
+
+ if(!recursed && !processed_syn_hashes.empty()) {
+ results.emplace_back(tokens);
+ }
+}
+
+Option<bool> Collection::remove_synonym(const std::string &id) {
+ const auto& syn_iter = synonym_definitions.find(id);
+
+ if(syn_iter != synonym_definitions.end()) {
+ bool removed = store->remove(Collection::get_synonym_key(name, id));
+ if(!removed) {
+ return Option<bool>(500, "Error while deleting the synonym from disk.");
+ }
+
+ const auto& synonym = syn_iter->second;
+ if(!synonym.root.empty()) {
+ uint64_t root_hash = synonym_t::get_hash(synonym.root);
+ synonym_index.erase(root_hash);
+ } else {
+ for(const auto & syn_tokens : synonym.synonyms) {
+ uint64_t syn_hash = synonym_t::get_hash(syn_tokens);
+ synonym_index.erase(syn_hash);
+ }
+ }
+
+ synonym_definitions.erase(id);
+ return Option<bool>(true);
+ }
+
+ return Option<bool>(404, "Could not find that `id`.");
+}
+
+spp::sparse_hash_map<std::string, synonym_t> &Collection::get_synonyms() {
+ return synonym_definitions;
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index dde7859b..2c31f6d8 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -140,6 +140,16 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
collection->add_override(override);
}
+ // initialize synonyms
+ std::vector<std::string> collection_synonym_jsons;
+ store->scan_fill(Collection::get_synonym_key(this_collection_name, ""), collection_synonym_jsons);
+
+ for(const auto & collection_synonym_json: collection_synonym_jsons) {
+ nlohmann::json collection_synonym = nlohmann::json::parse(collection_synonym_json);
+ synonym_t synonym(collection_synonym);
+ collection->add_synonym(synonym);
+ }
+
// Fetch records from the store and re-create memory index
std::vector<std::string> documents;
const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();
diff --git a/src/core_api.cpp b/src/core_api.cpp
index dcba0a2f..7270671d 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1041,7 +1041,7 @@ bool get_overrides(http_req &req, http_res &res) {
nlohmann::json res_json;
res_json["overrides"] = nlohmann::json::array();
- std::map<std::string, override_t> overrides = collection->get_overrides();
+ std::map<std::string, override_t>& overrides = collection->get_overrides();
for(const auto & kv: overrides) {
nlohmann::json override = kv.second.to_json();
res_json["overrides"].push_back(override);
@@ -1062,16 +1062,16 @@ bool get_override(http_req &req, http_res &res) {
std::string override_id = req.params["id"];
- std::map<std::string, override_t> overrides = collection->get_overrides();
+ std::map<std::string, override_t>& overrides = collection->get_overrides();
if(overrides.count(override_id) != 0) {
nlohmann::json override = overrides[override_id].to_json();
res.set_200(override.dump());
- return false;
+ return true;
}
res.set_404();
- return true;
+ return false;
}
bool put_override(http_req &req, http_res &res) {
@@ -1096,12 +1096,11 @@ bool put_override(http_req &req, http_res &res) {
}
// validate format of req_json
- if(
- !req_json.is_object() ||
- (req_json.count("rule") == 0) ||
- (req_json["rule"].count("query") == 0 || req_json["rule"].count("match") == 0) ||
- (req_json.count("includes") == 0 && req_json.count("excludes") == 0)
- ) {
+ if(!req_json.is_object() ||
+ (req_json.count("rule") == 0) ||
+ (req_json["rule"].count("query") == 0 || req_json["rule"].count("match") == 0) ||
+ (req_json.count("includes") == 0 && req_json.count("excludes") == 0)
+ ) {
res.set_400("Bad JSON.");
return false;
}
@@ -1289,3 +1288,113 @@ bool raft_write_send_response(void *data) {
return true;
}
+
+bool get_synonyms(http_req &req, http_res &res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection *collection = collectionManager.get_collection(req.params["collection"]);
+
+ if(collection == nullptr) {
+ res.set_404();
+ return false;
+ }
+
+ nlohmann::json res_json;
+ res_json["synonyms"] = nlohmann::json::array();
+
+ auto& synonyms = collection->get_synonyms();
+ for(const auto & kv: synonyms) {
+ nlohmann::json synonym = kv.second.to_json();
+ res_json["synonyms"].push_back(synonym);
+ }
+
+ res.set_200(res_json.dump());
+ return true;
+}
+
+bool get_synonym(http_req &req, http_res &res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection *collection = collectionManager.get_collection(req.params["collection"]);
+
+ if(collection == nullptr) {
+ res.set_404();
+ return false;
+ }
+
+ std::string synonym_id = req.params["id"];
+
+ synonym_t synonym;
+ bool found = collection->get_synonym(synonym_id, synonym);
+
+ if(found) {
+ nlohmann::json synonym_json = synonym.to_json();
+ res.set_200(synonym_json.dump());
+ return true;
+ }
+
+ res.set_404();
+ return false;
+}
+
+bool put_synonym(http_req &req, http_res &res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection *collection = collectionManager.get_collection(req.params["collection"]);
+
+ std::string synonym_id = req.params["id"];
+
+ if(collection == nullptr) {
+ res.set_404();
+ return false;
+ }
+
+ nlohmann::json syn_json;
+
+ try {
+ syn_json = nlohmann::json::parse(req.body);
+ } catch(const std::exception& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res.set_400("Bad JSON.");
+ return false;
+ }
+
+ syn_json["id"] = synonym_id;
+
+ synonym_t synonym;
+ Option<bool> syn_op = synonym_t::parse(syn_json, synonym);
+
+ if(!syn_op.ok()) {
+ res.set(syn_op.code(), syn_op.error());
+ return false;
+ }
+
+ Option<bool> upsert_op = collection->add_synonym(synonym);
+
+ if(!upsert_op.ok()) {
+ res.set(upsert_op.code(), upsert_op.error());
+ return false;
+ }
+
+ res.set_200(syn_json.dump());
+ return true;
+}
+
+bool del_synonym(http_req &req, http_res &res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection *collection = collectionManager.get_collection(req.params["collection"]);
+
+ if(collection == nullptr) {
+ res.set_404();
+ return false;
+ }
+
+ Option<bool> rem_op = collection->remove_synonym(req.params["id"]);
+ if(!rem_op.ok()) {
+ res.set(rem_op.code(), rem_op.error());
+ return false;
+ }
+
+ nlohmann::json res_json;
+ res_json["id"] = req.params["id"];
+
+ res.set_200(res_json.dump());
+ return true;
+}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index ae04e0a0..4932fadf 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -129,7 +129,7 @@ int HttpServer::create_listener() {
ctx.globalconf->server_name = h2o_strdup(nullptr, "", SIZE_MAX);
ctx.globalconf->http2.active_stream_window_size = ACTIVE_STREAM_WINDOW_SIZE;
ctx.globalconf->http2.idle_timeout = REQ_TIMEOUT_MS;
- ctx.globalconf->max_request_entity_size = (3 * 1024 * 1024 * 1024); // 3 GB
+ ctx.globalconf->max_request_entity_size = (size_t(3) * 1024 * 1024 * 1024); // 3 GB
ctx.globalconf->http1.req_timeout = REQ_TIMEOUT_MS;
ctx.globalconf->http1.req_io_timeout = REQ_TIMEOUT_MS;
diff --git a/src/index.cpp b/src/index.cpp
index ed18b108..b184b0fd 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1201,6 +1201,7 @@ void Index::run_search() {
// after the wait, we own the lock.
search(search_params->outcome, search_params->q_include_tokens, search_params->q_exclude_tokens,
+ search_params->q_synonyms,
search_params->search_fields,
search_params->filters, search_params->facets, search_params->facet_query,
search_params->included_ids, search_params->excluded_ids,
@@ -1275,11 +1276,12 @@ void Index::collate_included_ids(const std::vector<std::string>& q_included_toke
}
void Index::search(Option<uint32_t> & outcome,
- const std::vector<std::string> & q_include_tokens,
- const std::vector<std::string> & q_exclude_tokens,
- const std::vector<std::string> & search_fields,
- const std::vector<filter> & filters,
- std::vector<facet> & facets, facet_query_t & facet_query,
+ const std::vector<std::string>& q_include_tokens,
+ const std::vector<std::string>& q_exclude_tokens,
+ const std::vector<std::vector<std::string>>& q_synonyms,
+ const std::vector<std::string>& search_fields,
+ const std::vector<filter>& filters,
+ std::vector<facet>& facets, facet_query_t& facet_query,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
const std::vector<uint32_t> & excluded_ids,
const std::vector<sort_by> & sort_fields_std, const int num_typos,
@@ -1411,10 +1413,11 @@ void Index::search(Option<uint32_t> & outcome,
all_result_ids = filter_ids;
filter_ids = nullptr;
} else {
+ // non-wildcard
for(size_t i = 0; i < num_search_fields; i++) {
// proceed to query search only when no filters are provided or when filtering produces results
if(filters.empty() || filter_ids_length > 0) {
- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort results
+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - (2*i)); // Order of `fields` are used to sort results
const std::string & field = search_fields[i];
std::vector<std::string> query_tokens = q_include_tokens;
@@ -1427,9 +1430,23 @@ void Index::search(Option<uint32_t> & outcome,
field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,
token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
+
+ // do synonym based searches
+ for(const auto& syn_tokens: q_synonyms) {
+ num_tokens_dropped = 0;
+ query_tokens = search_tokens = syn_tokens;
+
+ // for synonym we use a smaller field id than for original tokens
+ search_field(field_id-1, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
+ field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
+ num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,
+ token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
+ }
+
collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
}
}
+
}
delete [] exclude_token_ids;
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 1b4e037b..b4d52def 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -36,6 +36,11 @@ void master_server_routes() {
server->put("/collections/:collection/overrides/:id", put_override);
server->del("/collections/:collection/overrides/:id", del_override);
+ server->get("/collections/:collection/synonyms", get_synonyms);
+ server->get("/collections/:collection/synonyms/:id", get_synonym);
+ server->put("/collections/:collection/synonyms/:id", put_synonym);
+ server->del("/collections/:collection/synonyms/:id", del_synonym);
+
server->get("/aliases", get_aliases);
server->get("/aliases/:alias", get_alias);
server->put("/aliases/:alias", put_upsert_alias);
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 61362579..04fa9a68 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -592,13 +592,13 @@ TEST_F(CollectionFilteringTest, FilterOnFloatFields) {
}
// when filters don't match any record, no results should be returned
- Option<nlohmann::json> results_op = coll_array_fields->search("Jeremy", query_fields, "rating:<-2.78", facets, sort_fields_desc, 0, 10, 1, FREQUENCY, false).get();
+ auto results_op = coll_array_fields->search("Jeremy", query_fields, "rating:<-2.78", facets, sort_fields_desc, 0, 10, 1, FREQUENCY, false);
ASSERT_TRUE(results_op.ok());
results = results_op.get();
ASSERT_EQ(0, results["hits"].size());
// rank tokens by default sorting field
- results_op = coll_array_fields->search("j", query_fields, "", facets, sort_fields_desc, 0, 10, 1, MAX_SCORE, true).get();
+ results_op = coll_array_fields->search("j", query_fields, "", facets, sort_fields_desc, 0, 10, 1, MAX_SCORE, true);
ASSERT_TRUE(results_op.ok());
results = results_op.get();
ASSERT_EQ(5, results["hits"].size());
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index da84b0d5..396dbddb 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -185,13 +185,13 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
override_t override_exclude(override_json);
nlohmann::json override_json_deleted = {
- {"id", "deleted-rule"},
- {
- "rule", {
- {"query", "of"},
- {"match", override_t::MATCH_EXACT}
- }
- }
+ {"id", "deleted-rule"},
+ {
+ "rule", {
+ {"query", "of"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ }
};
override_t override_deleted(override_json_deleted);
@@ -202,6 +202,17 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
collection1->remove_override("deleted-rule");
+ // make some synonym operation
+ synonym_t synonym1("id1", {"smart", "phone"}, {{"iphone"}});
+ synonym_t synonym2("id2", {"mobile", "phone"}, {{"samsung", "phone"}});
+ synonym_t synonym3("id3", {}, {{"football"}, {"foot", "ball"}});
+
+ collection1->add_synonym(synonym1);
+ collection1->add_synonym(synonym2);
+ collection1->add_synonym(synonym3);
+
+ collection1->remove_synonym("id2");
+
std::vector<std::string> search_fields = {"starring", "title"};
std::vector<std::string> facets;
@@ -244,6 +255,17 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
ASSERT_STREQ("exclude-rule", collection1->get_overrides()["exclude-rule"].id.c_str());
ASSERT_STREQ("include-rule", collection1->get_overrides()["include-rule"].id.c_str());
+ auto& synonyms = collection1->get_synonyms();
+ ASSERT_EQ(2, synonyms.size());
+
+ ASSERT_STREQ("id1", synonyms["id1"].id.c_str());
+ ASSERT_EQ(2, synonyms["id1"].root.size());
+ ASSERT_EQ(1, synonyms["id1"].synonyms.size());
+
+ ASSERT_STREQ("id3", synonyms["id3"].id.c_str());
+ ASSERT_EQ(0, synonyms["id3"].root.size());
+ ASSERT_EQ(2, synonyms["id3"].synonyms.size());
+
results = collection1->search("thomas", search_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(4, results["hits"].size());
}
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
new file mode 100644
index 00000000..539b1c6c
--- /dev/null
+++ b/test/collection_synonyms_test.cpp
@@ -0,0 +1,467 @@
+#include <gtest/gtest.h>
+#include <string>
+#include <vector>
+#include <fstream>
+#include <algorithm>
+#include <collection_manager.h>
+#include "collection.h"
+
+class CollectionSynonymsTest : public ::testing::Test {
+protected:
+ Store *store;
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection *coll_mul_fields;
+
+ void setupCollection() {
+ std::string state_dir_path = "/tmp/typesense_test/collection_override";
+ LOG(INFO) << "Truncating and creating: " << state_dir_path;
+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
+
+ store = new Store(state_dir_path);
+ collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.load();
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, true),
+ field("cast", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, false)
+ };
+
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ if(coll_mul_fields == nullptr) {
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_mul_fields->add(json_line);
+ }
+
+ infile.close();
+ }
+
+ virtual void SetUp() {
+ setupCollection();
+ }
+
+ virtual void TearDown() {
+ collectionManager.drop_collection("coll_mul_fields");
+ collectionManager.dispose();
+ delete store;
+ }
+};
+
+TEST_F(CollectionSynonymsTest, SynonymParsingFromJson) {
+ nlohmann::json syn_json = {
+ {"id", "syn-1"},
+ {"root", "Ocean"},
+ {"synonyms", {"Sea"} }
+ };
+
+ synonym_t synonym;
+ auto syn_op = synonym_t::parse(syn_json, synonym);
+ ASSERT_TRUE(syn_op.ok());
+
+ ASSERT_STREQ("syn-1", synonym.id.c_str());
+ ASSERT_STREQ("ocean", synonym.root[0].c_str());
+ ASSERT_STREQ("sea", synonym.synonyms[0][0].c_str());
+
+ // should accept without root
+ nlohmann::json syn_json_without_root = {
+ {"id", "syn-1"},
+ {"synonyms", {"Sea", "ocean"} }
+ };
+
+ syn_op = synonym_t::parse(syn_json_without_root, synonym);
+ ASSERT_TRUE(syn_op.ok());
+
+ // when `id` is not given
+ nlohmann::json syn_json_without_id = {
+ {"root", "Ocean"},
+ {"synonyms", {"Sea"} }
+ };
+
+ syn_op = synonym_t::parse(syn_json_without_id, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Missing `id` field.", syn_op.error().c_str());
+
+ // synonyms missing
+ nlohmann::json syn_json_without_synonyms = {
+ {"id", "syn-1"},
+ {"root", "Ocean"}
+ };
+
+ syn_op = synonym_t::parse(syn_json_without_synonyms, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Could not find an array of `synonyms`", syn_op.error().c_str());
+
+ // synonyms bad type
+
+ nlohmann::json syn_json_bad_type1 = {
+ {"id", "syn-1"},
+ {"root", "Ocean"},
+ {"synonyms", {"Sea", 1} }
+ };
+
+ syn_op = synonym_t::parse(syn_json_bad_type1, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Could not find a valid string array of `synonyms`", syn_op.error().c_str());
+
+ nlohmann::json syn_json_bad_type2 = {
+ {"id", "syn-1"},
+ {"root", "Ocean"},
+ {"synonyms", "foo" }
+ };
+
+ syn_op = synonym_t::parse(syn_json_bad_type2, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Could not find an array of `synonyms`", syn_op.error().c_str());
+
+ nlohmann::json syn_json_bad_type3 = {
+ {"id", "syn-1"},
+ {"root", "Ocean"},
+ {"synonyms", {} }
+ };
+
+ syn_op = synonym_t::parse(syn_json_bad_type3, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Could not find an array of `synonyms`", syn_op.error().c_str());
+
+ // root bad type
+
+ nlohmann::json syn_json_root_bad_type = {
+ {"id", "syn-1"},
+ {"root", 120},
+ {"synonyms", {"Sea"} }
+ };
+
+ syn_op = synonym_t::parse(syn_json_root_bad_type, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Key `root` should be a string.", syn_op.error().c_str());
+}
+
+TEST_F(CollectionSynonymsTest, SynonymReductionOneWay) {
+ std::vector<std::vector<std::string>> results;
+
+ synonym_t synonym1{"nyc-expansion", {"nyc"}, {{"new", "york"}} };
+ coll_mul_fields->add_synonym(synonym1);
+
+ results.clear();
+ coll_mul_fields->synonym_reduction({"red", "nyc", "tshirt"}, results);
+
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(4, results[0].size());
+
+ std::vector<std::string> red_new_york_tshirts = {"red", "new", "york", "tshirt"};
+ for(size_t i=0; i<red_new_york_tshirts.size(); i++) {
+ ASSERT_STREQ(red_new_york_tshirts[i].c_str(), results[0][i].c_str());
+ }
+
+ // when no synonyms exist, reduction should return nothing
+
+ results.clear();
+ coll_mul_fields->synonym_reduction({"foo", "bar", "baz"}, results);
+ ASSERT_EQ(0, results.size());
+
+ // compression and also ensure that it does not revert back to expansion rule
+
+ results.clear();
+ synonym_t synonym2{"new-york-compression", {"new", "york"}, {{"nyc"}} };
+ coll_mul_fields->add_synonym(synonym2);
+
+ coll_mul_fields->synonym_reduction({"red", "new", "york", "tshirt"}, results);
+
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(3, results[0].size());
+
+ std::vector<std::string> red_nyc_tshirts = {"red", "nyc", "tshirt"};
+ for(size_t i=0; i<red_nyc_tshirts.size(); i++) {
+ ASSERT_STREQ(red_nyc_tshirts[i].c_str(), results[0][i].c_str());
+ }
+
+ // replace two synonyms with the same length
+ results.clear();
+ synonym_t synonym3{"t-shirt-compression", {"t", "shirt"}, {{"tshirt"}} };
+ coll_mul_fields->add_synonym(synonym3);
+
+ coll_mul_fields->synonym_reduction({"new", "york", "t", "shirt"}, results);
+
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(2, results[0].size());
+
+ std::vector<std::string> nyc_tshirt = {"nyc", "tshirt"};
+ for(size_t i=0; i<nyc_tshirt.size(); i++) {
+ ASSERT_STREQ(nyc_tshirt[i].c_str(), results[0][i].c_str());
+ }
+
+ // replace two synonyms with different lengths
+ results.clear();
+ synonym_t synonym4{"red-crimson", {"red"}, {{"crimson"}} };
+ coll_mul_fields->add_synonym(synonym4);
+
+ coll_mul_fields->synonym_reduction({"red", "new", "york", "cap"}, results);
+
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(3, results[0].size());
+
+ std::vector<std::string> crimson_nyc_cap = {"crimson", "nyc", "cap"};
+ for(size_t i=0; i<crimson_nyc_cap.size(); i++) {
+ ASSERT_STREQ(crimson_nyc_cap[i].c_str(), results[0][i].c_str());
+ }
+}
+
+TEST_F(CollectionSynonymsTest, SynonymReductionMultiWay) {
+ synonym_t synonym1{"ipod-synonyms", {}, {{"ipod"}, {"i", "pod"}, {"pod"}} };
+ coll_mul_fields->add_synonym(synonym1);
+
+ std::vector<std::vector<std::string>> results;
+ coll_mul_fields->synonym_reduction({"ipod"}, results);
+
+ ASSERT_EQ(2, results.size());
+ ASSERT_EQ(2, results[0].size());
+ ASSERT_EQ(1, results[1].size());
+
+ std::vector<std::string> i_pod = {"i", "pod"};
+ for(size_t i=0; i<i_pod.size(); i++) {
+ ASSERT_STREQ(i_pod[i].c_str(), results[0][i].c_str());
+ }
+
+ ASSERT_STREQ("pod", results[1][0].c_str());
+
+ // multiple tokens
+ results.clear();
+ coll_mul_fields->synonym_reduction({"i", "pod"}, results);
+
+ ASSERT_EQ(2, results.size());
+ ASSERT_EQ(1, results[0].size());
+ ASSERT_EQ(1, results[1].size());
+
+ ASSERT_STREQ("ipod", results[0][0].c_str());
+ ASSERT_STREQ("pod", results[1][0].c_str());
+
+ // multi-token synonym + multi-token synonym definitions
+ synonym_t synonym2{"usa-synonyms", {}, {{"usa"}, {"united", "states"}, {"us"},
+ {"united", "states", "of", "america"}, {"states"}}};
+ coll_mul_fields->add_synonym(synonym2);
+
+ results.clear();
+ coll_mul_fields->synonym_reduction({"united", "states"}, results);
+ ASSERT_EQ(4, results.size());
+
+ ASSERT_EQ(1, results[0].size());
+ ASSERT_EQ(1, results[1].size());
+ ASSERT_EQ(4, results[2].size());
+ ASSERT_EQ(1, results[3].size());
+
+ ASSERT_STREQ("usa", results[0][0].c_str());
+ ASSERT_STREQ("us", results[1][0].c_str());
+
+ std::vector<std::string> red_new_york_tshirts = {"united", "states", "of", "america"};
+ for(size_t i=0; i<red_new_york_tshirts.size(); i++) {
+ ASSERT_STREQ(red_new_york_tshirts[i].c_str(), results[2][i].c_str());
+ }
+
+ ASSERT_STREQ("states", results[3][0].c_str());
+}
+
+TEST_F(CollectionSynonymsTest, OneWaySynonym) {
+ nlohmann::json syn_json = {
+ {"id", "syn-1"},
+ {"root", "Ocean"},
+ {"synonyms", {"Sea"} }
+ };
+
+ synonym_t synonym;
+ auto syn_op = synonym_t::parse(syn_json, synonym);
+ ASSERT_TRUE(syn_op.ok());
+
+ // without synonym
+
+ auto res = coll_mul_fields->search("ocean", {"title"}, "", {}, {}, 0, 10).get();
+ ASSERT_EQ(0, res["hits"].size());
+ ASSERT_EQ(0, res["found"].get<uint32_t>());
+
+ // add synonym and redo search
+
+ coll_mul_fields->add_synonym(synonym);
+
+ res = coll_mul_fields->search("ocean", {"title"}, "", {}, {}, 0, 10).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(1, res["found"].get<uint32_t>());
+}
+
+TEST_F(CollectionSynonymsTest, MultiWaySynonym) {
+ nlohmann::json syn_json = {
+ {"id", "syn-1"},
+ {"synonyms", {"Home Land", "Homeland", "homǝland"}}
+ };
+
+ synonym_t synonym;
+ auto syn_op = synonym_t::parse(syn_json, synonym);
+ ASSERT_TRUE(syn_op.ok());
+
+ // without synonym
+
+ auto res = coll_mul_fields->search("homǝland", {"title"}, "", {}, {}, 0, 10).get();
+ ASSERT_EQ(0, res["hits"].size());
+ ASSERT_EQ(0, res["found"].get<uint32_t>());
+
+ coll_mul_fields->add_synonym(synonym);
+
+ res = coll_mul_fields->search("homǝland", {"title"}, "", {}, {}, 0, 10).get();
+
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(1, res["found"].get<uint32_t>());
+ ASSERT_STREQ("<mark>Homeland</mark> Security", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ nlohmann::json syn_json2 = {
+ {"id", "syn-2"},
+ {"synonyms", {"Samuel L. Jackson", "Sam Jackson", "Leroy"}}
+ };
+
+ synonym_t synonym2;
+ syn_op = synonym_t::parse(syn_json2, synonym2);
+ ASSERT_TRUE(syn_op.ok());
+
+ res = coll_mul_fields->search("samuel leroy jackson", {"starring"}, "", {}, {}, 0, 10, 1, FREQUENCY, false, 0).get();
+ ASSERT_EQ(0, res["hits"].size());
+
+ coll_mul_fields->add_synonym(synonym2);
+
+ res = coll_mul_fields->search("samuel leroy jackson", {"starring"}, "", {}, {}, 0, 10).get();
+
+ ASSERT_EQ(2, res["hits"].size());
+ ASSERT_EQ(2, res["found"].get<uint32_t>());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
+}
+
+TEST_F(CollectionSynonymsTest, ExactMatchRankedHigherThanSynonymMatch) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("description", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Laughing out Loud", "Description 1", "100"},
+ {"Stop Laughing", "Description 2", "120"},
+ {"LOL sure", "Laughing out loud sure", "200"},
+ {"Really ROFL now", "Description 3", "250"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["description"] = records[i][1];
+ doc["points"] = std::stoi(records[i][2]);
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ nlohmann::json syn_json = {
+ {"id", "syn-1"},
+ {"synonyms", {"Lol", "ROFL", "laughing"}}
+ };
+
+ synonym_t synonym;
+ auto syn_op = synonym_t::parse(syn_json, synonym);
+ ASSERT_TRUE(syn_op.ok());
+
+ coll1->add_synonym(synonym);
+
+ auto res = coll1->search("laughing", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY, false, 0).get();
+
+ ASSERT_EQ(4, res["hits"].size());
+ ASSERT_EQ(4, res["found"].get<uint32_t>());
+
+ ASSERT_STREQ("1", res["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", res["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", res["hits"][2]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("2", res["hits"][3]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSynonymsTest, SynonymFieldOrdering) {
+ // Synonym match on a field earlier in the fields list should rank above exact match of another field
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("description", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+ std::vector<std::vector<std::string>> records = {
+ {"LOL really", "Description 1", "50"},
+ {"Never stop", "Description 2", "120"},
+ {"Yes and no", "Laughing out loud sure", "100"},
+ {"And so on", "Description 3", "250"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["description"] = records[i][1];
+ doc["points"] = std::stoi(records[i][2]);
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ nlohmann::json syn_json = {
+ {"id", "syn-1"},
+ {"synonyms", {"Lol", "ROFL", "laughing"}}
+ };
+
+ synonym_t synonym;
+ auto syn_op = synonym_t::parse(syn_json, synonym);
+ ASSERT_TRUE(syn_op.ok());
+
+ coll1->add_synonym(synonym);
+
+ auto res = coll1->search("laughing", {"title", "description"}, "", {}, {}, 0, 10, 1, FREQUENCY, false, 0).get();
+
+ ASSERT_EQ(2, res["hits"].size());
+ ASSERT_EQ(2, res["found"].get<uint32_t>());
+
+ ASSERT_STREQ("0", res["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("2", res["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSynonymsTest, DeleteAndUpsertDuplicationOfSynonms) {
+ synonym_t synonym1{"ipod-synonyms", {}, {{"ipod"}, {"i", "pod"}, {"pod"}}};
+ synonym_t synonym2{"samsung-synonyms", {}, {{"s3"}, {"s3", "phone"}, {"samsung"}}};
+ coll_mul_fields->add_synonym(synonym1);
+ coll_mul_fields->add_synonym(synonym2);
+
+ ASSERT_EQ(2, coll_mul_fields->get_synonyms().size());
+ coll_mul_fields->remove_synonym("ipod-synonyms");
+
+ ASSERT_EQ(1, coll_mul_fields->get_synonyms().size());
+ ASSERT_STREQ("samsung-synonyms", coll_mul_fields->get_synonyms()["samsung-synonyms"].id.c_str());
+
+ // try to upsert synonym with same ID
+ auto upsert_op = coll_mul_fields->add_synonym(synonym2);
+ ASSERT_FALSE(upsert_op.ok());
+ ASSERT_STREQ("There is already another synonym with that `id`.", upsert_op.error().c_str());
+
+ coll_mul_fields->remove_synonym("samsung-synonyms");
+ ASSERT_EQ(0, coll_mul_fields->get_synonyms().size());
+}