Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

refactor and adding tests

krunal1313a2796450aed9
diff --git a/include/collection.h b/include/collection.h
index 960e25df..1998f66c 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -391,7 +391,7 @@ public:
void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
- const std::string& locale, const bool already_segmented, const std::string& stopword) const;
+ const std::string& locale, const bool already_segmented, const std::string& stopword_set="") const;
// PUBLIC OPERATIONS
@@ -592,3 +592,4 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::js
}
}
+
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 7c355241..02b4dcbe 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -68,8 +68,6 @@ private:
spp::sparse_hash_map<std::string, nlohmann::json> preset_configs;
- spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> stopword_configs;
-
// Auto incrementing ID assigned to each collection
// Using a ID instead of a collection's name makes renaming possible
std::atomic<uint32_t> next_collection_id;
@@ -103,7 +101,6 @@ public:
static constexpr const char* SYMLINK_PREFIX = "$SL";
static constexpr const char* PRESET_PREFIX = "$PS";
static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";
- static constexpr const char* STOPWORD_PREFIX = "$SW";
static CollectionManager & get_instance() {
static CollectionManager instance;
@@ -204,15 +201,4 @@ public:
Option<bool> upsert_preset(const std::string & preset_name, const nlohmann::json& preset_config);
Option<bool> delete_preset(const std::string & preset_name);
-
- //stopwords
- spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> get_stopwords() const;
-
- Option<bool> get_stopword(const std::string&, spp::sparse_hash_set<std::string>&) const;
-
- Option<bool> upsert_stopword(const std::string&, const nlohmann::json&, const std::string&);
-
- Option<bool> delete_stopword(const std::string&);
-
- static std::string get_stopword_key(const std::string & stopword_name);
-};
\ No newline at end of file
+};
diff --git a/include/stopwords_manager.h b/include/stopwords_manager.h
new file mode 100644
index 00000000..90a461a9
--- /dev/null
+++ b/include/stopwords_manager.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include "sparsepp.h"
+#include "option.h"
+#include "json.hpp"
+#include "shared_mutex"
+#include "mutex"
+#include "store.h"
+
+class StopwordsManager{
+private:
+ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> stopword_configs;
+
+ static std::string get_stopword_key(const std::string & stopword_name);
+
+ mutable std::shared_mutex mutex;
+
+ Store* store = nullptr;
+public:
+ StopwordsManager() = default;
+
+ ~StopwordsManager() {
+ stopword_configs.clear();
+ }
+
+ static constexpr const char* STOPWORD_PREFIX = "$SW";
+
+ static StopwordsManager& get_instance() {
+ static StopwordsManager instance;
+ return instance;
+ }
+
+ void init(Store* store);
+
+ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> get_stopwords() const;
+
+ Option<bool> get_stopword(const std::string&, spp::sparse_hash_set<std::string>&) const;
+
+ Option<bool> upsert_stopword(const std::string&, const nlohmann::json&, const std::string&);
+
+ Option<bool> delete_stopword(const std::string&);
+};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 25d4a655..21d4086a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -18,6 +18,7 @@
#include "thread_local_vars.h"
#include "vector_query_ops.h"
#include "text_embedder_manager.h"
+#include "stopwords_manager.h"
const std::string override_t::MATCH_EXACT = "exact";
const std::string override_t::MATCH_CONTAINS = "contains";
@@ -2430,7 +2431,7 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
std::vector<std::string> tokens;
spp::sparse_hash_set<std::string> stopwords_list;
if(!stopwords_set.empty()) {
- const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopwords_set, stopwords_list);
+ const auto &stopword_op = StopwordsManager::get_instance().get_stopword(stopwords_set, stopwords_list);
if (!stopword_op.ok()) {
LOG(ERROR) << "Error fetching stopword_list for stopword " << stopwords_set << " "<<stopword_op.error();
}
@@ -4878,3 +4879,4 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t k
return Option<bool>(true);
}
+
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 0e8910cb..40a43983 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -316,7 +316,6 @@ void CollectionManager::dispose() {
collections.clear();
collection_symlinks.clear();
preset_configs.clear();
- stopword_configs.clear();
store->close();
}
@@ -754,7 +753,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
}
//check if stopword set is supplied
- const auto stopword_it = req_params.find("stopword");
+ const auto stopword_it = req_params.find("stopwords");
std::string stopwords_set="";
if(stopword_it != req_params.end()) {
@@ -1477,63 +1476,6 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {
return Option<bool>(true);
}
-
-spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> CollectionManager::get_stopwords() const {
- std::shared_lock lock(mutex);
- return stopword_configs;
-}
-
-Option<bool> CollectionManager::get_stopword(const string& stopword_name, spp::sparse_hash_set<std::string>& stopwords) const {
- std::shared_lock lock(mutex);
-
- const auto& it = stopword_configs.find(stopword_name);
- if(it != stopword_configs.end()) {
- stopwords = it->second;
- return Option<bool>(true);
- }
-
- return Option<bool>(404, "Not found.");
-}
-
-Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {
- std::unique_lock lock(mutex);
-
- bool inserted = store->insert(get_stopword_key(stopword_name), stopwords.dump());
- if(!inserted) {
- return Option<bool>(500, "Unable to insert into store.");
- }
-
- std::vector<std::string> tokens;
-
- for (const auto &stopword: stopwords.items()) {
- const auto& val = stopword.value().get<std::string>();
- Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);
-
- for(const auto& tok : tokens) {
- stopword_configs[stopword_name].emplace(tok);
- }
- tokens.clear();
- }
-
- return Option<bool>(true);
-}
-
-std::string CollectionManager::get_stopword_key(const string& stopword_name) {
- return std::string(STOPWORD_PREFIX) + "_" + stopword_name;
-}
-
-Option<bool> CollectionManager::delete_stopword(const string& stopword_name) {
- std::unique_lock lock(mutex);
-
- bool removed = store->remove(get_stopword_key(stopword_name));
- if(!removed) {
- return Option<bool>(500, "Unable to delete from store.");
- }
-
- stopword_configs.erase(stopword_name);
- return Option<bool>(true);
-}
-
Option<Collection*> CollectionManager::clone_collection(const string& existing_name, const nlohmann::json& req_json) {
std::shared_lock lock(mutex);
@@ -1593,3 +1535,4 @@ Option<Collection*> CollectionManager::clone_collection(const string& existing_n
return Option<Collection*>(new_coll);
}
+
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 80eada4c..a51f24c0 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -16,6 +16,7 @@
#include "ratelimit_manager.h"
#include "event_manager.h"
#include "http_proxy.h"
+#include "include/stopwords_manager.h"
using namespace std::chrono_literals;
@@ -1850,8 +1851,8 @@ bool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<http
}
bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
- CollectionManager & collectionManager = CollectionManager::get_instance();
- const spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>>& stopwords = collectionManager.get_stopwords();
+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();
+ const spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>>& stopwords = stopwordManager.get_stopwords();
nlohmann::json res_json = nlohmann::json::object();
res_json["stopwords"] = nlohmann::json::array();
@@ -1870,10 +1871,10 @@ bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<h
bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
const std::string & stopword_name = req->params["stopword_name"];
- CollectionManager & collectionManager = CollectionManager::get_instance();
+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();
spp::sparse_hash_set<std::string> stopwords;
- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);
+ Option<bool> stopword_op = stopwordManager.get_stopword(stopword_name, stopwords);
if(!stopword_op.ok()) {
res->set(stopword_op.code(), stopword_op.error());
@@ -1901,18 +1902,33 @@ bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::shared
return false;
}
- CollectionManager & collectionManager = CollectionManager::get_instance();
+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();
const std::string & stopword_name = req->params["name"];
const char* STOPWORD_VALUES = "stopwords";
const char* STOPWORD_LOCALE = "locale";
- if(req_json.count(STOPWORD_VALUES) == 0) {
- res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required.");
+ if(req_json.count(STOPWORD_VALUES) == 0){
+ res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required");
+ return false;
+ }
+
+ if((!req_json[STOPWORD_VALUES].is_array()) || (!req_json[STOPWORD_VALUES][0].is_string())) {
+ res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required as string array value");
+ return false;
+ }
+
+ if(req_json.count(STOPWORD_LOCALE) == 0) {
+ res->set_400(std::string("Parameter `") + STOPWORD_LOCALE + "` is required");
return false;
}
- Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUES], req_json[STOPWORD_LOCALE]);
+ if(!req_json[STOPWORD_LOCALE].is_string()) {
+ res->set_400(std::string("Parameter `") + STOPWORD_LOCALE + "` is required as string value");
+ return false;
+ }
+
+ Option<bool> success_op = stopwordManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUES], req_json[STOPWORD_LOCALE]);
if(!success_op.ok()) {
res->set_500(success_op.error());
return false;
@@ -1926,16 +1942,16 @@ bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::shared
bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
const std::string & stopword_name = req->params["name"];
- CollectionManager & collectionManager = CollectionManager::get_instance();
+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();
spp::sparse_hash_set<std::string> stopwords;
- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);
+ Option<bool> stopword_op = stopwordManager.get_stopword(stopword_name, stopwords);
if(!stopword_op.ok()) {
res->set(stopword_op.code(), stopword_op.error());
return false;
}
- Option<bool> delete_op = collectionManager.delete_stopword(stopword_name);
+ Option<bool> delete_op = stopwordManager.delete_stopword(stopword_name);
if(!delete_op.ok()) {
res->set_500(delete_op.error());
@@ -2256,4 +2272,4 @@ bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<http
res->set_200(response.body);
return true;
-}
\ No newline at end of file
+}
diff --git a/src/stopwords_manager.cpp b/src/stopwords_manager.cpp
new file mode 100644
index 00000000..a29f6c18
--- /dev/null
+++ b/src/stopwords_manager.cpp
@@ -0,0 +1,62 @@
+#include "include/stopwords_manager.h"
+#include "include/tokenizer.h"
+
+void StopwordsManager::init(Store* _store) {
+ store = _store;
+}
+
+spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> StopwordsManager::get_stopwords() const {
+ std::shared_lock lock(mutex);
+ return stopword_configs;
+}
+
+Option<bool> StopwordsManager::get_stopword(const std::string& stopword_name, spp::sparse_hash_set<std::string>& stopwords) const {
+ std::shared_lock lock(mutex);
+
+ const auto& it = stopword_configs.find(stopword_name);
+ if(it != stopword_configs.end()) {
+ stopwords = it->second;
+ return Option<bool>(true);
+ }
+
+ return Option<bool>(404, "Not found.");
+}
+
+Option<bool> StopwordsManager::upsert_stopword(const std::string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {
+ std::unique_lock lock(mutex);
+
+ bool inserted = store->insert(get_stopword_key(stopword_name), stopwords.dump());
+ if(!inserted) {
+ return Option<bool>(500, "Unable to insert into store.");
+ }
+
+ std::vector<std::string> tokens;
+
+ for (const auto &stopword: stopwords.items()) {
+ const auto& val = stopword.value().get<std::string>();
+ Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);
+
+ for(const auto& tok : tokens) {
+ stopword_configs[stopword_name].emplace(tok);
+ }
+ tokens.clear();
+ }
+
+ return Option<bool>(true);
+}
+
+std::string StopwordsManager::get_stopword_key(const std::string& stopword_name) {
+ return std::string(STOPWORD_PREFIX) + "_" + stopword_name;
+}
+
+Option<bool> StopwordsManager::delete_stopword(const std::string& stopword_name) {
+ std::unique_lock lock(mutex);
+
+ bool removed = store->remove(get_stopword_key(stopword_name));
+ if(!removed) {
+ return Option<bool>(500, "Unable to delete from store.");
+ }
+
+ stopword_configs.erase(stopword_name);
+ return Option<bool>(true);
+}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index c628c569..3c96e25b 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -21,6 +21,7 @@
#include "typesense_server_utils.h"
#include "file_utils.h"
#include "threadpool.h"
+#include "stopwords_manager.h"
#ifndef ASAN_BUILD
#include "jemalloc.h"
@@ -424,7 +425,10 @@ int run_server(const Config & config, const std::string & version, void (*master
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),
config.get_api_key(), quit_raft_service, batch_indexer);
-
+
+ StopwordsManager& stopwordsManager = StopwordsManager::get_instance();
+ stopwordsManager.init(&store);
+
RateLimitManager *rateLimitManager = RateLimitManager::getInstance();
auto rate_limit_manager_init = rateLimitManager->init(&meta_store);
@@ -523,3 +527,4 @@ int run_server(const Config & config, const std::string & version, void (*master
return ret_code;
}
+
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index 95af7546..9221dbce 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -4,11 +4,13 @@
#include <collection_manager.h>
#include <core_api.h>
#include "core_api_utils.h"
+#include "stopwords_manager.h"
class CoreAPIUtilsTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ StopwordsManager& stopwordsManager = StopwordsManager::get_instance();
std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
@@ -22,6 +24,8 @@ protected:
store = new Store(state_dir_path);
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
+
+ stopwordsManager.init(store);
}
virtual void SetUp() {
@@ -1116,4 +1120,202 @@ TEST_F(CoreAPIUtilsTest, TestProxyInvalid) {
ASSERT_EQ(400, resp->status_code);
ASSERT_EQ("Headers must be a JSON object.", nlohmann::json::parse(resp->body)["message"]);
-}
\ No newline at end of file
+}
+
+TEST_F(CoreAPIUtilsTest, StopwordsBasics) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string" },
+ {"name": "points", "type": "int32" }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll1 = op.get();
+
+ nlohmann::json doc;
+ doc["title"] = "The Dark Knight Europe";
+ doc["points"] = 10;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "An American America";
+ doc["points"] = 12;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "An the";
+ doc["points"] = 17;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "A Deadman";
+ doc["points"] = 13;
+ coll1->add(doc.dump(), CREATE);
+
+ doc["title"] = "A Village Of The Deadman";
+ doc["points"] = 20;
+ coll1->add(doc.dump(), CREATE);
+
+ //when all words in query are stopwords
+ auto stopword_value = R"(
+ {"stopwords": ["the", "a", "an"], "locale": "en"}
+ )"_json;
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+ req->params["collection"] = "coll1";
+ req->params["name"] = "articles";
+ req->body = stopword_value.dump();
+
+ auto result = put_upsert_stopword(req, res);
+ if(!result) {
+ LOG(ERROR) << res->body;
+ FAIL();
+ }
+
+ nlohmann::json body;
+
+ body["searches"] = nlohmann::json::array();
+ nlohmann::json search;
+ search["collection"] = "coll1";
+ search["q"] = "the";
+ search["query_by"] = "title";
+ body["searches"].push_back(search);
+
+ req->body = body.dump();
+ nlohmann::json embedded_params;
+ embedded_params["stopwords"] = "articles";
+ req->embedded_params_vec.push_back(embedded_params);
+
+ post_multi_search(req, res);
+ nlohmann::json results = nlohmann::json::parse(res->body)["results"][0];
+
+ ASSERT_EQ(0, results["hits"].size());
+
+ req->params.clear();
+ req->embedded_params_vec.clear();
+ body.clear();
+
+ //when not all words in query are stopwords then it should match the remaining words
+ stopword_value = R"(
+ {"stopwords": ["america", "europe"], "locale": "en"}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ if(!result) {
+ LOG(ERROR) << res->body;
+ FAIL();
+ }
+
+ body["searches"] = nlohmann::json::array();
+ search["collection"] = "coll1";
+ search["q"] = "America Man";
+ search["query_by"] = "title";
+ body["searches"].push_back(search);
+
+ req->body = body.dump();
+ embedded_params["stopwords"] = "continents";
+ req->embedded_params_vec.push_back(embedded_params);
+
+ post_multi_search(req, res);
+ results = nlohmann::json::parse(res->body)["results"][0];
+
+ ASSERT_EQ(0, results["hits"].size());
+
+ req->params.clear();
+ req->embedded_params_vec.clear();
+ body.clear();
+
+ req->params["collection"] = "coll1";
+
+ body["searches"] = nlohmann::json::array();
+ search["collection"] = "coll1";
+ search["q"] = "a deadman";
+ search["query_by"] = "title";
+ body["searches"].push_back(search);
+
+ req->body = body.dump();
+ embedded_params["stopwords"] = "articles";
+ req->embedded_params_vec.push_back(embedded_params);
+
+ post_multi_search(req, res);
+ results = nlohmann::json::parse(res->body)["results"][0];
+
+ ASSERT_EQ(2, results["hits"].size());
+
+ collectionManager.drop_collection("coll1");
+}
+
+
+TEST_F(CoreAPIUtilsTest, StopwordsValidation) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string" },
+ {"name": "points", "type": "int32" }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll1 = op.get();
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ auto stopword_value = R"(
+ {"stopwords": ["america", "europe"]}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ auto result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_STREQ("{\"message\": \"Parameter `locale` is required\"}", res->body.c_str());
+
+ //with a typo
+ stopword_value = R"(
+ {"stopword": ["america", "europe"], "locale": "en"}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required\"}", res->body.c_str());
+
+ //check for value types
+ stopword_value = R"(
+ {"stopwords": ["america", "europe"], "locale": 12}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_STREQ("{\"message\": \"Parameter `locale` is required as string value\"}", res->body.c_str());
+
+ stopword_value = R"(
+ {"stopwords": [1, 5, 2], "locale": "ko"}
+ )"_json;
+
+ req->params["collection"] = "coll1";
+ req->params["name"] = "continents";
+ req->body = stopword_value.dump();
+
+ result = put_upsert_stopword(req, res);
+ ASSERT_EQ(400, res->status_code);
+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required as string array value\"}", res->body.c_str());
+
+ collectionManager.drop_collection("coll1");
+}