Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add stemming support for fields (#1498)
Ozan Armağan0f54bf71010a
diff --git a/BUILD b/BUILDindex 37e75672..27fdf254 100644--- a/BUILD+++ b/BUILD@@ -62,6 +62,8 @@ cc_library("@clip_tokenizer//:clip","@whisper.cpp//:whisper","@whisper.cpp//:whisper_headers",+ "@snowball",+ "@snowball//:snowball_headers",# "@zip",])diff --git a/WORKSPACE b/WORKSPACEindex df35a0d1..d090cccf 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -336,4 +336,11 @@ http_archive()load("@rules_cuda//cuda:repositories.bzl", "register_detected_cuda_toolchains", "rules_cuda_dependencies")rules_cuda_dependencies()-register_detected_cuda_toolchains()\ No newline at end of file+register_detected_cuda_toolchains()++new_git_repository(+ name= "snowball",+ build_file = "//bazel:snowball.BUILD",+ branch = "master",+ remote = "https://github.com/snowballstem/snowball.git"+)\ No newline at end of filediff --git a/bazel/snowball.BUILD b/bazel/snowball.BUILDnew file mode 100644index 00000000..975d5e95--- /dev/null+++ b/bazel/snowball.BUILD@@ -0,0 +1,33 @@+load("@rules_foreign_cc//foreign_cc:defs.bzl", "make")++package(default_visibility = ["//visibility:public"])+++filegroup(+ name = "snowball_src",+ srcs = glob(["**"]),+ visibility = ["//visibility:public"],+)++cc_library(+ name = "snowball_headers",+ hdrs = glob(["include/**/*.h"]),+ includes = ["include"],+ visibility = ["//visibility:public"],+ strip_include_prefix = "include"+)+++make(+ name = "snowball",+ lib_source = "//:snowball_src",+ out_static_libs = ["libstemmer.a"],+ out_include_dir = "include",+ args= ["-j12"],+ tags = ["no-sandbox"],+ targets = [""],+ postfix_script= """+ echo "Installing snowball"+ cp $BUILD_TMPDIR/libstemmer.a $INSTALLDIR/lib/libstemmer.a+ """+)diff --git a/include/field.h b/include/field.hindex 20c34427..49f35f84 100644--- a/include/field.h+++ b/include/field.h@@ -12,6 +12,8 @@#include "json.hpp"#include "embedder_manager.h"#include "vector_query_ops.h"+#include <mutex>+#include "stemmer_manager.h"namespace field_types {// first field value indexed will determine the type@@ -62,6 +64,7 @@ namespace fields {static const std::string from = "from";static const std::string model_name = "model_name";static const std::string range_index = "range_index";+ static const std::string stem = "stem";// Some models require additional parameters to be passed to the model during indexing/querying// For e.g. e5-small model requires prefix "passage:" for indexing and "query:" for querying@@ -112,20 +115,26 @@ struct field {bool is_reference_helper = false;+ bool stem = false;+ std::shared_ptr<Stemmer> stemmer;+field() {}field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine,- std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true) :+ std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true, const bool stem = false) :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference),- embed(embed), range_index(range_index), store(store) {+ embed(embed), range_index(range_index), store(store), stem(stem) {set_computed_defaults(sort, infix);auto const suffix = std::string(fields::REFERENCE_HELPER_FIELD_SUFFIX);is_reference_helper = name.size() > suffix.size() && name.substr(name.size() - suffix.size()) == suffix;+ if (stem) {+ stemmer = StemmerManager::get_instance().get_stemmer(locale);+ }}void set_computed_defaults(int sort, int infix) {@@ -256,6 +265,10 @@ struct field {return is_num_sortable() || is_str_sortable();}+ bool is_stem() const {+ return stem;+ }+bool has_valid_type() const {bool is_basic_type = is_string() || is_integer() || is_float() || is_bool() || is_geopoint() ||is_object() || is_auto() || is_image();@@ -269,6 +282,10 @@ struct field {return (facet && !is_string()) ? "_fstr_" + name : name;}+ std::shared_ptr<Stemmer> get_stemmer() const {+ return stemmer;+ }+static bool get_type(const nlohmann::json& obj, std::string& field_type) {if(obj.is_array()) {if(obj.empty()) {diff --git a/include/stemmer_manager.h b/include/stemmer_manager.hnew file mode 100644index 00000000..9f63e710--- /dev/null+++ b/include/stemmer_manager.h@@ -0,0 +1,43 @@+#pragma once++#include <unordered_map>+#include <string>+#include <vector>+#include <memory>+#include <mutex>+#include <libstemmer.h>+#include "lru/lru.hpp"+++class Stemmer {+ private:+ sb_stemmer * stemmer = nullptr;+ LRU::Cache<std::string, std::string> cache;+ std::mutex mutex;+ public:+ Stemmer(const char * language);+ ~Stemmer();+ std::string stem(const std::string & word);+};+++class StemmerManager {+ private:+ std::unordered_map<std::string, std::shared_ptr<Stemmer>> stemmers;+ StemmerManager() {}+ std::mutex mutex;+ public:+ static StemmerManager& get_instance() {+ static StemmerManager instance;+ return instance;+ }+ StemmerManager(StemmerManager const&) = delete;+ void operator=(StemmerManager const&) = delete;+ StemmerManager(StemmerManager&&) = delete;+ void operator=(StemmerManager&&) = delete;+ ~StemmerManager();+ std::shared_ptr<Stemmer> get_stemmer(const std::string& language);+ void delete_stemmer(const std::string& language);+ void delete_all_stemmers();+ const bool validate_language(const std::string& language);+};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 80623c24..ac13d4b7 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -431,6 +431,7 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::sort] = coll_field.sort;field_json[fields::infix] = coll_field.infix;field_json[fields::locale] = coll_field.locale;+ field_json[fields::stem] = coll_field.stem;if(coll_field.embed.count(fields::from) != 0) {field_json[fields::embed] = coll_field.embed;diff --git a/src/field.cpp b/src/field.cppindex 28b67aef..cc2e4e56 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -6,6 +6,7 @@#include <collection_manager.h>#include <regex>+Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,std::vector<field>& the_fields,string& fallback_field_type, size_t& num_auto_detect_fields) {@@ -80,6 +81,31 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::reference] = "";}+ if(field_json.count(fields::stem) != 0) {+ if(!field_json.at(fields::stem).is_boolean()) {+ return Option<bool>(400, std::string("The `stem` property of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));+ }++ if(field_json[fields::stem] && field_json[fields::type] != field_types::STRING && field_json[fields::type] != field_types::STRING_ARRAY) {+ return Option<bool>(400, std::string("The `stem` property is only allowed for string and string[] fields."));+ }++ if(field_json[fields::stem].get<bool>()) {+ std::string locale;+ if(field_json.count(fields::locale) != 0) {+ locale = field_json[fields::locale].get<std::string>();+ }+ auto stem_validation = StemmerManager::get_instance().validate_language(locale);+ if(!stem_validation) {+ return Option<bool>(400, std::string("The `locale` value of the field `") ++ field_json[fields::name].get<std::string>() + std::string("` is not supported for stem."));+ }+ }+ } else {+ field_json[fields::stem] = false;+ }+if (field_json.count(fields::range_index) != 0) {if (!field_json.at(fields::range_index).is_boolean()) {return Option<bool>(400, std::string("The `range_index` property of the field `") +@@ -338,7 +364,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::optional], field_json[fields::index], field_json[fields::locale],field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,- field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index], field_json[fields::store])+ field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index], field_json[fields::store], field_json[fields::stem]));if (!field_json[fields::reference].get<std::string>().empty()) {diff --git a/src/index.cpp b/src/index.cppindex 4a67afe6..235f46b6 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -547,7 +547,8 @@ void Index::validate_and_preprocess(Index *index,}}-size_t Index::batch_memory_index(Index *index,+size_t Index::+batch_memory_index(Index *index,std::vector<index_record>& iter_batch,const std::string & default_sorting_field,const tsl::htrie_map<char, field> & actual_search_schema,@@ -1160,6 +1161,15 @@ void Index::tokenize_string(const std::string& text, const field& a_field,if(token.size() > 100) {token.erase(100);}++ if(a_field.is_stem()) {+ auto stemmer = a_field.get_stemmer();+ if(stemmer) {+ token = stemmer->stem(token);+ } else {+ LOG(INFO) << "Stemmer couldn't be initialized for field: " << a_field.name;+ }+ }token_to_offsets[token].push_back(token_index + 1);last_token = token;@@ -1195,6 +1205,15 @@ void Index::tokenize_string_array(const std::vector<std::string>& strings,token.erase(100);}+ if(a_field.is_stem()) {+ auto stemmer = a_field.get_stemmer();+ if(stemmer) {+ token = stemmer->stem(token);+ } else {+ LOG(INFO) << "Stemmer couldn't be initialized for field: " << a_field.name;+ }+ }+token_to_offsets[token].push_back(token_index + 1);token_set.insert(token);last_token = token;diff --git a/src/stemmer_manager.cpp b/src/stemmer_manager.cppnew file mode 100644index 00000000..072b96c1--- /dev/null+++ b/src/stemmer_manager.cpp@@ -0,0 +1,59 @@+#include "stemmer_manager.h"+++Stemmer::Stemmer(const char * language) {+ this->stemmer = sb_stemmer_new(language, nullptr);+ this->cache = LRU::Cache<std::string, std::string>(20);+}++Stemmer::~Stemmer() {+ sb_stemmer_delete(stemmer);+}++std::string Stemmer::stem(const std::string & word) {+ std::unique_lock<std::mutex> lock(mutex);+ std::string stemmed_word;+ if (cache.contains(word)) {+ return cache.lookup(word);+ }+ auto stemmed = sb_stemmer_stem(stemmer, reinterpret_cast<const sb_symbol*>(word.c_str()), word.length());+ stemmed_word = std::string(reinterpret_cast<const char*>(stemmed));+ cache.insert(word, stemmed_word);+ return stemmed_word;+}++StemmerManager::~StemmerManager() {+ delete_all_stemmers();+}++std::shared_ptr<Stemmer> StemmerManager::get_stemmer(const std::string& language) {+ std::unique_lock<std::mutex> lock(mutex);+ // use english as default language+ const std::string language_ = language.empty() ? "english" : language;+ if (stemmers.find(language_) == stemmers.end()) {+ stemmers[language] = std::make_shared<Stemmer>(language_.c_str());+ }+ return stemmers[language];+}++void StemmerManager::delete_stemmer(const std::string& language) {+ std::unique_lock<std::mutex> lock(mutex);+ if (stemmers.find(language) != stemmers.end()) {+ stemmers.erase(language);+ }+}++void StemmerManager::delete_all_stemmers() {+ std::unique_lock<std::mutex> lock(mutex);+ stemmers.clear();+}++const bool StemmerManager::validate_language(const std::string& language) {+ const std::string language_ = language.empty() ? "english" : language;+ auto stemmer = sb_stemmer_new(language_.c_str(), nullptr);+ if (stemmer == nullptr) {+ return false;+ }+ sb_stemmer_delete(stemmer);+ return true;+}\ No newline at end of filediff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex fab4adb3..8cd97179 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1707,4 +1707,29 @@ TEST_F(CollectionAllFieldsTest, StoreInvalidInput) {ASSERT_FALSE(obj_coll_op.ok());ASSERT_EQ("The `store` property of the field `age` should be a boolean.", obj_coll_op.error());-}\ No newline at end of file+}++TEST_F(CollectionAllFieldsTest, InvalidstemValue) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string", "stem": "qwerty"}+ ]+ })"_json;++ auto obj_coll_op = collectionManager.create_collection(schema);+ ASSERT_FALSE(obj_coll_op.ok());+ ASSERT_EQ("The `stem` property of the field `name` should be a boolean.", obj_coll_op.error());++ schema = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "int32", "stem": true}+ ]+ })"_json;++ obj_coll_op = collectionManager.create_collection(schema);+ ASSERT_FALSE(obj_coll_op.ok());+ ASSERT_EQ("The `stem` property is only allowed for string and string[] fields.", obj_coll_op.error());+}+diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex b84edc14..bcc29a7c 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2759,4 +2759,47 @@ TEST_F(CollectionSpecificMoreTest, DisableTyposForNumericalTokens) {ASSERT_TRUE(res_op.ok());ASSERT_EQ(2, res_op.get()["hits"].size());+}++TEST_F(CollectionSpecificMoreTest, TestStemming) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string", "stem": true}+ ]+ })"_json;++ auto coll_stem_res = collectionManager.create_collection(schema);+ LOG(INFO) << coll_stem_res.error();+ ASSERT_TRUE(coll_stem_res.ok());++ auto coll_stem = coll_stem_res.get();++ schema = R"({+ "name": "test2",+ "fields": [+ {"name": "name", "type": "string", "stem": false}+ ]+ })"_json;++ auto coll_no_stem_res = collectionManager.create_collection(schema);+ LOG(INFO) << coll_no_stem_res.error();+ ASSERT_TRUE(coll_no_stem_res.ok());++ auto coll_no_stem = coll_no_stem_res.get();++ nlohmann::json doc;++ doc["name"] = "running";+ ASSERT_TRUE(coll_stem->add(doc.dump()).ok());+ ASSERT_TRUE(coll_no_stem->add(doc.dump()).ok());++ auto stem_res = coll_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);+ ASSERT_TRUE(stem_res.ok());+ ASSERT_EQ(1, stem_res.get()["hits"].size());+ ASSERT_EQ("running", stem_res.get()["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());++ auto no_stem_res = coll_no_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);+ ASSERT_TRUE(no_stem_res.ok());+ ASSERT_EQ(0, no_stem_res.get()["hits"].size());}\ No newline at end of file