Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add stemming support for fields (#1498)

Ozan Armağan0f54bf71010a
diff --git a/BUILD b/BUILD
index 37e75672..27fdf254 100644
--- a/BUILD
+++ b/BUILD
@@ -62,6 +62,8 @@ cc_library(
"@clip_tokenizer//:clip",
"@whisper.cpp//:whisper",
"@whisper.cpp//:whisper_headers",
+ "@snowball",
+ "@snowball//:snowball_headers",
# "@zip",
])
diff --git a/WORKSPACE b/WORKSPACE
index df35a0d1..d090cccf 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -336,4 +336,11 @@ http_archive(
)
load("@rules_cuda//cuda:repositories.bzl", "register_detected_cuda_toolchains", "rules_cuda_dependencies")
rules_cuda_dependencies()
-register_detected_cuda_toolchains()
\ No newline at end of file
+register_detected_cuda_toolchains()
+
+new_git_repository(
+ name= "snowball",
+ build_file = "//bazel:snowball.BUILD",
+ branch = "master",
+ remote = "https://github.com/snowballstem/snowball.git"
+)
\ No newline at end of file
diff --git a/bazel/snowball.BUILD b/bazel/snowball.BUILD
new file mode 100644
index 00000000..975d5e95
--- /dev/null
+++ b/bazel/snowball.BUILD
@@ -0,0 +1,33 @@
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "make")
+
+package(default_visibility = ["//visibility:public"])
+
+
+filegroup(
+ name = "snowball_src",
+ srcs = glob(["**"]),
+ visibility = ["//visibility:public"],
+)
+
+cc_library(
+ name = "snowball_headers",
+ hdrs = glob(["include/**/*.h"]),
+ includes = ["include"],
+ visibility = ["//visibility:public"],
+ strip_include_prefix = "include"
+)
+
+
+make(
+ name = "snowball",
+ lib_source = "//:snowball_src",
+ out_static_libs = ["libstemmer.a"],
+ out_include_dir = "include",
+ args= ["-j12"],
+ tags = ["no-sandbox"],
+ targets = [""],
+ postfix_script= """
+ echo "Installing snowball"
+ cp $BUILD_TMPDIR/libstemmer.a $INSTALLDIR/lib/libstemmer.a
+ """
+)
diff --git a/include/field.h b/include/field.h
index 20c34427..49f35f84 100644
--- a/include/field.h
+++ b/include/field.h
@@ -12,6 +12,8 @@
#include "json.hpp"
#include "embedder_manager.h"
#include "vector_query_ops.h"
+#include <mutex>
+#include "stemmer_manager.h"
namespace field_types {
// first field value indexed will determine the type
@@ -62,6 +64,7 @@ namespace fields {
static const std::string from = "from";
static const std::string model_name = "model_name";
static const std::string range_index = "range_index";
+ static const std::string stem = "stem";
// Some models require additional parameters to be passed to the model during indexing/querying
// For e.g. e5-small model requires prefix "passage:" for indexing and "query:" for querying
@@ -112,20 +115,26 @@ struct field {
bool is_reference_helper = false;
+ bool stem = false;
+ std::shared_ptr<Stemmer> stemmer;
+
field() {}
field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,
bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,
int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine,
- std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true) :
+ std::string reference = "", const nlohmann::json& embed = nlohmann::json(), const bool range_index = false, const bool store = true, const bool stem = false) :
name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),
nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference),
- embed(embed), range_index(range_index), store(store) {
+ embed(embed), range_index(range_index), store(store), stem(stem) {
set_computed_defaults(sort, infix);
auto const suffix = std::string(fields::REFERENCE_HELPER_FIELD_SUFFIX);
is_reference_helper = name.size() > suffix.size() && name.substr(name.size() - suffix.size()) == suffix;
+ if (stem) {
+ stemmer = StemmerManager::get_instance().get_stemmer(locale);
+ }
}
void set_computed_defaults(int sort, int infix) {
@@ -256,6 +265,10 @@ struct field {
return is_num_sortable() || is_str_sortable();
}
+ bool is_stem() const {
+ return stem;
+ }
+
bool has_valid_type() const {
bool is_basic_type = is_string() || is_integer() || is_float() || is_bool() || is_geopoint() ||
is_object() || is_auto() || is_image();
@@ -269,6 +282,10 @@ struct field {
return (facet && !is_string()) ? "_fstr_" + name : name;
}
+ std::shared_ptr<Stemmer> get_stemmer() const {
+ return stemmer;
+ }
+
static bool get_type(const nlohmann::json& obj, std::string& field_type) {
if(obj.is_array()) {
if(obj.empty()) {
diff --git a/include/stemmer_manager.h b/include/stemmer_manager.h
new file mode 100644
index 00000000..9f63e710
--- /dev/null
+++ b/include/stemmer_manager.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include <unordered_map>
+#include <string>
+#include <vector>
+#include <memory>
+#include <mutex>
+#include <libstemmer.h>
+#include "lru/lru.hpp"
+
+
+class Stemmer {
+ private:
+ sb_stemmer * stemmer = nullptr;
+ LRU::Cache<std::string, std::string> cache;
+ std::mutex mutex;
+ public:
+ Stemmer(const char * language);
+ ~Stemmer();
+ std::string stem(const std::string & word);
+};
+
+
+class StemmerManager {
+ private:
+ std::unordered_map<std::string, std::shared_ptr<Stemmer>> stemmers;
+ StemmerManager() {}
+ std::mutex mutex;
+ public:
+ static StemmerManager& get_instance() {
+ static StemmerManager instance;
+ return instance;
+ }
+ StemmerManager(StemmerManager const&) = delete;
+ void operator=(StemmerManager const&) = delete;
+ StemmerManager(StemmerManager&&) = delete;
+ void operator=(StemmerManager&&) = delete;
+ ~StemmerManager();
+ std::shared_ptr<Stemmer> get_stemmer(const std::string& language);
+ void delete_stemmer(const std::string& language);
+ void delete_all_stemmers();
+ const bool validate_language(const std::string& language);
+};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 80623c24..ac13d4b7 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -431,6 +431,7 @@ nlohmann::json Collection::get_summary_json() const {
field_json[fields::sort] = coll_field.sort;
field_json[fields::infix] = coll_field.infix;
field_json[fields::locale] = coll_field.locale;
+ field_json[fields::stem] = coll_field.stem;
if(coll_field.embed.count(fields::from) != 0) {
field_json[fields::embed] = coll_field.embed;
diff --git a/src/field.cpp b/src/field.cpp
index 28b67aef..cc2e4e56 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -6,6 +6,7 @@
#include <collection_manager.h>
#include <regex>
+
Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,
std::vector<field>& the_fields,
string& fallback_field_type, size_t& num_auto_detect_fields) {
@@ -80,6 +81,31 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
field_json[fields::reference] = "";
}
+ if(field_json.count(fields::stem) != 0) {
+ if(!field_json.at(fields::stem).is_boolean()) {
+ return Option<bool>(400, std::string("The `stem` property of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` should be a boolean."));
+ }
+
+ if(field_json[fields::stem] && field_json[fields::type] != field_types::STRING && field_json[fields::type] != field_types::STRING_ARRAY) {
+ return Option<bool>(400, std::string("The `stem` property is only allowed for string and string[] fields."));
+ }
+
+ if(field_json[fields::stem].get<bool>()) {
+ std::string locale;
+ if(field_json.count(fields::locale) != 0) {
+ locale = field_json[fields::locale].get<std::string>();
+ }
+ auto stem_validation = StemmerManager::get_instance().validate_language(locale);
+ if(!stem_validation) {
+ return Option<bool>(400, std::string("The `locale` value of the field `") +
+ field_json[fields::name].get<std::string>() + std::string("` is not supported for stem."));
+ }
+ }
+ } else {
+ field_json[fields::stem] = false;
+ }
+
if (field_json.count(fields::range_index) != 0) {
if (!field_json.at(fields::range_index).is_boolean()) {
return Option<bool>(400, std::string("The `range_index` property of the field `") +
@@ -338,7 +364,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
field_json[fields::optional], field_json[fields::index], field_json[fields::locale],
field_json[fields::sort], field_json[fields::infix], field_json[fields::nested],
field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,
- field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index], field_json[fields::store])
+ field_json[fields::reference], field_json[fields::embed], field_json[fields::range_index], field_json[fields::store], field_json[fields::stem])
);
if (!field_json[fields::reference].get<std::string>().empty()) {
diff --git a/src/index.cpp b/src/index.cpp
index 4a67afe6..235f46b6 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -547,7 +547,8 @@ void Index::validate_and_preprocess(Index *index,
}
}
-size_t Index::batch_memory_index(Index *index,
+size_t Index::
+batch_memory_index(Index *index,
std::vector<index_record>& iter_batch,
const std::string & default_sorting_field,
const tsl::htrie_map<char, field> & actual_search_schema,
@@ -1160,6 +1161,15 @@ void Index::tokenize_string(const std::string& text, const field& a_field,
if(token.size() > 100) {
token.erase(100);
}
+
+ if(a_field.is_stem()) {
+ auto stemmer = a_field.get_stemmer();
+ if(stemmer) {
+ token = stemmer->stem(token);
+ } else {
+ LOG(INFO) << "Stemmer couldn't be initialized for field: " << a_field.name;
+ }
+ }
token_to_offsets[token].push_back(token_index + 1);
last_token = token;
@@ -1195,6 +1205,15 @@ void Index::tokenize_string_array(const std::vector<std::string>& strings,
token.erase(100);
}
+ if(a_field.is_stem()) {
+ auto stemmer = a_field.get_stemmer();
+ if(stemmer) {
+ token = stemmer->stem(token);
+ } else {
+ LOG(INFO) << "Stemmer couldn't be initialized for field: " << a_field.name;
+ }
+ }
+
token_to_offsets[token].push_back(token_index + 1);
token_set.insert(token);
last_token = token;
diff --git a/src/stemmer_manager.cpp b/src/stemmer_manager.cpp
new file mode 100644
index 00000000..072b96c1
--- /dev/null
+++ b/src/stemmer_manager.cpp
@@ -0,0 +1,59 @@
+#include "stemmer_manager.h"
+
+
+Stemmer::Stemmer(const char * language) {
+ this->stemmer = sb_stemmer_new(language, nullptr);
+ this->cache = LRU::Cache<std::string, std::string>(20);
+}
+
+Stemmer::~Stemmer() {
+ sb_stemmer_delete(stemmer);
+}
+
+std::string Stemmer::stem(const std::string & word) {
+ std::unique_lock<std::mutex> lock(mutex);
+ std::string stemmed_word;
+ if (cache.contains(word)) {
+ return cache.lookup(word);
+ }
+ auto stemmed = sb_stemmer_stem(stemmer, reinterpret_cast<const sb_symbol*>(word.c_str()), word.length());
+ stemmed_word = std::string(reinterpret_cast<const char*>(stemmed));
+ cache.insert(word, stemmed_word);
+ return stemmed_word;
+}
+
+StemmerManager::~StemmerManager() {
+ delete_all_stemmers();
+}
+
+std::shared_ptr<Stemmer> StemmerManager::get_stemmer(const std::string& language) {
+ std::unique_lock<std::mutex> lock(mutex);
+ // use english as default language
+ const std::string language_ = language.empty() ? "english" : language;
+ if (stemmers.find(language_) == stemmers.end()) {
+ stemmers[language] = std::make_shared<Stemmer>(language_.c_str());
+ }
+ return stemmers[language];
+}
+
+void StemmerManager::delete_stemmer(const std::string& language) {
+ std::unique_lock<std::mutex> lock(mutex);
+ if (stemmers.find(language) != stemmers.end()) {
+ stemmers.erase(language);
+ }
+}
+
+void StemmerManager::delete_all_stemmers() {
+ std::unique_lock<std::mutex> lock(mutex);
+ stemmers.clear();
+}
+
+const bool StemmerManager::validate_language(const std::string& language) {
+ const std::string language_ = language.empty() ? "english" : language;
+ auto stemmer = sb_stemmer_new(language_.c_str(), nullptr);
+ if (stemmer == nullptr) {
+ return false;
+ }
+ sb_stemmer_delete(stemmer);
+ return true;
+}
\ No newline at end of file
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index fab4adb3..8cd97179 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1707,4 +1707,29 @@ TEST_F(CollectionAllFieldsTest, StoreInvalidInput) {
ASSERT_FALSE(obj_coll_op.ok());
ASSERT_EQ("The `store` property of the field `age` should be a boolean.", obj_coll_op.error());
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionAllFieldsTest, InvalidstemValue) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string", "stem": "qwerty"}
+ ]
+ })"_json;
+
+ auto obj_coll_op = collectionManager.create_collection(schema);
+ ASSERT_FALSE(obj_coll_op.ok());
+ ASSERT_EQ("The `stem` property of the field `name` should be a boolean.", obj_coll_op.error());
+
+ schema = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "int32", "stem": true}
+ ]
+ })"_json;
+
+ obj_coll_op = collectionManager.create_collection(schema);
+ ASSERT_FALSE(obj_coll_op.ok());
+ ASSERT_EQ("The `stem` property is only allowed for string and string[] fields.", obj_coll_op.error());
+}
+
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index b84edc14..bcc29a7c 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2759,4 +2759,47 @@ TEST_F(CollectionSpecificMoreTest, DisableTyposForNumericalTokens) {
ASSERT_TRUE(res_op.ok());
ASSERT_EQ(2, res_op.get()["hits"].size());
+}
+
+TEST_F(CollectionSpecificMoreTest, TestStemming) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string", "stem": true}
+ ]
+ })"_json;
+
+ auto coll_stem_res = collectionManager.create_collection(schema);
+ LOG(INFO) << coll_stem_res.error();
+ ASSERT_TRUE(coll_stem_res.ok());
+
+ auto coll_stem = coll_stem_res.get();
+
+ schema = R"({
+ "name": "test2",
+ "fields": [
+ {"name": "name", "type": "string", "stem": false}
+ ]
+ })"_json;
+
+ auto coll_no_stem_res = collectionManager.create_collection(schema);
+ LOG(INFO) << coll_no_stem_res.error();
+ ASSERT_TRUE(coll_no_stem_res.ok());
+
+ auto coll_no_stem = coll_no_stem_res.get();
+
+ nlohmann::json doc;
+
+ doc["name"] = "running";
+ ASSERT_TRUE(coll_stem->add(doc.dump()).ok());
+ ASSERT_TRUE(coll_no_stem->add(doc.dump()).ok());
+
+ auto stem_res = coll_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);
+ ASSERT_TRUE(stem_res.ok());
+ ASSERT_EQ(1, stem_res.get()["hits"].size());
+ ASSERT_EQ("running", stem_res.get()["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());
+
+ auto no_stem_res = coll_no_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);
+ ASSERT_TRUE(no_stem_res.ok());
+ ASSERT_EQ(0, no_stem_res.get()["hits"].size());
}
\ No newline at end of file