Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review changes II
ozanarmagane1d452b713d2
diff --git a/include/field.h b/include/field.hindex 9aae16c8..eca39f1e 100644--- a/include/field.h+++ b/include/field.h@@ -50,7 +50,7 @@ namespace fields {static const std::string vec_dist = "vec_dist";static const std::string reference = "reference";static const std::string create_from = "create_from";- static const std::string model_path = "model_path";+ static const std::string model_name = "model_name";}enum vector_distance_type_t {@@ -77,7 +77,7 @@ struct field {size_t num_dim;std::vector<std::string> create_from;- std::string model_path;+ std::string model_name;vector_distance_type_t vec_dist;static constexpr int VAL_UNKNOWN = 2;@@ -88,9 +88,9 @@ struct field {field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1, bool nested = false,- int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &create_from = {}, const std::string& model_path = "") :+ int nested_array = 0, size_t num_dim = 0, vector_distance_type_t vec_dist = cosine, std::string reference = "", const std::vector<std::string> &create_from = {}, const std::string& model_name = "") :name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),- nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), create_from(create_from), model_path(model_path) {+ nested(nested), nested_array(nested_array), num_dim(num_dim), vec_dist(vec_dist), reference(reference), create_from(create_from), model_name(model_name) {set_computed_defaults(sort, infix);}@@ -320,8 +320,8 @@ struct field {}if(!field.create_from.empty()) {field_val[fields::create_from] = field.create_from;- if(!field.model_path.empty()) {- field_val[fields::model_path] = field.model_path;+ if(!field.model_name.empty()) {+ field_val[fields::model_name] = field.model_name;}}fields_json.push_back(field_val);diff --git a/include/text_embedder.h b/include/text_embedder.hindex d65b7345..c1cb33fd 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -24,6 +24,5 @@ class TextEmbedder {encoded_input_t Encode(const std::string& text);BertTokenizer* tokenizer_;static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);- static std::string get_absolute_model_path(const std::string& model_path);std::string output_tensor_name;};diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex 4dd033e5..d0c3ea0d 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -52,10 +52,32 @@ public:}static constexpr char* DEFAULT_MODEL_URL = "https://huggingface.co/ozanarmagan/e5-small-onnx/resolve/main/model.onnx";- static constexpr char* DEFAULT_MODEL_NAME = "model.onnx";+ static constexpr char* DEFAULT_MODEL_NAME = "model";static constexpr char* DEFAULT_VOCAB_URL = "https://huggingface.co/ozanarmagan/e5-small-onnx/resolve/main/vocab.txt";static constexpr char* DEFAULT_VOCAB_NAME = "vocab.txt";inline static std::string model_dir = "";+ inline static const std::string get_absolute_model_path(const std::string& model_name) {+ if(model_dir.back() != '/') {+ if(model_name.front() != '/') {+ return model_dir + "/" + model_name + ".onnx";+ } else {+ return model_dir + model_name + ".onnx";+ }+ } else {+ if(model_name.front() != '/') {+ return model_dir + model_name + ".onnx";+ } else {+ return model_dir + "/" + model_name + ".onnx";+ }+ }+ };+ inline static const std::string get_absolute_vocab_path() {+ if(model_dir.back() != '/') {+ return model_dir + "/" + TextEmbedderManager::DEFAULT_VOCAB_NAME;+ } else {+ return model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME;+ }+ }private:TextEmbedderManager() = default;std::unordered_map<std::string, TextEmbedder*> text_embedders;diff --git a/src/collection.cpp b/src/collection.cppindex f111eb34..54dc6b87 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -234,8 +234,8 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::create_from] = coll_field.create_from;}- if(coll_field.model_path.size() > 0) {- field_json[fields::model_path] = coll_field.model_path;+ if(coll_field.model_name.size() > 0) {+ field_json[fields::model_name] = coll_field.model_name;}if(coll_field.num_dim > 0) {@@ -1163,7 +1163,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();- auto embedder = embedder_manager.get_text_embedder(search_field.model_path.size() > 0 ? search_field.model_path : TextEmbedderManager::DEFAULT_MODEL_NAME);+ auto embedder = embedder_manager.get_text_embedder(search_field.model_name.size() > 0 ? search_field.model_name : TextEmbedderManager::DEFAULT_MODEL_NAME);std::vector<float> embedding = embedder->Embed(raw_query);vector_query._reset();@@ -4654,7 +4654,7 @@ Option<bool> Collection::embed_fields(nlohmann::json& document) {}TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();- auto embedder = embedder_manager.get_text_embedder(field.model_path.size() > 0 ? field.model_path : "model.onnx");+ auto embedder = embedder_manager.get_text_embedder(field.model_name.size() > 0 ? field.model_name : TextEmbedderManager::DEFAULT_MODEL_NAME);std::vector<float> embedding = embedder->Embed(text_to_embed);document[field.name] = embedding;}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 59ee80ff..30721aa0 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -62,8 +62,8 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::create_from] = std::vector<std::string>();}- if(field_obj.count(fields::model_path) == 0) {- field_obj[fields::model_path] = "";+ if(field_obj.count(fields::model_name) == 0) {+ field_obj[fields::model_name] = "";}vector_distance_type_t vec_dist_type = vector_distance_type_t::cosine;@@ -79,7 +79,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionfield_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],field_obj[fields::num_dim], vec_dist_type, field_obj[fields::reference], field_obj[fields::create_from],- field_obj[fields::model_path]);+ field_obj[fields::model_name]);// value of `sort` depends on field typeif(field_obj.count(fields::sort) == 0) {diff --git a/src/field.cpp b/src/field.cppindex 2971e257..0e4b5193 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -663,26 +663,26 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}- if(!field_json[fields::model_path].is_null() && field_json[fields::create_from].is_null()) {- return Option<bool>(400, "Property `" + fields::model_path + "` can only be used with `" + fields::create_from + "`.");+ if(field_json.count(fields::model_name) > 0 && field_json.count(fields::create_from) == 0) {+ return Option<bool>(400, "Property `" + fields::model_name + "` can only be used with `" + fields::create_from + "`.");}if(field_json.count(fields::create_from) != 0) {// If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)field_json[fields::num_dim] = static_cast<unsigned int>(384);- if(!field_json[fields::model_path].is_null()) {+ if(field_json.count(fields::model_name) != 0) {unsigned int num_dim = 0;- if(!field_json[fields::model_path].is_string()) {- return Option<bool>(400, "Property `" + fields::model_path + "` must be a string.");+ if(!field_json[fields::model_name].is_string()) {+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a string.");}- if(field_json[fields::model_path].get<std::string>().empty()) {- return Option<bool>(400, "Property `" + fields::model_path + "` must be a non-empty string.");+ if(field_json[fields::model_name].get<std::string>().empty()) {+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");}- if(TextEmbedder::is_model_valid(field_json[fields::model_path].get<std::string>(), num_dim)) {+ if(TextEmbedder::is_model_valid(field_json[fields::model_name].get<std::string>(), num_dim)) {field_json[fields::num_dim] = num_dim;} else {- return Option<bool>(400, "Property `" + fields::model_path + "` must be a valid model path.");+ return Option<bool>(400, "Property `" + fields::model_name + "` must be a valid model path.");}}} else {@@ -690,8 +690,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}- if(field_json[fields::model_path].is_null()) {- field_json[fields::model_path] = "";+ if(field_json.count(fields::model_name) == 0) {+ field_json[fields::model_name] = "";}@@ -776,7 +776,7 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsofield_json[fields::sort], field_json[fields::infix], field_json[fields::nested],field_json[fields::nested_array], field_json[fields::num_dim], vec_dist,field_json[fields::reference], field_json[fields::create_from].get<std::vector<std::string>>(),- field_json[fields::model_path])+ field_json[fields::model_name]));if (!field_json[fields::reference].get<std::string>().empty()) {diff --git a/src/index.cpp b/src/index.cppindex ce5fb65c..d0d0f4ec 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3011,34 +3011,47 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscurated_ids_sorted, all_result_ids, all_result_ids_len, groups_processed);if(!vector_query.field_name.empty()) {- VectorFilterFunctor filterFunctor(seq_ids->uncompress(), seq_ids->num_ids());- auto& field_vector_index = vector_index.at(vector_query.field_name);- std::vector<std::pair<float, size_t>> dist_labels;- auto k = std::max<size_t>(vector_query.k, per_page * page);-- if(field_vector_index->distance_type == cosine) {- std::vector<float> normalized_q(vector_query.values.size());- hnsw_index_t::normalize_vector(vector_query.values, normalized_q);- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, filterFunctor);- } else {- dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, filterFunctor);+ // check at least one of sort fields is text match+ bool has_text_match = false;+ for(auto& sort_field : sort_fields_std) {+ if(sort_field.name == sort_field_const::text_match) {+ has_text_match = true;+ break;+ }}- for (const auto& dist_label : dist_labels) {- uint32 seq_id = dist_label.second;+ if(has_text_match) {+ auto seq_ids_arr = seq_ids->uncompress();+ VectorFilterFunctor filterFunctor(seq_ids_arr, seq_ids->num_ids());+ auto& field_vector_index = vector_index.at(vector_query.field_name);+ std::vector<std::pair<float, size_t>> dist_labels;+ auto k = std::max<size_t>(vector_query.k, per_page * page);- auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :- dist_label.first;+ if(field_vector_index->distance_type == cosine) {+ std::vector<float> normalized_q(vector_query.values.size());+ hnsw_index_t::normalize_vector(vector_query.values, normalized_q);+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(normalized_q.data(), k, filterFunctor);+ } else {+ dist_labels = field_vector_index->vecdex->searchKnnCloserFirst(vector_query.values.data(), k, filterFunctor);+ }- auto score = (1.0 - vec_dist_score) * 100000000000.0;-- auto found = topster->kv_map.find(seq_id);-- if (found != topster->kv_map.end()) {- found->second->scores[0] += score;+ for (const auto& dist_label : dist_labels) {+ uint32 seq_id = dist_label.second;++ auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :+ dist_label.first;++ auto score = (1.0 - vec_dist_score) * 100000000000.0;++ auto found = topster->kv_map.find(seq_id);++ if (found != topster->kv_map.end() && found->second->match_score_index >= 0 && found->second->match_score_index <= 2) {+ found->second->scores[found->second->match_score_index] += score;+ }}- }+ delete [] seq_ids_arr;+ }}/*auto timeMillis0 = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex 976da0f5..92b3ebfb 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -10,7 +10,7 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {// create environmentOrt::SessionOptions session_options;- std::string abs_path = get_absolute_model_path(model_path);+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);LOG(INFO) << "Loading model from: " << abs_path;session_ = new Ort::Session(env_, abs_path.c_str(), session_options);std::ifstream stream("vocab.txt");@@ -21,11 +21,9 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {ustring("[CLS]"), ustring("[MASK]"), true, true, ustring("##"),512, std::string("longest_first"));auto output_tensor_count = session_->GetOutputCount();- bool found_output_tensor = false;- for (int i = 0; i < output_tensor_count; i++) {+ for (size_t i = 0; i < output_tensor_count; i++) {auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();if (shape.size() == 3 && shape[0] == -1 && shape[1] == -1 && shape[2] > 0) {- found_output_tensor = true;Ort::AllocatorWithDefaultOptions allocator;output_tensor_name = std::string(session_->GetOutputNameAllocated(i, allocator).get());break;@@ -57,8 +55,6 @@ std::vector<float> TextEmbedder::mean_pooling(const std::vector<std::vector<floa}std::vector<float> TextEmbedder::Embed(const std::string& text) {-- LOG(INFO) << "Embedding text: " << text;auto encoded_input = Encode(text);// create input tensor object from data valuesOrt::AllocatorWithDefaultOptions allocator;@@ -109,7 +105,7 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& nLOG(INFO) << "Loading model: " << model_path;Ort::SessionOptions session_options;Ort::Env env;- std::string abs_path = get_absolute_model_path(model_path);+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);if(!std::filesystem::exists(abs_path)) {LOG(ERROR) << "Model file not found: " << abs_path;@@ -142,7 +138,7 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& nauto output_tensor_count = session.GetOutputCount();bool found_output_tensor = false;- for (int i = 0; i < output_tensor_count; i++) {+ for (size_t i = 0; i < output_tensor_count; i++) {auto shape = session.GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();if (shape.size() == 3 && shape[0] == -1 && shape[1] == -1 && shape[2] > 0) {num_dims = shape[2];@@ -158,22 +154,3 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& nreturn true;}--std::string TextEmbedder::get_absolute_model_path(const std::string & model_path) {- const std::string model_dir = TextEmbedderManager::model_dir;-- if(model_dir.back() != '/') {- if(model_path.front() != '/') {- return model_dir + "/" + model_path;- } else {- return model_dir + model_path;- }- } else {- if(model_path.front() != '/') {- return model_dir + model_path;- } else {- return model_dir + "/" + model_path;- }- }-}-diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 44565b61..d924515e 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -459,12 +459,12 @@ int run_server(const Config & config, const std::string & version, void (*masterTextEmbedderManager::get_instance().set_model_dir(config.get_model_dir());LOG(INFO) << "Downloading default model and vocab";- long res = httpClient.download_file(TextEmbedderManager::DEFAULT_MODEL_URL, config.get_model_dir() + "/" + TextEmbedderManager::DEFAULT_MODEL_NAME);+ long res = httpClient.download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));if(res != 200) {LOG(INFO) << "Failed to download default model: " << res;}- res = httpClient.download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, config.get_model_dir() + "/" + TextEmbedderManager::DEFAULT_VOCAB_NAME);+ res = httpClient.download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());if(res != 200) {LOG(INFO) << "Failed to download default vocab: " << res;}diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 43f8525d..8dc3a257 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1650,7 +1650,7 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {nlohmann::json field_json;field_json["name"] = "embedding";field_json["type"] = "float[]";- field_json["model_path"] = "model.onnx";+ field_json["model_name"] = "model";std::vector<field> fields;std::string fallback_field_type;@@ -1659,15 +1659,15 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);ASSERT_FALSE(field_op.ok());- ASSERT_EQ("Property `model_path` can only be used with `create_from`.", field_op.error());+ ASSERT_EQ("Property `model_name` can only be used with `create_from`.", field_op.error());}TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {TextEmbedderManager::model_dir = "./models/";- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());field embedding = field("embedding", field_types::FLOAT_ARRAY, false);embedding.create_from.push_back("name");diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 9ca811cf..98ebe805 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4622,8 +4622,8 @@ TEST_F(CollectionTest, SemanticSearchTest) {})"_json;TextEmbedderManager::model_dir = "./models/";- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4661,8 +4661,8 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {})"_json;TextEmbedderManager::model_dir = "./models/";- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());auto op = collectionManager.create_collection(schema);LOG(INFO) << "op.error(): " << op.error();@@ -4696,8 +4696,8 @@ TEST_F(CollectionTest, HybridSearch) {})"_json;TextEmbedderManager::model_dir = "./models/";- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());auto op = collectionManager.create_collection(schema);LOG(INFO) << collectionManager.get_collection("objects")->get_summary_json();@@ -4737,8 +4737,8 @@ TEST_F(CollectionTest, EmbedFielsTest) {})"_json;TextEmbedderManager::model_dir = "./models/";- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_MODEL_NAME);- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME);+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());