Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactoring model download and checking MD5 hash before downloading
ozanarmaganc71b0574e912
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex 252d2fcf..559f782a 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -2,7 +2,10 @@#include <unordered_map>-+#include <openssl/md5.h>+#include <fstream>+#include "logger.h"+#include "http_client.h"#include "text_embedder.h"// singleton class@@ -55,6 +58,8 @@ public:static constexpr char* DEFAULT_MODEL_NAME = "ts-e5-small";static constexpr char* DEFAULT_VOCAB_URL = "https://huggingface.co/typesense/models/resolve/main/e5-small/vocab.txt";static constexpr char* DEFAULT_VOCAB_NAME = "vocab.txt";+ static constexpr char* DEFAULT_VOCAB_MD5 = "6480d5d8528ce344256daf115d4965e";+ static constexpr char* DEFAULT_MODEL_MD5 = "3d421dc72859a72368c106415cdebf2";inline static std::string model_dir = "";inline static const std::string get_absolute_model_path(const std::string& model_name) {if(model_dir.back() != '/') {@@ -78,6 +83,37 @@ public:return model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME;}}+ inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {+ std::ifstream stream(file_path);+ if (stream.fail()) {+ return false;+ }+ unsigned char md5[MD5_DIGEST_LENGTH];+ std::stringstream ss,res;+ ss << stream.rdbuf();+ MD5((unsigned char*)ss.str().c_str(), ss.str().length(), md5);+ for (int i = 0; i < MD5_DIGEST_LENGTH; i++) {+ res << std::hex << (int)md5[i];+ }+ return res.str() == target_md5;+ }+ inline static void download_default_model() {+ HttpClient& httpClient = HttpClient::get_instance();+ if(!check_md5(get_absolute_model_path(DEFAULT_MODEL_NAME), DEFAULT_MODEL_MD5)) {+ LOG(INFO) << "Downloading default model";+ long res = httpClient.download_file(DEFAULT_MODEL_URL, get_absolute_model_path(DEFAULT_MODEL_NAME));+ if(res != 200) {+ LOG(INFO) << "Failed to download default model: " << res;+ }+ }+ if(!check_md5(get_absolute_vocab_path(), DEFAULT_VOCAB_MD5)) {+ LOG(INFO) << "Downloading default vocab";+ long res = httpClient.download_file(DEFAULT_VOCAB_URL, get_absolute_vocab_path());+ if(res != 200) {+ LOG(INFO) << "Failed to download default vocab: " << res;+ }+ }+ }private:TextEmbedderManager() = default;std::unordered_map<std::string, TextEmbedder*> text_embedders;diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex d924515e..cd9b4952 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -456,18 +456,8 @@ int run_server(const Config & config, const std::string & version, void (*masterif(config.get_model_dir().size() > 0) {LOG(INFO) << "Loading text embedding models from " << config.get_model_dir();- TextEmbedderManager::get_instance().set_model_dir(config.get_model_dir());-- LOG(INFO) << "Downloading default model and vocab";- long res = httpClient.download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- if(res != 200) {- LOG(INFO) << "Failed to download default model: " << res;- }-- res = httpClient.download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- if(res != 200) {- LOG(INFO) << "Failed to download default vocab: " << res;- }+ TextEmbedderManager::model_dir = config.get_model_dir();+ TextEmbedderManager::download_default_model();}// first we start the peering servicediff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex ef1a2d62..9234d6bd 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1667,12 +1667,7 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- }- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- }+ TextEmbedderManager::download_default_model();field embedding = field("embedding", field_types::FLOAT_ARRAY, false);embedding.create_from.push_back("name");diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 77032ad1..13c3b2de 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4623,12 +4623,7 @@ TEST_F(CollectionTest, SemanticSearchTest) {})"_json;TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- }- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- }+ TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4663,12 +4658,7 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {})"_json;TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- }- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- }+ TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);LOG(INFO) << "op.error(): " << op.error();@@ -4699,12 +4689,7 @@ TEST_F(CollectionTest, HybridSearch) {})"_json;TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- }- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- }+ TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4737,12 +4722,7 @@ TEST_F(CollectionTest, EmbedFielsTest) {})"_json;TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- }- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- }+ TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4770,12 +4750,7 @@ TEST_F(CollectionTest, HybridSearchRankFusionTest) {})"_json;TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- }- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- }+ TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4849,12 +4824,7 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {})"_json;TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));- }- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());- }+ TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());