Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactoring model download and checking MD5 hash before downloading

ozanarmaganc71b0574e912
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index 252d2fcf..559f782a 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -2,7 +2,10 @@
#include <unordered_map>
-
+#include <openssl/md5.h>
+#include <fstream>
+#include "logger.h"
+#include "http_client.h"
#include "text_embedder.h"
// singleton class
@@ -55,6 +58,8 @@ public:
static constexpr char* DEFAULT_MODEL_NAME = "ts-e5-small";
static constexpr char* DEFAULT_VOCAB_URL = "https://huggingface.co/typesense/models/resolve/main/e5-small/vocab.txt";
static constexpr char* DEFAULT_VOCAB_NAME = "vocab.txt";
+ static constexpr char* DEFAULT_VOCAB_MD5 = "6480d5d8528ce344256daf115d4965e";
+ static constexpr char* DEFAULT_MODEL_MD5 = "3d421dc72859a72368c106415cdebf2";
inline static std::string model_dir = "";
inline static const std::string get_absolute_model_path(const std::string& model_name) {
if(model_dir.back() != '/') {
@@ -78,6 +83,37 @@ public:
return model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME;
}
}
+ inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {
+ std::ifstream stream(file_path);
+ if (stream.fail()) {
+ return false;
+ }
+ unsigned char md5[MD5_DIGEST_LENGTH];
+ std::stringstream ss,res;
+ ss << stream.rdbuf();
+ MD5((unsigned char*)ss.str().c_str(), ss.str().length(), md5);
+ for (int i = 0; i < MD5_DIGEST_LENGTH; i++) {
+ res << std::hex << (int)md5[i];
+ }
+ return res.str() == target_md5;
+ }
+ inline static void download_default_model() {
+ HttpClient& httpClient = HttpClient::get_instance();
+ if(!check_md5(get_absolute_model_path(DEFAULT_MODEL_NAME), DEFAULT_MODEL_MD5)) {
+ LOG(INFO) << "Downloading default model";
+ long res = httpClient.download_file(DEFAULT_MODEL_URL, get_absolute_model_path(DEFAULT_MODEL_NAME));
+ if(res != 200) {
+ LOG(INFO) << "Failed to download default model: " << res;
+ }
+ }
+ if(!check_md5(get_absolute_vocab_path(), DEFAULT_VOCAB_MD5)) {
+ LOG(INFO) << "Downloading default vocab";
+ long res = httpClient.download_file(DEFAULT_VOCAB_URL, get_absolute_vocab_path());
+ if(res != 200) {
+ LOG(INFO) << "Failed to download default vocab: " << res;
+ }
+ }
+ }
private:
TextEmbedderManager() = default;
std::unordered_map<std::string, TextEmbedder*> text_embedders;
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index d924515e..cd9b4952 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -456,18 +456,8 @@ int run_server(const Config & config, const std::string & version, void (*master
if(config.get_model_dir().size() > 0) {
LOG(INFO) << "Loading text embedding models from " << config.get_model_dir();
- TextEmbedderManager::get_instance().set_model_dir(config.get_model_dir());
-
- LOG(INFO) << "Downloading default model and vocab";
- long res = httpClient.download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- if(res != 200) {
- LOG(INFO) << "Failed to download default model: " << res;
- }
-
- res = httpClient.download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- if(res != 200) {
- LOG(INFO) << "Failed to download default vocab: " << res;
- }
+ TextEmbedderManager::model_dir = config.get_model_dir();
+ TextEmbedderManager::download_default_model();
}
// first we start the peering service
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index ef1a2d62..9234d6bd 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1667,12 +1667,7 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {
TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- }
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- }
+ TextEmbedderManager::download_default_model();
field embedding = field("embedding", field_types::FLOAT_ARRAY, false);
embedding.create_from.push_back("name");
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 77032ad1..13c3b2de 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4623,12 +4623,7 @@ TEST_F(CollectionTest, SemanticSearchTest) {
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- }
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- }
+ TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4663,12 +4658,7 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- }
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- }
+ TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
LOG(INFO) << "op.error(): " << op.error();
@@ -4699,12 +4689,7 @@ TEST_F(CollectionTest, HybridSearch) {
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- }
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- }
+ TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4737,12 +4722,7 @@ TEST_F(CollectionTest, EmbedFielsTest) {
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- }
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- }
+ TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4770,12 +4750,7 @@ TEST_F(CollectionTest, HybridSearchRankFusionTest) {
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- }
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- }
+ TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4849,12 +4824,7 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
- }
- if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
- HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
- }
+ TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());