Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Update CLIP tokenizer (#1845)

Ozan Armağan4ef900983d56
diff --git a/WORKSPACE b/WORKSPACE
index 4e1f41fa..acf7a81a 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -54,7 +54,7 @@ new_git_repository(
new_git_repository(
name="clip_tokenizer",
branch="master",
- remote="https://github.com/ozanarmagan/clip_tokenizer_cpp",
+ remote="https://github.com/typesense/clip_tokenizer_cpp.git",
build_file = "//bazel:clip_tokenizer.BUILD",
)
diff --git a/src/validator.cpp b/src/validator.cpp
index 931fc805..7654705d 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -723,7 +723,7 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
if(schema_field_it == search_schema.end()) {
return Option<bool>(400, "Field `" + field.name + "` has invalid fields to create embeddings from.");
}
- if(doc_field_it == document.end()) {
+ if(doc_field_it == document.end() || doc_field_it.value().is_null()) {
if(!is_update && !schema_field_it->optional) {
return Option<bool>(400, "Field `" + field_name + "` is needed to create embedding.");
} else {
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 115754a5..af1c9516 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -3383,6 +3383,45 @@ TEST_F(CollectionVectorTest, TryAddingMultipleImageFieldToEmbedFrom) {
ASSERT_TRUE(collection_create_op.ok());
}
+TEST_F(CollectionVectorTest, TestLongTextForImageEmbedding) {
+ auto schema_json = R"({
+ "name": "images2",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/clip-vit-b-p32"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ EmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ auto coll = collection_create_op.get();
+
+ const std::string long_text = "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?";
+
+ nlohmann::json doc;
+ doc["name"] = long_text;
+
+ auto add_op = coll->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+}
+
+
TEST_F(CollectionVectorTest, TestMultipleFieldsForImageEmbedding) {
auto schema_json = R"({
"name": "images",