Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add support for pq in vector queries
ozanarmagan1f4f197fd566
diff --git a/include/vector_query_ops.h b/include/vector_query_ops.hindex ac124ca8..21b17053 100644--- a/include/vector_query_ops.h+++ b/include/vector_query_ops.h@@ -17,6 +17,8 @@ struct vector_query_t {bool query_doc_given = false;float alpha = 0.3;+ std::vector<std::string> qs;+void _reset() {// used for testing onlyfield_name.clear();diff --git a/src/collection.cpp b/src/collection.cppindex e2564a40..a7df515f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1691,6 +1691,63 @@ Option<nlohmann::json> Collection::search(std::string raw_query,return Option<nlohmann::json>(400, "Field `" + vector_query.field_name + "` is marked as a non-indexed field in the schema.");}+ if(!vector_query.qs.empty()) {+ if(embedding_fields.find(vector_query.field_name) == embedding_fields.end()) {+ return Option<nlohmann::json>(400, "`qs` parameter is only supported for auto-embedding fields.");+ }++ std::vector<std::vector<float>> embeddings;+ for(const auto& q: vector_query.qs) {+ EmbedderManager& embedder_manager = EmbedderManager::get_instance();+ auto embedder_op = embedder_manager.get_text_embedder(vector_field_it.value().embed[fields::model_config]);+ if(!embedder_op.ok()) {+ return Option<nlohmann::json>(400, embedder_op.error());+ }++ auto remote_embedding_timeout_us = remote_embedding_timeout_ms * 1000;+ if((std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > remote_embedding_timeout_us) {+ std::string error = "Request timed out.";+ return Option<nlohmann::json>(500, error);+ }++ auto embedder = embedder_op.get();++ if(embedder->is_remote()) {+ if(remote_embedding_num_tries == 0) {+ std::string error = "`remote_embedding_num_tries` must be greater than 0.";+ return Option<nlohmann::json>(400, error);+ }+ }++ std::string embed_query = embedder_manager.get_query_prefix(vector_field_it.value().embed[fields::model_config]) + q;+ auto embedding_op = embedder->Embed(embed_query, remote_embedding_timeout_ms, remote_embedding_num_tries);++ if(!embedding_op.success) {+ if(!embedding_op.error["error"].get<std::string>().empty()) {+ return Option<nlohmann::json>(400, embedding_op.error["error"].get<std::string>());+ } else {+ return Option<nlohmann::json>(400, embedding_op.error.dump());+ }+ }++ embeddings.emplace_back(embedding_op.embedding);+ }++ // get average of all embeddings+ std::vector<float> avg_embedding(vector_field_it.value().num_dim, 0);+ for(const auto& embedding: embeddings) {+ for(size_t i = 0; i < embedding.size(); i++) {+ avg_embedding[i] += embedding[i];+ }+ }+ for(size_t i = 0; i < avg_embedding.size(); i++) {+ avg_embedding[i] /= embeddings.size();+ }++ vector_query.values = avg_embedding;+ }+if(is_wildcard_query) {if(vector_query.values.empty() && !vector_query.query_doc_given) {// for usability we will treat this as non-vector querydiff --git a/src/vector_query_ops.cpp b/src/vector_query_ops.cppindex 74189948..c40fa07c 100644--- a/src/vector_query_ops.cpp+++ b/src/vector_query_ops.cpp@@ -85,11 +85,18 @@ Option<bool> VectorQueryOps::parse_vector_query_str(const std::string& vector_qustd::vector<std::string> param_kvs;StringUtils::split(param_str, param_kvs, ",");- for(auto& param_kv_str: param_kvs) {+ for(size_t i = 0; i < param_kvs.size(); i++) {+ auto& param_kv_str = param_kvs[i];if(param_kv_str.back() == ')') {param_kv_str.pop_back();}+ if(i < param_kvs.size() - 1 && StringUtils::get_occurence_count(param_kv_str, '[') != StringUtils::get_occurence_count(param_kv_str, ']')) {+ // this is a hack to handle the case where the query string contains a comma in the `qs` parameter+ param_kvs[i+1] = param_kv_str + "," + param_kvs[i+1];+ continue;+ }+std::vector<std::string> param_kv;StringUtils::split(param_kv_str, param_kv, ":");if(param_kv.size() != 2) {@@ -166,6 +173,23 @@ Option<bool> VectorQueryOps::parse_vector_query_str(const std::string& vector_quvector_query.alpha = std::stof(param_kv[1]);}++ if(param_kv[0] == "qs") {+ if(param_kv[1].front() != '[' || param_kv[1].back() != ']') {+ return Option<bool>(400, "Malformed vector query string: "+ "`qs` parameter must be a list of strings.");+ }++ param_kv[1].erase(0, 1);+ param_kv[1].pop_back();++ std::vector<std::string> qs;+ StringUtils::split(param_kv[1], qs, ",");+ for(auto& q: qs) {+ StringUtils::trim(q);+ vector_query.qs.push_back(q);+ }+ }}return Option<bool>(true);diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 9c0b6b0e..de643e48 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -3390,4 +3390,97 @@ TEST_F(CollectionVectorTest, InvalidMultiSearchConversation) {ASSERT_EQ(res->status_code, 400);ASSERT_EQ(res_json["message"], "`conversation` cannot be used in POST body. Please set `conversation` as a query parameter in the request, instead of inside the POST body");+}++++TEST_F(CollectionVectorTest, TestVectorQueryQs) {+ auto schema_json =+ R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/all-MiniLM-L12-v2"}}}+ ]+ })"_json;++ EmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "Stark Industries"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto results = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], qs:[superhero, company])");++ ASSERT_TRUE(results.ok());+ ASSERT_EQ(results.get()["hits"].size(), 1);+}+++TEST_F(CollectionVectorTest, TestVectorQueryInvalidQs) {+ auto schema_json =+ R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/all-MiniLM-L12-v2"}}}+ ]+ })"_json;++ EmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "Stark Industries"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto results = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], qs:\"test\")");++ ASSERT_FALSE(results.ok());++ ASSERT_EQ(results.error(), "Malformed vector query string: "+ "`qs` parameter must be a list of strings.");++ results = coll->search("*", {}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([], qs:11)");++ ASSERT_FALSE(results.ok());++ ASSERT_EQ(results.error(), "Malformed vector query string: "+ "`qs` parameter must be a list of strings.");}\ No newline at end of file