Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge conflict resolution.
Kishore Nallan40ac195be7aa
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 649da61a..280e69e9 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -38,15 +38,17 @@ jobs:uses: bazelbuild/setup-bazelisk@v2- name: Download bazel cache- uses: dawidd6/action-download-artifact@v2+ uses: jasonbosco/action-download-artifact@709b71d3729e8980f52a5a2a9ec04261060945c1with:name: bazel-cachesearch_artifacts: trueworkflow_conclusion: ""if_no_artifact_found: warn+ skip_unpack: true- name: Uncompress bazel cacherun: |+ unzip bazel-cache.zipmkdir -p ~/.cache/bazeltar_file="bazel-cache.tar.gz" && \[ -f "$tar_file" ] && \diff --git a/include/index.h b/include/index.hindex 1e8d4f6b..04f8d297 100644--- a/include/index.h+++ b/include/index.h@@ -276,12 +276,26 @@ struct index_record {class VectorFilterFunctor: public hnswlib::BaseFilterFunctor {filter_result_iterator_t* const filter_result_iterator;+ const uint32_t* excluded_ids = nullptr;+ const uint32_t excluded_ids_length = 0;+public:- explicit VectorFilterFunctor(filter_result_iterator_t* const filter_result_iterator) :- filter_result_iterator(filter_result_iterator) {}++ explicit VectorFilterFunctor(filter_result_iterator_t* const filter_result_iterator,+ const uint32_t* excluded_ids = nullptr, const uint32_t excluded_ids_length = 0) :+ filter_result_iterator(filter_result_iterator),+ excluded_ids(excluded_ids), excluded_ids_length(excluded_ids_length) {}bool operator()(hnswlib::labeltype id) override {- if (filter_result_iterator->approx_filter_ids_length == 0) {+ if (filter_result_iterator->approx_filter_ids_length == 0 && excluded_ids_length == 0) {+ return true;+ }++ if(excluded_ids_length > 0 && excluded_ids && std::binary_search(excluded_ids, excluded_ids + excluded_ids_length, id)) {+ return false;+ }++ if(filter_result_iterator->approx_filter_ids_length == 0) {return true;}diff --git a/src/index.cpp b/src/index.cppindex d22a0ae2..f0825726 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2542,6 +2542,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consk++;}+ VectorFilterFunctor filterFunctor(filter_result_iterator, excluded_result_ids, excluded_result_ids_size);auto& field_vector_index = vector_index.at(vector_query.field_name);std::vector<std::pair<float, single_filter_result_t>> dist_results;@@ -2946,7 +2947,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consconst float VECTOR_SEARCH_WEIGHT = vector_query.alpha;const float TEXT_MATCH_WEIGHT = 1.0 - VECTOR_SEARCH_WEIGHT;- VectorFilterFunctor filterFunctor(filter_result_iterator);+ VectorFilterFunctor filterFunctor(filter_result_iterator, excluded_result_ids, excluded_result_ids_size);auto& field_vector_index = vector_index.at(vector_query.field_name);std::vector<std::pair<float, size_t>> dist_labels;diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex b07aa9e1..1fe5e3bb 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -106,7 +106,7 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {return Option<bool>(400, "OpenAI API error: " + res);}- return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ return Option<bool>(400, "OpenAI API error: " + json_res["error"]["message"].get<std::string>());}nlohmann::json models_json;@@ -152,7 +152,7 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {return Option<bool>(400, "OpenAI API error: " + embedding_res);}- return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ return Option<bool>(400, "OpenAI API error: " + json_res["error"]["message"].get<std::string>());}std::vector<float> embedding;try {@@ -337,7 +337,7 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,return Option<bool>(400, "Google API error: " + res);}- return Option<bool>(400, "Google API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ return Option<bool>(400, "Google API error: " + json_res["error"]["message"].get<std::string>());}try {@@ -477,7 +477,7 @@ Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, sizif(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {return Option<bool>(400, "GCP API error: " + res);}- return Option<bool>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ return Option<bool>(400, "GCP API error: " + json_res["error"]["message"].get<std::string>());}nlohmann::json res_json;try {@@ -680,7 +680,7 @@ Option<std::string> GCPEmbedder::generate_access_token(const std::string& refresif(res_code == 408) {return Option<std::string>(408, "GCP API timeout.");}- return Option<std::string>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ return Option<std::string>(400, "GCP API error: " + json_res["error"]["message"].get<std::string>());}nlohmann::json res_json;try {diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 243a0594..755cb6ee 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -3026,6 +3026,83 @@ TEST_F(CollectionVectorTest, TestImageEmbedding) {ASSERT_EQ(results2["hits"][1]["document"]["id"], "0");}+TEST_F(CollectionVectorTest, TestHybridSearchHiddenHits) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "name",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "name"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ EmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto add_op = coll->add(R"({+ "name": "soccer",+ "id": "0"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "guitar",+ "id": "1"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "typesense",+ "id": "2"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ add_op = coll->add(R"({+ "name": "potato",+ "id": "3"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto results = coll->search("sports", {"name", "embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ ASSERT_EQ(4, results["hits"].size());+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+++ // do hybrid search with hidden_hits+ auto hybrid_results = coll->search("sports", {"name", "embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, "", "0").get();++ ASSERT_EQ(3, hybrid_results["hits"].size());+ ASSERT_FALSE(hybrid_results["hits"][0]["document"]["id"] == 0);+}+TEST_F(CollectionVectorTest, TryAddingMultipleImageFieldToEmbedFrom) {auto schema_json =R"({@@ -3045,6 +3122,7 @@ TEST_F(CollectionVectorTest, TryAddingMultipleImageFieldToEmbedFrom) {ASSERT_EQ(collection_create_op.error(), "Only one field can be used in the `embed.from` property of an embed field when embedding from an image field.");}+<<<<<<< Updated upstreamTEST_F(CollectionVectorTest, TestInvalidImage) {auto schema_json =@@ -3073,4 +3151,4 @@ TEST_F(CollectionVectorTest, TestInvalidImage) {ASSERT_EQ(add_op.error(), "Error while processing image");-}\ No newline at end of file+}