Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge conflict resolution.

Kishore Nallan40ac195be7aa
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 649da61a..280e69e9 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -38,15 +38,17 @@ jobs:
uses: bazelbuild/setup-bazelisk@v2
- name: Download bazel cache
- uses: dawidd6/action-download-artifact@v2
+ uses: jasonbosco/action-download-artifact@709b71d3729e8980f52a5a2a9ec04261060945c1
with:
name: bazel-cache
search_artifacts: true
workflow_conclusion: ""
if_no_artifact_found: warn
+ skip_unpack: true
- name: Uncompress bazel cache
run: |
+ unzip bazel-cache.zip
mkdir -p ~/.cache/bazel
tar_file="bazel-cache.tar.gz" && \
[ -f "$tar_file" ] && \
diff --git a/include/index.h b/include/index.h
index 1e8d4f6b..04f8d297 100644
--- a/include/index.h
+++ b/include/index.h
@@ -276,12 +276,26 @@ struct index_record {
class VectorFilterFunctor: public hnswlib::BaseFilterFunctor {
filter_result_iterator_t* const filter_result_iterator;
+ const uint32_t* excluded_ids = nullptr;
+ const uint32_t excluded_ids_length = 0;
+
public:
- explicit VectorFilterFunctor(filter_result_iterator_t* const filter_result_iterator) :
- filter_result_iterator(filter_result_iterator) {}
+
+ explicit VectorFilterFunctor(filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t* excluded_ids = nullptr, const uint32_t excluded_ids_length = 0) :
+ filter_result_iterator(filter_result_iterator),
+ excluded_ids(excluded_ids), excluded_ids_length(excluded_ids_length) {}
bool operator()(hnswlib::labeltype id) override {
- if (filter_result_iterator->approx_filter_ids_length == 0) {
+ if (filter_result_iterator->approx_filter_ids_length == 0 && excluded_ids_length == 0) {
+ return true;
+ }
+
+ if(excluded_ids_length > 0 && excluded_ids && std::binary_search(excluded_ids, excluded_ids + excluded_ids_length, id)) {
+ return false;
+ }
+
+ if(filter_result_iterator->approx_filter_ids_length == 0) {
return true;
}
diff --git a/src/index.cpp b/src/index.cpp
index d22a0ae2..f0825726 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2542,6 +2542,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
k++;
}
+ VectorFilterFunctor filterFunctor(filter_result_iterator, excluded_result_ids, excluded_result_ids_size);
auto& field_vector_index = vector_index.at(vector_query.field_name);
std::vector<std::pair<float, single_filter_result_t>> dist_results;
@@ -2946,7 +2947,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
const float VECTOR_SEARCH_WEIGHT = vector_query.alpha;
const float TEXT_MATCH_WEIGHT = 1.0 - VECTOR_SEARCH_WEIGHT;
- VectorFilterFunctor filterFunctor(filter_result_iterator);
+ VectorFilterFunctor filterFunctor(filter_result_iterator, excluded_result_ids, excluded_result_ids_size);
auto& field_vector_index = vector_index.at(vector_query.field_name);
std::vector<std::pair<float, size_t>> dist_labels;
diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cpp
index b07aa9e1..1fe5e3bb 100644
--- a/src/text_embedder_remote.cpp
+++ b/src/text_embedder_remote.cpp
@@ -106,7 +106,7 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,
if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
return Option<bool>(400, "OpenAI API error: " + res);
}
- return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ return Option<bool>(400, "OpenAI API error: " + json_res["error"]["message"].get<std::string>());
}
nlohmann::json models_json;
@@ -152,7 +152,7 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,
if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
return Option<bool>(400, "OpenAI API error: " + embedding_res);
}
- return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ return Option<bool>(400, "OpenAI API error: " + json_res["error"]["message"].get<std::string>());
}
std::vector<float> embedding;
try {
@@ -337,7 +337,7 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,
return Option<bool>(400, "Google API error: " + res);
}
- return Option<bool>(400, "Google API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ return Option<bool>(400, "Google API error: " + json_res["error"]["message"].get<std::string>());
}
try {
@@ -477,7 +477,7 @@ Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, siz
if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
return Option<bool>(400, "GCP API error: " + res);
}
- return Option<bool>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ return Option<bool>(400, "GCP API error: " + json_res["error"]["message"].get<std::string>());
}
nlohmann::json res_json;
try {
@@ -680,7 +680,7 @@ Option<std::string> GCPEmbedder::generate_access_token(const std::string& refres
if(res_code == 408) {
return Option<std::string>(408, "GCP API timeout.");
}
- return Option<std::string>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ return Option<std::string>(400, "GCP API error: " + json_res["error"]["message"].get<std::string>());
}
nlohmann::json res_json;
try {
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 243a0594..755cb6ee 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -3026,6 +3026,83 @@ TEST_F(CollectionVectorTest, TestImageEmbedding) {
ASSERT_EQ(results2["hits"][1]["document"]["id"], "0");
}
+TEST_F(CollectionVectorTest, TestHybridSearchHiddenHits) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "name",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "name"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ EmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto add_op = coll->add(R"({
+ "name": "soccer",
+ "id": "0"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "guitar",
+ "id": "1"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "typesense",
+ "id": "2"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ add_op = coll->add(R"({
+ "name": "potato",
+ "id": "3"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto results = coll->search("sports", {"name", "embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+
+ // do hybrid search with hidden_hits
+ auto hybrid_results = coll->search("sports", {"name", "embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, "", "0").get();
+
+ ASSERT_EQ(3, hybrid_results["hits"].size());
+ ASSERT_FALSE(hybrid_results["hits"][0]["document"]["id"] == 0);
+}
+
TEST_F(CollectionVectorTest, TryAddingMultipleImageFieldToEmbedFrom) {
auto schema_json =
R"({
@@ -3045,6 +3122,7 @@ TEST_F(CollectionVectorTest, TryAddingMultipleImageFieldToEmbedFrom) {
ASSERT_EQ(collection_create_op.error(), "Only one field can be used in the `embed.from` property of an embed field when embedding from an image field.");
}
+<<<<<<< Updated upstream
TEST_F(CollectionVectorTest, TestInvalidImage) {
auto schema_json =
@@ -3073,4 +3151,4 @@ TEST_F(CollectionVectorTest, TestInvalidImage) {
ASSERT_EQ(add_op.error(), "Error while processing image");
-}
\ No newline at end of file
+}