Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallan2e2448288213
diff --cc src/index.cpp
index 0e93d6ad,4d81126a..de07a27a
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -2765,16 -3235,8 +2766,17 @@@ Option<bool> Index::search(std::vector<
int64_t match_score_index = -1;
compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);
KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);
+ kv.text_match_score = 0;
kv.vector_distance = vec_result.second;
+
+ if (filter_result_iterator->is_valid &&
+ !filter_result_iterator->reference.empty()) {
+ // The doc_id must be valid otherwise it would've been filtered out upstream.
+ filter_result_iterator->skip_to(doc_id);
+ kv.reference_filter_results = std::move(filter_result_iterator->reference);
+ filter_result_iterator->reset();
+ }
+
topster->add(&kv);
vec_search_ids.push_back(doc_id);
}
@@@ -3726,10 -4163,9 +3728,11 @@@ void Index::search_across_fields(const
<< ", aggregated_score: " << aggregated_score;*/
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
+ kv.reference_filter_results = std::move(filter_result.reference_filter_results);
+
if(match_score_index != -1) {
kv.scores[match_score_index] = aggregated_score;
+ kv.text_match_score = aggregated_score;
}
int ret = topster->add(&kv);
diff --cc test/collection_specific_more_test.cpp
index 68980d21,83181283..aa83229e
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@@ -2310,79 -2370,288 +2380,131 @@@ TEST_F(CollectionSpecificMoreTest, Cros
ASSERT_EQ(1, res["hits"].size());
}
-TEST_F(CollectionSpecificMoreTest, RearrangingFilterTree) {
- nlohmann::json schema =
- R"({
- "name": "Collection",
- "fields": [
- {"name": "name", "type": "string"},
- {"name": "age", "type": "int32"},
- {"name": "years", "type": "int32[]"},
- {"name": "rating", "type": "float"}
- ]
- })"_json;
+TEST_F(CollectionSpecificMoreTest, TruncateAterTopK) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "points", "type": "int32"}
+ ]
+ })"_json;
- Collection* coll = collectionManager.create_collection(schema).get();
+ Collection* coll1 = collectionManager.create_collection(schema).get();
- std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
- std::string json_line;
- while (std::getline(infile, json_line)) {
- auto add_op = coll->add(json_line);
- ASSERT_TRUE(add_op.ok());
+ for(auto i = -10; i < 5; i++) {
+ nlohmann::json doc;
+ doc["title"] = std::to_string(i);
+ doc["points"] = i;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
- infile.close();
-
- const std::string doc_id_prefix = std::to_string(coll->get_collection_id()) + "_" + Collection::DOC_ID_PREFIX + "_";
- filter_node_t* filter_tree_root = nullptr;
- Option<bool> filter_op = filter::parse_filter_query("years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5))",
- coll->get_schema(), store, doc_id_prefix, filter_tree_root);
- ASSERT_TRUE(filter_op.ok());
- std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
-
- // &&
- // / \
- // years>2000 ||
- // 4 / \
- // / &&
- // && / \
- // / \ age>50 rating<5
- // / \ 1 2
- // / \
- // age<30 rating>5
- // 2 3
- ASSERT_TRUE(filter_tree_root != nullptr);
- ASSERT_TRUE(filter_tree_root->isOperator);
- ASSERT_EQ(filter_tree_root->filter_operator, AND);
-
- auto root = filter_tree_root->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "years");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_TRUE(root->isOperator);
- ASSERT_EQ(root->filter_operator, OR);
-
- root = filter_tree_root->right->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_TRUE(root->isOperator);
- ASSERT_EQ(root->filter_operator, AND);
-
- root = filter_tree_root->right->left->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "age");
- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "30");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->right->left->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "rating");
- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "5");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->right->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_TRUE(root->isOperator);
- ASSERT_EQ(root->filter_operator, AND);
-
- root = filter_tree_root->right->right->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "age");
- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "50");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->right->right->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "rating");
- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "5");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- uint32_t count = 0;
- coll->_get_index()->rearrange_filter_tree(filter_tree_root, count);
-
- // &&
- // / \
- // || years>2000
- // / \
- // && \
- // / \ \
- // age>50 rating<5 &&
- // / \
- // age<30 rating>5
- ASSERT_TRUE(filter_tree_root != nullptr);
- ASSERT_TRUE(filter_tree_root->isOperator);
- ASSERT_EQ(filter_tree_root->filter_operator, AND);
-
- root = filter_tree_root->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_TRUE(root->isOperator);
- ASSERT_EQ(root->filter_operator, OR);
-
- root = filter_tree_root->left->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_TRUE(root->isOperator);
- ASSERT_EQ(root->filter_operator, AND);
-
- root = filter_tree_root->left->left->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "age");
- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "50");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->left->left->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "rating");
- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "5");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->left->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_TRUE(root->isOperator);
- ASSERT_EQ(root->filter_operator, AND);
-
- root = filter_tree_root->left->right->left;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "age");
- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "30");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->left->right->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "rating");
- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);
- ASSERT_EQ(root->filter_exp.values.front(), "5");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- root = filter_tree_root->right;
- ASSERT_TRUE(root != nullptr);
- ASSERT_FALSE(root->isOperator);
- ASSERT_EQ(root->filter_exp.field_name, "years");
- ASSERT_TRUE(root->left == nullptr);
- ASSERT_TRUE(root->right == nullptr);
-
- collectionManager.drop_collection("Collection");
-}
-
-TEST_F(CollectionSpecificMoreTest, ApproxFilterMatchCount) {
- nlohmann::json schema =
- R"({
- "name": "Collection",
- "fields": [
- {"name": "name", "type": "string"},
- {"name": "age", "type": "int32"},
- {"name": "years", "type": "int32[]"},
- {"name": "rating", "type": "float"},
- {"name": "location", "type": "geopoint", "optional": true}
- ]
- })"_json;
-
- Collection *coll = collectionManager.create_collection(schema).get();
- std::ifstream infile(std::string(ROOT_DIR) + "test/numeric_array_documents.jsonl");
- std::string json_line;
- while (std::getline(infile, json_line)) {
- auto add_op = coll->add(json_line);
- ASSERT_TRUE(add_op.ok());
+ for(auto i = 0; i < 5; i++) {
+ nlohmann::json doc;
+ doc["title"] = std::to_string(10 + i);
+ doc["points"] = i;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
}
- infile.close();
- const std::string doc_id_prefix = std::to_string(coll->get_collection_id()) + "_" + Collection::DOC_ID_PREFIX + "_";
- filter_node_t* filter_tree_root = nullptr;
- Option<bool> filter_op = filter::parse_filter_query("name: Jeremy", coll->get_schema(), store, doc_id_prefix,
- filter_tree_root);
- ASSERT_TRUE(filter_op.ok());
+/* Values Doc ids
+ -10 0
+ -9 1
+ -8 2
+ -7 3
+ -6 4
+ -5 5
+ -4 6
+ -3 7
+ -2 8
+ -1 9
+ 0 10, 15
+ 1 11, 16
+ 2 12, 17
+ 3 13, 18
+ 4 14, 19
+ */
+
+ auto results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();
+ ASSERT_EQ(20, results["found"]);
+
+ coll1->truncate_after_top_k("points", 15);
+
+ results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();
+ ASSERT_EQ(15, results["found"]);
+
+ std::vector<std::string> ids = {"19", "18", "17", "16", "15", "14", "13", "12", "11", "10", "9", "8", "7", "6", "5"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ ASSERT_EQ(ids[i], results["hits"][i]["document"]["id"]);
+ }
- uint32_t approx_count = 0;
- coll->_get_index()->_approximate_filter_ids(filter_tree_root->filter_exp, approx_count);
- ASSERT_EQ(approx_count, 5);
+ coll1->truncate_after_top_k("points", 11);
- delete filter_tree_root;
- filter_op = filter::parse_filter_query("location:(48.8662, 2.3255, 48.8581, 2.3209, 48.8561, 2.3448, 48.8641, 2.3469)",
- coll->get_schema(), store, doc_id_prefix, filter_tree_root);
- ASSERT_TRUE(filter_op.ok());
+ results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();
+ ASSERT_EQ(11, results["found"]);
- coll->_get_index()->_approximate_filter_ids(filter_tree_root->filter_exp, approx_count);
- ASSERT_EQ(approx_count, 100);
+ ids = {"19", "18", "17", "16", "15", "14", "13", "12", "11", "10", "9"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ ASSERT_EQ(ids[i], results["hits"][i]["document"]["id"]);
+ }
- delete filter_tree_root;
- filter_op = filter::parse_filter_query("years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5))",
- coll->get_schema(), store, doc_id_prefix, filter_tree_root);
- ASSERT_TRUE(filter_op.ok());
+ coll1->truncate_after_top_k("points", 5);
- coll->_get_index()->rearrange_filter_tree(filter_tree_root, approx_count);
- ASSERT_EQ(approx_count, 3);
+ results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();
+ ASSERT_EQ(5, results["found"]);
- delete filter_tree_root;
- collectionManager.drop_collection("Collection");
+ ids = {"19", "18", "14", "13", "12"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ ASSERT_EQ(ids[i], results["hits"][i]["document"]["id"]);
+ }
}
+
+ TEST_F(CollectionSpecificMoreTest, HybridSearchTextMatchInfo) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_id", "type": "string"},
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "product_description", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "product_id": "product_a",
+ "product_name": "shampoo",
+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."
+ })"_json,
+ R"({
+ "product_id": "product_b",
+ "product_name": "soap",
+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."
+ })"_json
+ };
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ auto coll1 = collection_create_op.get();
+ auto results = coll1->search("natural products", {"product_name", "embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+
+ // It's a hybrid search with only vector match
+ ASSERT_EQ("0", results["hits"][0]["text_match_info"]["score"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["text_match_info"]["score"].get<std::string>());
+
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["fields_matched"].get<size_t>());
+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["fields_matched"].get<size_t>());
+
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["tokens_matched"].get<size_t>());
+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["tokens_matched"].get<size_t>());
+ }
-
-