Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-facets
Kishore Nallan2e2448288213
diff --cc src/index.cppindex 0e93d6ad,4d81126a..de07a27a--- a/src/index.cpp+++ b/src/index.cpp@@@ -2765,16 -3235,8 +2766,17 @@@ Option<bool> Index::search(std::vector<int64_t match_score_index = -1;compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);+ kv.text_match_score = 0;kv.vector_distance = vec_result.second;++ if (filter_result_iterator->is_valid &&+ !filter_result_iterator->reference.empty()) {+ // The doc_id must be valid otherwise it would've been filtered out upstream.+ filter_result_iterator->skip_to(doc_id);+ kv.reference_filter_results = std::move(filter_result_iterator->reference);+ filter_result_iterator->reset();+ }+topster->add(&kv);vec_search_ids.push_back(doc_id);}@@@ -3726,10 -4163,9 +3728,11 @@@ void Index::search_across_fields(const<< ", aggregated_score: " << aggregated_score;*/KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);+ kv.reference_filter_results = std::move(filter_result.reference_filter_results);+if(match_score_index != -1) {kv.scores[match_score_index] = aggregated_score;+ kv.text_match_score = aggregated_score;}int ret = topster->add(&kv);diff --cc test/collection_specific_more_test.cppindex 68980d21,83181283..aa83229e--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@@ -2310,79 -2370,288 +2380,131 @@@ TEST_F(CollectionSpecificMoreTest, CrosASSERT_EQ(1, res["hits"].size());}-TEST_F(CollectionSpecificMoreTest, RearrangingFilterTree) {- nlohmann::json schema =- R"({- "name": "Collection",- "fields": [- {"name": "name", "type": "string"},- {"name": "age", "type": "int32"},- {"name": "years", "type": "int32[]"},- {"name": "rating", "type": "float"}- ]- })"_json;+TEST_F(CollectionSpecificMoreTest, TruncateAterTopK) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "points", "type": "int32"}+ ]+ })"_json;- Collection* coll = collectionManager.create_collection(schema).get();+ Collection* coll1 = collectionManager.create_collection(schema).get();- std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");- std::string json_line;- while (std::getline(infile, json_line)) {- auto add_op = coll->add(json_line);- ASSERT_TRUE(add_op.ok());+ for(auto i = -10; i < 5; i++) {+ nlohmann::json doc;+ doc["title"] = std::to_string(i);+ doc["points"] = i;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());}- infile.close();-- const std::string doc_id_prefix = std::to_string(coll->get_collection_id()) + "_" + Collection::DOC_ID_PREFIX + "_";- filter_node_t* filter_tree_root = nullptr;- Option<bool> filter_op = filter::parse_filter_query("years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5))",- coll->get_schema(), store, doc_id_prefix, filter_tree_root);- ASSERT_TRUE(filter_op.ok());- std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);-- // &&- // / \- // years>2000 ||- // 4 / \- // / &&- // && / \- // / \ age>50 rating<5- // / \ 1 2- // / \- // age<30 rating>5- // 2 3- ASSERT_TRUE(filter_tree_root != nullptr);- ASSERT_TRUE(filter_tree_root->isOperator);- ASSERT_EQ(filter_tree_root->filter_operator, AND);-- auto root = filter_tree_root->left;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "years");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->right;- ASSERT_TRUE(root != nullptr);- ASSERT_TRUE(root->isOperator);- ASSERT_EQ(root->filter_operator, OR);-- root = filter_tree_root->right->left;- ASSERT_TRUE(root != nullptr);- ASSERT_TRUE(root->isOperator);- ASSERT_EQ(root->filter_operator, AND);-- root = filter_tree_root->right->left->left;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "age");- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "30");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->right->left->right;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "rating");- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "5");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->right->right;- ASSERT_TRUE(root != nullptr);- ASSERT_TRUE(root->isOperator);- ASSERT_EQ(root->filter_operator, AND);-- root = filter_tree_root->right->right->left;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "age");- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "50");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->right->right->right;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "rating");- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "5");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- uint32_t count = 0;- coll->_get_index()->rearrange_filter_tree(filter_tree_root, count);-- // &&- // / \- // || years>2000- // / \- // && \- // / \ \- // age>50 rating<5 &&- // / \- // age<30 rating>5- ASSERT_TRUE(filter_tree_root != nullptr);- ASSERT_TRUE(filter_tree_root->isOperator);- ASSERT_EQ(filter_tree_root->filter_operator, AND);-- root = filter_tree_root->left;- ASSERT_TRUE(root != nullptr);- ASSERT_TRUE(root->isOperator);- ASSERT_EQ(root->filter_operator, OR);-- root = filter_tree_root->left->left;- ASSERT_TRUE(root != nullptr);- ASSERT_TRUE(root->isOperator);- ASSERT_EQ(root->filter_operator, AND);-- root = filter_tree_root->left->left->left;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "age");- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "50");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->left->left->right;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "rating");- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "5");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->left->right;- ASSERT_TRUE(root != nullptr);- ASSERT_TRUE(root->isOperator);- ASSERT_EQ(root->filter_operator, AND);-- root = filter_tree_root->left->right->left;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "age");- ASSERT_EQ(root->filter_exp.comparators.front(), LESS_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "30");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->left->right->right;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "rating");- ASSERT_EQ(root->filter_exp.comparators.front(), GREATER_THAN);- ASSERT_EQ(root->filter_exp.values.front(), "5");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- root = filter_tree_root->right;- ASSERT_TRUE(root != nullptr);- ASSERT_FALSE(root->isOperator);- ASSERT_EQ(root->filter_exp.field_name, "years");- ASSERT_TRUE(root->left == nullptr);- ASSERT_TRUE(root->right == nullptr);-- collectionManager.drop_collection("Collection");-}--TEST_F(CollectionSpecificMoreTest, ApproxFilterMatchCount) {- nlohmann::json schema =- R"({- "name": "Collection",- "fields": [- {"name": "name", "type": "string"},- {"name": "age", "type": "int32"},- {"name": "years", "type": "int32[]"},- {"name": "rating", "type": "float"},- {"name": "location", "type": "geopoint", "optional": true}- ]- })"_json;-- Collection *coll = collectionManager.create_collection(schema).get();- std::ifstream infile(std::string(ROOT_DIR) + "test/numeric_array_documents.jsonl");- std::string json_line;- while (std::getline(infile, json_line)) {- auto add_op = coll->add(json_line);- ASSERT_TRUE(add_op.ok());+ for(auto i = 0; i < 5; i++) {+ nlohmann::json doc;+ doc["title"] = std::to_string(10 + i);+ doc["points"] = i;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());}- infile.close();- const std::string doc_id_prefix = std::to_string(coll->get_collection_id()) + "_" + Collection::DOC_ID_PREFIX + "_";- filter_node_t* filter_tree_root = nullptr;- Option<bool> filter_op = filter::parse_filter_query("name: Jeremy", coll->get_schema(), store, doc_id_prefix,- filter_tree_root);- ASSERT_TRUE(filter_op.ok());+/* Values Doc ids+ -10 0+ -9 1+ -8 2+ -7 3+ -6 4+ -5 5+ -4 6+ -3 7+ -2 8+ -1 9+ 0 10, 15+ 1 11, 16+ 2 12, 17+ 3 13, 18+ 4 14, 19+ */++ auto results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();+ ASSERT_EQ(20, results["found"]);++ coll1->truncate_after_top_k("points", 15);++ results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();+ ASSERT_EQ(15, results["found"]);++ std::vector<std::string> ids = {"19", "18", "17", "16", "15", "14", "13", "12", "11", "10", "9", "8", "7", "6", "5"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ ASSERT_EQ(ids[i], results["hits"][i]["document"]["id"]);+ }- uint32_t approx_count = 0;- coll->_get_index()->_approximate_filter_ids(filter_tree_root->filter_exp, approx_count);- ASSERT_EQ(approx_count, 5);+ coll1->truncate_after_top_k("points", 11);- delete filter_tree_root;- filter_op = filter::parse_filter_query("location:(48.8662, 2.3255, 48.8581, 2.3209, 48.8561, 2.3448, 48.8641, 2.3469)",- coll->get_schema(), store, doc_id_prefix, filter_tree_root);- ASSERT_TRUE(filter_op.ok());+ results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();+ ASSERT_EQ(11, results["found"]);- coll->_get_index()->_approximate_filter_ids(filter_tree_root->filter_exp, approx_count);- ASSERT_EQ(approx_count, 100);+ ids = {"19", "18", "17", "16", "15", "14", "13", "12", "11", "10", "9"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ ASSERT_EQ(ids[i], results["hits"][i]["document"]["id"]);+ }- delete filter_tree_root;- filter_op = filter::parse_filter_query("years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5))",- coll->get_schema(), store, doc_id_prefix, filter_tree_root);- ASSERT_TRUE(filter_op.ok());+ coll1->truncate_after_top_k("points", 5);- coll->_get_index()->rearrange_filter_tree(filter_tree_root, approx_count);- ASSERT_EQ(approx_count, 3);+ results = coll1->search("*", {"*"}, "", {}, {}, {0}).get();+ ASSERT_EQ(5, results["found"]);- delete filter_tree_root;- collectionManager.drop_collection("Collection");+ ids = {"19", "18", "14", "13", "12"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ ASSERT_EQ(ids[i], results["hits"][i]["document"]["id"]);+ }}++ TEST_F(CollectionSpecificMoreTest, HybridSearchTextMatchInfo) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_id", "type": "string"},+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "product_description", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "product_id": "product_a",+ "product_name": "shampoo",+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."+ })"_json,+ R"({+ "product_id": "product_b",+ "product_name": "soap",+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."+ })"_json+ };++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ ASSERT_TRUE(add_op.ok());+ }++ auto coll1 = collection_create_op.get();+ auto results = coll1->search("natural products", {"product_name", "embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ ASSERT_EQ(2, results["hits"].size());++ // It's a hybrid search with only vector match+ ASSERT_EQ("0", results["hits"][0]["text_match_info"]["score"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["text_match_info"]["score"].get<std::string>());++ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["fields_matched"].get<size_t>());+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["fields_matched"].get<size_t>());++ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["tokens_matched"].get<size_t>());+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["tokens_matched"].get<size_t>());+ }--