Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
More tests for grouping.
kishorencde28d33f2436
diff --git a/src/collection.cpp b/src/collection.cppindex 0f7336f7..1c244855 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -351,8 +351,6 @@ void Collection::populate_overrides(std::string query,include_ids[hit.position].push_back(seq_id);}}-- break;}}@@ -805,8 +803,6 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::// for grouping we have to aggregate group set sizes to a count valueif(group_limit) {- LOG(INFO) << "override_result_kvs size: " << override_result_kvs.size();-for(auto& acc_facet: facets) {for(auto& facet_kv: acc_facet.result_map) {facet_kv.second.count = facet_kv.second.groups.size();@@ -877,7 +873,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::group_keys.push_back(group_key);}- group_hits["group_key"] = StringUtils::join(group_keys, ":");+ group_hits["group_key"] = StringUtils::join(group_keys, ",");}nlohmann::json& hits_array = (group_limit > 1) ? group_hits["hits"] : result["hits"];@@ -1391,7 +1387,7 @@ Option<uint32_t> Collection::add_override(const override_t & override) {return Option<uint32_t>(500, "Error while storing the override on disk.");}- overrides[override.id] = override;+ overrides.emplace(override.id, override);return Option<uint32_t>(200);}diff --git a/src/index.cpp b/src/index.cppindex 4e50fb6d..6384228f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1077,7 +1077,7 @@ void Index::collate_included_ids(const std::string & query, const std::string &uint64_t distinct_id = outer_pos; // outer pos is the group distinct keyuint64_t match_score = (64000 - inner_pos); // inner pos within a group is the match score- LOG(INFO) << "seq_id: " << seq_id << " - " << match_score;+ // LOG(INFO) << "seq_id: " << seq_id << " - " << match_score;int64_t scores[3];scores[0] = match_score;diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 8a9a3849..f3e0e6a2 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -10,6 +10,7 @@ class CollectionGroupingTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ Collection *coll_group;void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/collection_grouping";@@ -19,6 +20,33 @@ protected:store = new Store(state_dir_path);collectionManager.init(store, 4, "auth_key");collectionManager.load();++ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("brand", field_types::STRING, true, true),+ field("size", field_types::INT32, true, false),+ field("colors", field_types::STRING_ARRAY, true, false),+ field("rating", field_types::FLOAT, true, false)+ };++ coll_group = collectionManager.get_collection("coll_group");+ if(coll_group == nullptr) {+ coll_group = collectionManager.create_collection("coll_group", fields, "rating").get();+ }++ std::ifstream infile(std::string(ROOT_DIR)+"test/group_documents.jsonl");++ std::string json_line;++ while (std::getline(infile, json_line)) {+ auto add_op = coll_group->add(json_line);+ if(!add_op.ok()) {+ std::cout << add_op.error() << std::endl;+ }+ ASSERT_TRUE(add_op.ok());+ }++ infile.close();}virtual void SetUp() {@@ -31,39 +59,248 @@ protected:}};-TEST_F(CollectionGroupingTest, GroupingOnOptionalIntegerArray) {- Collection *coll_group;+TEST_F(CollectionGroupingTest, GroupingBasics) {+ // group by size (int32)+ auto res = coll_group->search("*", {}, "", {"brand"}, {}, 0, 50, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30,+ "", 10,+ {}, {}, {"size"}, 2).get();- std::vector<field> fields = {- field("title", field_types::STRING, false),- field("description", field_types::STRING, true, true),- field("max", field_types::INT32, false),- field("scores", field_types::INT64_ARRAY, true, true),- field("average", field_types::FLOAT, false, true),- field("is_valid", field_types::BOOL, false, true),- };+ ASSERT_EQ(3, res["found"].get<size_t>());+ ASSERT_EQ(3, res["grouped_hits"].size());+ ASSERT_STREQ("size", res["grouped_hits"][0]["group_key"].get<std::string>().c_str());++ ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());+ ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][0]["hits"][1]["document"]["rating"].get<float>());+ ASSERT_STREQ("1", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());++ ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());+ ASSERT_STREQ("4", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][1]["hits"][1]["document"]["rating"].get<float>());+ ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());++ ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][2]["hits"][0]["document"]["rating"].get<float>());+ ASSERT_STREQ("2", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][2]["hits"][1]["document"]["rating"].get<float>());+ ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());++ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());++ ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());++ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);+ ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());++ // group by rating (float) and sort by size+ std::vector<sort_by> sort_size = {sort_by("size", "DESC")};+ res = coll_group->search("*", {}, "", {"brand"}, sort_size, 0, 50, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "brand: omeg", 30,+ "", 10,+ {}, {}, {"rating"}, 2).get();++ // 7 unique ratings+ ASSERT_EQ(7, res["found"].get<size_t>());+ ASSERT_EQ(7, res["grouped_hits"].size());+ ASSERT_STREQ("rating", res["grouped_hits"][0]["group_key"].get<std::string>().c_str());++ ASSERT_EQ(12, res["grouped_hits"][0]["hits"][0]["document"]["size"].get<uint32_t>());+ ASSERT_STREQ("8", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());++ ASSERT_EQ(12, res["grouped_hits"][1]["hits"][0]["document"]["size"].get<uint32_t>());+ ASSERT_STREQ("6", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());++ ASSERT_EQ(11, res["grouped_hits"][1]["hits"][1]["document"]["size"].get<uint32_t>());+ ASSERT_STREQ("1", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][1]["hits"][1]["document"]["rating"].get<float>());++ ASSERT_EQ(10, res["grouped_hits"][5]["hits"][0]["document"]["size"].get<uint32_t>());+ ASSERT_STREQ("9", res["grouped_hits"][5]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.1, res["grouped_hits"][5]["hits"][0]["document"]["rating"].get<float>());++ ASSERT_EQ(10, res["grouped_hits"][6]["hits"][0]["document"]["size"].get<uint32_t>());+ ASSERT_STREQ("0", res["grouped_hits"][6]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.5, res["grouped_hits"][6]["hits"][0]["document"]["rating"].get<float>());++ ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Omeg</mark>a", res["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+}++TEST_F(CollectionGroupingTest, GroupingCompoundKey) {+ // group by size+brand (int32, string)+ auto res = coll_group->search("*", {}, "", {"brand"}, {}, 0, 50, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30,+ "", 10,+ {}, {}, {"size", "brand"}, 2).get();++ ASSERT_EQ(10, res["found"].get<size_t>());+ ASSERT_EQ(10, res["grouped_hits"].size());+ ASSERT_STREQ("size,brand", res["grouped_hits"][0]["group_key"].get<std::string>().c_str());++ ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());+ ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());+ ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());- coll_group = collectionManager.get_collection("coll_group");- if(coll_group == nullptr) {- coll_group = collectionManager.create_collection("coll_group", fields, "max").get();+ ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());+ ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());+ ASSERT_STREQ("4", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());++ ASSERT_EQ(2, res["grouped_hits"][2]["hits"].size());+ ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][2]["hits"][0]["document"]["rating"].get<float>());+ ASSERT_STREQ("3", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.5, res["grouped_hits"][2]["hits"][1]["document"]["rating"].get<float>());+ ASSERT_STREQ("0", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());++ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());++ ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());++ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);+ ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());++ // pagination with page=2, per_page=2+ res = coll_group->search("*", {}, "", {"brand"}, {}, 0, 2, 2, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30,+ "", 10,+ {}, {}, {"size", "brand"}, 2).get();+++ // 3rd result from previous assertion will be in the first position+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());+ ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());+ ASSERT_STREQ("3", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_FLOAT_EQ(4.5, res["grouped_hits"][0]["hits"][1]["document"]["rating"].get<float>());+ ASSERT_STREQ("0", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());++ // total count and facet counts should be the same+ ASSERT_EQ(10, res["found"].get<size_t>());+ ASSERT_EQ(2, res["grouped_hits"].size());+ ASSERT_STREQ("size,brand", res["grouped_hits"][0]["group_key"].get<std::string>().c_str());++ ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());++ ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());++ ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());++ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);+ ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+}++TEST_F(CollectionGroupingTest, GroupingWithSingleDistinct) {+ auto res = coll_group->search("*", {}, "", {"brand"}, {}, 0, 50, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30,+ "", 10,+ {}, {}, {"brand"}, 1).get();++ ASSERT_EQ(5, res["found"].get<size_t>());+ ASSERT_EQ(5, res["hits"].size());++ ASSERT_STREQ("4", res["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("2", res["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("8", res["hits"][2]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("10", res["hits"][3]["document"]["id"].get<std::string>().c_str()); // unbranded+ ASSERT_STREQ("9", res["hits"][4]["document"]["id"].get<std::string>().c_str());++ // facet counts should each be 1, including unbranded+ ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());++ for(size_t i=0; i < 4; i++) {+ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][i]["count"]);}+}++TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {+ nlohmann::json override_json_include = {+ {"id", "include-rule"},+ {+ "rule", {+ {"query", "shirt"},+ {"match", override_t::MATCH_EXACT}+ }+ }+ };- std::ifstream infile(std::string(ROOT_DIR)+"test/optional_fields.jsonl");+ override_json_include["includes"] = nlohmann::json::array();+ override_json_include["includes"][0] = nlohmann::json::object();+ override_json_include["includes"][0]["id"] = "11";+ override_json_include["includes"][0]["position"] = 1;- std::string json_line;+ override_json_include["includes"][1] = nlohmann::json::object();+ override_json_include["includes"][1]["id"] = "10";+ override_json_include["includes"][1]["position"] = 1;- while (std::getline(infile, json_line)) {- auto add_op = coll_group->add(json_line);- if(!add_op.ok()) {- std::cout << add_op.error() << std::endl;+ nlohmann::json override_json_exclude = {+ {"id", "exclude-rule"},+ {+ "rule", {+ {"query", "shirt"},+ {"match", override_t::MATCH_EXACT}+ }}- ASSERT_TRUE(add_op.ok());- }+ };+ override_json_exclude["excludes"] = nlohmann::json::array();+ override_json_exclude["excludes"][0] = nlohmann::json::object();+ override_json_exclude["excludes"][0]["id"] = "2";++ override_t override1(override_json_include);+ override_t override2(override_json_exclude);+ Option<uint32_t> ov1_op = coll_group->add_override(override1);+ Option<uint32_t> ov2_op = coll_group->add_override(override2);++ ASSERT_TRUE(ov1_op.ok());+ ASSERT_TRUE(ov2_op.ok());++ auto res = coll_group->search("shirt", {"title"}, "", {"brand"}, {}, 0, 50, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30,+ "", 10,+ {}, {}, {"colors"}, 2).get();++ ASSERT_EQ(4, res["found"].get<size_t>());+ ASSERT_EQ(4, res["grouped_hits"].size());+ ASSERT_STREQ("colors", res["grouped_hits"][0]["group_key"].get<std::string>().c_str());++ ASSERT_STREQ("11", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("10", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());- infile.close();+ ASSERT_STREQ("5", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());- // first must be able to fetch all records (i.e. all must have been index)+ ASSERT_STREQ("4", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());- auto res = coll_group->search("*", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(6, res["found"].get<size_t>());+ ASSERT_EQ(1, res["grouped_hits"][3]["hits"].size());+ ASSERT_STREQ("8", res["grouped_hits"][3]["hits"][0]["document"]["id"].get<std::string>().c_str());}\ No newline at end of filediff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex 30d223da..a36fb85a 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -377,4 +377,16 @@ TEST_F(CollectionOverrideTest, PinnedHitsGrouping) {pinned_hits, {}, {"cast"}, 2).get();ASSERT_EQ(8, results["found"].get<size_t>());++ ASSERT_STREQ("cast", results["grouped_hits"][0]["group_key"].get<std::string>().c_str());+ ASSERT_STREQ("6", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("8", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("1", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("13", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("4", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("11", results["grouped_hits"][3]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("16", results["grouped_hits"][4]["hits"][0]["document"]["id"].get<std::string>().c_str());}\ No newline at end of filediff --git a/test/group_documents.jsonl b/test/group_documents.jsonlnew file mode 100644index 00000000..5d673ff0--- /dev/null+++ b/test/group_documents.jsonl@@ -0,0 +1,12 @@+{"title": "Omega Casual Poplin Shirt", "brand": "Omega", "size": 10, "colors": ["white", "blue"], "rating": 4.5}+{"title": "Omega Casual Poplin Shirt", "brand": "Omega", "size": 11, "colors": ["white", "blue"], "rating": 4.3}+{"title": "Omega Casual Poplin Shirt", "brand": "Omega", "size": 12, "colors": ["white", "blue"], "rating": 4.6}+{"title": "Omega Casual Poplin Shirt", "brand": "Omega", "size": 10, "colors": ["blue"], "rating": 4.6}+{"title": "Beta Casual Poplin Shirt", "brand": "Beta", "size": 10, "colors": ["white", "blue"], "rating": 4.8}+{"title": "Beta Casual Poplin Shirt", "brand": "Beta", "size": 11, "colors": ["blue"], "rating": 4.8}+{"title": "Beta Casual Poplin Shirt", "brand": "Beta", "size": 12, "colors": ["white", "blue"], "rating": 4.3}+{"title": "Xorp Casual Shirt", "brand": "Xorp", "size": 10, "colors": ["white", "blue"], "rating": 4.3}+{"title": "Xorp Casual Shirt", "brand": "Xorp", "size": 12, "colors": ["white", "red"], "rating": 4.4}+{"title": "Zeta Casual Shirt", "brand": "Zeta", "size": 10, "colors": ["white", "blue"], "rating": 4.1}+{"title": "White Casual Shirt", "size": 10, "colors": ["white"], "rating": 4.3}+{"title": "White Casual Shirt", "size": 10, "colors": ["white"], "rating": 3.3}\ No newline at end of file