Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.24-nested' into v0.25
Kishore Nallanb8b9fb20b3c3
diff --cc include/collection.hindex 19d1fe9b,859967b5..6b3d9e71--- a/include/collection.h+++ b/include/collection.h@@@ -409,8 -409,8 +409,10 @@@ publicconst size_t filter_curated_hits_option = 2,const bool prioritize_token_position = false,const std::string& vector_query_str = "",+ const bool enable_highlight_v1 = true,- const uint64_t search_time_start_us = 0) const;++ const uint64_t search_time_start_us = 0,+ const size_t facet_sample_percent = 100,+ const size_t facet_sample_threshold = 0) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --cc src/collection.cppindex d2c697db,edb6e388..b6ef3405--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -867,8 -867,8 +868,10 @@@ Option<nlohmann::json> Collection::searconst size_t filter_curated_hits_option,const bool prioritize_token_position,const std::string& vector_query_str,+ const bool enable_highlight_v1,- const uint64_t search_time_start_us) const {++ const uint64_t search_time_start_us,+ const size_t facet_sample_percent,+ const size_t facet_sample_threshold) const {std::shared_lock lock(mutex);@@@ -1650,12 -1668,9 +1662,12 @@@for(facet & a_facet: facets) {nlohmann::json facet_result = nlohmann::json::object();facet_result["field_name"] = a_facet.field_name;+ facet_result["sampled"] = a_facet.sampled;facet_result["counts"] = nlohmann::json::array();+ std::vector<facet_value_t> facet_values;- std::vector<std::pair<int64_t, facet_count_t>> facet_hash_counts;-+ std::vector<std::pair<uint64_t, facet_count_t>> facet_hash_counts;++for (const auto & kv : a_facet.result_map) {facet_hash_counts.emplace_back(kv);}diff --cc src/collection_manager.cppindex e04ce8f9,ffbb704f..f51791aa--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -695,9 -697,8 +697,11 @@@ Option<bool> CollectionManager::do_searconst char *EXHAUSTIVE_SEARCH = "exhaustive_search";const char *SPLIT_JOIN_TOKENS = "split_join_tokens";+ const char *ENABLE_HIGHLIGHT_V1 = "enable_highlight_v1";++ const char *FACET_SAMPLE_PERCENT = "facet_sample_percent";+ const char *FACET_SAMPLE_THRESHOLD = "facet_sample_threshold";+// enrich params with values from embedded paramsfor(auto& item: embedded_params.items()) {if(item.key() == "expires_at") {@@@ -773,10 -775,8 +777,11 @@@std::vector<enable_t> infixes;size_t max_extra_prefix = INT16_MAX;size_t max_extra_suffix = INT16_MAX;+ bool enable_highlight_v1 = true;+ size_t facet_sample_percent = 100;+ size_t facet_sample_threshold = 0;+std::unordered_map<std::string, size_t*> unsigned_int_values = {{MIN_LEN_1TYPO, &min_len_1typo},{MIN_LEN_2TYPO, &min_len_2typo},@@@ -990,8 -983,8 +996,10 @@@filter_curated_hits_option,prioritize_token_position,vector_query,+ enable_highlight_v1,- start_ts++ start_ts,+ facet_sample_percent,+ facet_sample_threshold);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --cc test/collection_faceting_test.cppindex 544bddab,497eb588..4edb30b5--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@@ -981,375 -980,35 +981,407 @@@ TEST_F(CollectionFacetingTest, FacetByNASSERT_EQ("2000", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());}+TEST_F(CollectionFacetingTest, FacetParseTest){+ std::vector<field> fields = {- field("score", field_types::INT32, true),- field("grade", field_types::INT32, true),- field("rank", field_types::INT32, true),++ field("score", field_types::INT32, true),++ field("grade", field_types::INT32, true),++ field("rank", field_types::INT32, true),+ };++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ std::vector<std::string> range_facet_fields {- "score(fail:[0, 40], pass:[40, 100])",- "grade(A:[80, 100], B:[60, 80], C:[40, 60])"++ "score(fail:[0, 40], pass:[40, 100])",++ "grade(A:[80, 100], B:[60, 80], C:[40, 60])"+ };+ std::vector<facet> range_facets;+ for(const std::string & facet_field: range_facet_fields) {+ coll1->parse_facet(facet_field, range_facets);+ }+ ASSERT_EQ(2, range_facets.size());-+++ ASSERT_STREQ("score", range_facets[0].field_name.c_str());+ ASSERT_TRUE(range_facets[0].is_range_query);+ ASSERT_GT(range_facets[0].facet_range_map.size(), 0);++ ASSERT_STREQ("grade", range_facets[1].field_name.c_str());+ ASSERT_TRUE(range_facets[1].is_range_query);+ ASSERT_GT(range_facets[1].facet_range_map.size(), 0);++ std::vector<std::string> normal_facet_fields {- "score",- "grade"++ "score",++ "grade"+ };+ std::vector<facet> normal_facets;+ for(const std::string & facet_field: normal_facet_fields) {+ coll1->parse_facet(facet_field, normal_facets);+ }+ ASSERT_EQ(2, normal_facets.size());++ ASSERT_STREQ("score", normal_facets[0].field_name.c_str());+ ASSERT_STREQ("grade", normal_facets[1].field_name.c_str());++ std::vector<std::string> mixed_facet_fields {- "score",- "grade(A:[80, 100], B:[60, 80], C:[40, 60])",- "rank"++ "score",++ "grade(A:[80, 100], B:[60, 80], C:[40, 60])",++ "rank"+ };+ std::vector<facet> mixed_facets;+ for(const std::string & facet_field: mixed_facet_fields) {+ coll1->parse_facet(facet_field, mixed_facets);+ }+ ASSERT_EQ(3, mixed_facets.size());-+++ ASSERT_STREQ("score", mixed_facets[0].field_name.c_str());-+++ ASSERT_STREQ("grade", mixed_facets[1].field_name.c_str());+ ASSERT_TRUE(mixed_facets[1].is_range_query);+ ASSERT_GT(mixed_facets[1].facet_range_map.size(), 0);++ ASSERT_STREQ("rank", mixed_facets[2].field_name.c_str());+}+-+TEST_F(CollectionFacetingTest, RangeFacetTest) {+ std::vector<field> fields = {field("place", field_types::STRING, false),+ field("state", field_types::STRING, false),+ field("visitors", field_types::INT32, true),};+ Collection* coll1 = collectionManager.create_collection(+ "coll1", 1, fields, "", 0, "", {}, {}+ ).get();+ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["place"] = "Mysore Palace";+ doc1["state"] = "Karnataka";+ doc1["visitors"] = 235486;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["place"] = "Hampi";+ doc2["state"] = "Karnataka";+ doc2["visitors"] = 187654;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["place"] = "Mahabalipuram";+ doc3["state"] = "TamilNadu";+ doc3["visitors"] = 174684;++ nlohmann::json doc4;+ doc4["id"] = "3";+ doc4["place"] = "Meenakshi Amman Temple";+ doc4["state"] = "TamilNadu";+ doc4["visitors"] = 246676;++ nlohmann::json doc5;+ doc5["id"] = "4";+ doc5["place"] = "Staue of Unity";+ doc5["state"] = "Gujarat";+ doc5["visitors"] = 345878;+++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());++ auto results = coll1->search("Karnataka", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true).get();+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("Busy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());++ auto results2 = coll1->search("Gujarat", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true).get();+ ASSERT_EQ(1, results2["facet_counts"][0]["counts"].size());+ ASSERT_EQ(1, results2["facet_counts"][0]["counts"][0]["count"].get<std::size_t>());+ ASSERT_STREQ("VeryBusy", results2["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_TRUE(results2["facet_counts"][0]["counts"][1]["value"] == nullptr);++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionFacetingTest, RangeFacetContinuity) {+ std::vector<field> fields = {field("place", field_types::STRING, false),+ field("state", field_types::STRING, false),+ field("visitors", field_types::INT32, true),};+ Collection* coll1 = collectionManager.create_collection(+ "coll1", 1, fields, "", 0, "", {}, {}+ ).get();+ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["place"] = "Mysore Palace";+ doc1["state"] = "Karnataka";+ doc1["visitors"] = 235486;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["place"] = "Hampi";+ doc2["state"] = "Karnataka";+ doc2["visitors"] = 187654;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["place"] = "Mahabalipuram";+ doc3["state"] = "TamilNadu";+ doc3["visitors"] = 174684;++ nlohmann::json doc4;+ doc4["id"] = "3";+ doc4["place"] = "Meenakshi Amman Temple";+ doc4["state"] = "TamilNadu";+ doc4["visitors"] = 246676;++ nlohmann::json doc5;+ doc5["id"] = "4";+ doc5["place"] = "Staue of Unity";+ doc5["state"] = "Gujarat";+ doc5["visitors"] = 345878;+++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());++ auto results = coll1->search("TamilNadu", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200001, 500000])"},+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ ASSERT_STREQ("Ranges in range facet syntax should be continous.", results.error().c_str());++ auto results2 = coll1->search("TamilNadu", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:[199999, 500000])"},+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ ASSERT_STREQ("Ranges in range facet syntax should be continous.", results2.error().c_str());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionFacetingTest, RangeFacetTypo) {+ std::vector<field> fields = {field("place", field_types::STRING, false),+ field("state", field_types::STRING, false),+ field("visitors", field_types::INT32, true),};+ Collection* coll1 = collectionManager.create_collection(+ "coll1", 1, fields, "", 0, "", {}, {}+ ).get();+ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["place"] = "Mysore Palace";+ doc1["state"] = "Karnataka";+ doc1["visitors"] = 235486;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["place"] = "Hampi";+ doc2["state"] = "Karnataka";+ doc2["visitors"] = 187654;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["place"] = "Mahabalipuram";+ doc3["state"] = "TamilNadu";+ doc3["visitors"] = 174684;++ nlohmann::json doc4;+ doc4["id"] = "3";+ doc4["place"] = "Meenakshi Amman Temple";+ doc4["state"] = "TamilNadu";+ doc4["visitors"] = 246676;++ nlohmann::json doc5;+ doc5["id"] = "4";+ doc5["place"] = "Staue of Unity";+ doc5["state"] = "Gujarat";+ doc5["visitors"] = 345878;+++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());++ auto results = coll1->search("TamilNadu", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000)"}, //missing ']' at end+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ ASSERT_STREQ("Error splitting the range string.", results.error().c_str());++ auto results2 = coll1->search("TamilNadu", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:200000, 500000])"}, //missing '[' in second range+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ ASSERT_STREQ("Error splitting the range string.", results2.error().c_str());++ auto results3 = coll1->search("TamilNadu", {"state"},+ "", {"visitors(Busy:[0, 200000] VeryBusy:[200000, 500000])"}, //missing ',' between ranges+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ ASSERT_STREQ("Error splitting the range string.", results3.error().c_str());++ auto results4 = coll1->search("TamilNadu", {"state"},+ "", {"visitors(Busy:[0 200000], VeryBusy:[200000, 500000])"}, //missing ',' between first ranges values+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ ASSERT_STREQ("Range String range pattern not matched.", results4.error().c_str());++ auto results5 = coll1->search("TamilNadu", {"state"},+ "", {"visitors(Busy:[0, 200000 VeryBusy:200000, 500000])"}, //missing '],' and '['+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ ASSERT_STREQ("Range String range pattern not matched.", results5.error().c_str());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionFacetingTest, SampleFacetCounts) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "color", "type": "string", "facet": true}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ std::mt19937 gen(137723); // use constant seed to make sure that counts don't jump around+ std::uniform_int_distribution<> distr(1, 100); // 1 to 100 inclusive++ size_t count_blue = 0, count_red = 0;++ for(size_t i = 0; i < 1000; i++) {+ nlohmann::json doc;+ if(distr(gen) % 2 == 0) {+ doc["color"] = "blue";+ count_blue++;+ } else {+ doc["color"] = "red";+ count_red++;+ }++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto res = coll1->search("*", {}, "", {"color"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 3, 3, 2, 2, false, "", 10, 0).get();++ ASSERT_EQ(1000, res["found"].get<size_t>());+ ASSERT_EQ(1, res["facet_counts"].size());+ ASSERT_EQ(2, res["facet_counts"][0]["counts"].size());++ // verify approximate counts+ ASSERT_GE(res["facet_counts"][0]["counts"][0]["count"].get<size_t>(), 250);+ ASSERT_GE(res["facet_counts"][0]["counts"][1]["count"].get<size_t>(), 250);+ ASSERT_TRUE(res["facet_counts"][0]["sampled"].get<bool>());++ // when sample threshold is high, don't estimate+ res = coll1->search("*", {}, "", {"color"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 3, 3, 2, 2, false, "", 10, 10000).get();++ ASSERT_EQ(1000, res["found"].get<size_t>());+ ASSERT_EQ(1, res["facet_counts"].size());+ ASSERT_EQ(2, res["facet_counts"][0]["counts"].size());++ for(size_t i = 0; i < res["facet_counts"][0]["counts"].size(); i++) {+ if(res["facet_counts"][0]["counts"][i]["value"].get<std::string>() == "red") {+ ASSERT_EQ(count_red, res["facet_counts"][0]["counts"][i]["count"].get<size_t>());+ } else {+ ASSERT_EQ(count_blue, res["facet_counts"][0]["counts"][i]["count"].get<size_t>());+ }+ }++ ASSERT_FALSE(res["facet_counts"][0]["sampled"].get<bool>());++ // test for sample percent > 100++ auto res_op = coll1->search("*", {}, "", {"color"}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,- spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,- 4, {off}, 3, 3, 2, 2, false, "", 200, 0);++ spp::sparse_hash_set<std::string>(),++ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,++ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,++ 4, {off}, 3, 3, 2, 2, false, "", 200, 0);++ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Value of `facet_sample_percent` must be less than 100.", res_op.error());+}+++ TEST_F(CollectionFacetingTest, FacetOnArrayFieldWithSpecialChars) {+ std::vector<field> fields = {- field("tags", field_types::STRING_ARRAY, true),- field("points", field_types::INT32, true),++ field("tags", field_types::STRING_ARRAY, true),++ field("points", field_types::INT32, true),+ };++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["tags"] = {"gamma"};+ doc["points"] = 10;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["tags"] = {"alpha", "| . |", "beta", "gamma"};+ doc["points"] = 10;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("*", {},+ "", {"tags"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(4, results["facet_counts"][0]["counts"].size());++ for(size_t i = 0; i < results["facet_counts"][0]["counts"].size(); i++) {+ auto fvalue = results["facet_counts"][0]["counts"][i]["value"].get<std::string>();+ if(fvalue == "gamma") {+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][i]["count"].get<size_t>());+ } else {+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][i]["count"].get<size_t>());+ }+ }+ }