Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-facets
Kishore Nallan4e3374a945e8
diff --cc src/index.cppindex ee4021a9,6a1c99bf..e4546983--- a/src/index.cpp+++ b/src/index.cpp@@@ -1155,41 -1112,19 +1155,42 @@@ void Index::tokenize_string_array(constoken_to_offsets[last_token].push_back(0);}}++void Index::initialize_facet_indexes(const field& facet_field) {- if(facet_field.is_array()) {- array_mapped_facet_t facet_array;- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- facet_array[i] = new facet_map_t();+ facet_index_v4->initialize(facet_field.name);+}++void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {+ if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {+ int32_t val = std::stoi(raw_value);+ if (val < a_facet.stats.fvmin) {+ a_facet.stats.fvmin = val;}- facet_index_v3.emplace(facet_field.name, facet_array);- } else {- array_mapped_single_val_facet_t facet_array;- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- facet_array[i] = new single_val_facet_map_t();+ if (val > a_facet.stats.fvmax) {+ a_facet.stats.fvmax = val;+ }+ a_facet.stats.fvsum += val;+ a_facet.stats.fvcount++;+ } else if(field_type == field_types::INT64 || field_type == field_types::INT64_ARRAY) {+ int64_t val = std::stoll(raw_value);+ if(val < a_facet.stats.fvmin) {+ a_facet.stats.fvmin = val;}- single_val_facet_index_v3.emplace(facet_field.name, facet_array);+ if(val > a_facet.stats.fvmax) {+ a_facet.stats.fvmax = val;+ }+ a_facet.stats.fvsum += val;+ a_facet.stats.fvcount++;+ } else if(field_type == field_types::FLOAT || field_type == field_types::FLOAT_ARRAY) {+ float val = std::stof(raw_value);+ if(val < a_facet.stats.fvmin) {+ a_facet.stats.fvmin = val;+ }+ if(val > a_facet.stats.fvmax) {+ a_facet.stats.fvmax = val;+ }+ a_facet.stats.fvsum += val;+ a_facet.stats.fvcount++;}}diff --cc test/collection_faceting_test.cppindex 85ace93a,fec66c48..cadc95c8--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@@ -1607,272 -1675,3 +1679,272 @@@ TEST_F(CollectionFacetingTest, FacetOnA}}}++TEST_F(CollectionFacetingTest, FloatFieldValueTruncation) {+ std::vector<field> fields = {+ field("tags", field_types::STRING_ARRAY, true),+ field("points", field_types::FLOAT, true),+ };++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["tags"] = {"gamma"};+ doc["points"] = 300;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("*", {},+ "", {"points"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());++ ASSERT_EQ("300", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+}++TEST_F(CollectionFacetingTest, FacetingReturnParent) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value.color", "type": "string", "optional": false, "facet": true },+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "value": {+ "color": "red",+ "r": 255,+ "g": 0,+ "b": 0+ }+ })"_json;++ nlohmann::json doc2 = R"({+ "value": {+ "color": "blue",+ "r": 0,+ "g": 0,+ "b": 255+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto search_op = coll1->search("*", {},"", {"value.color"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"value.color"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["value"]);++ //not passing facet_fields in facet_return_parent list will only return facet value, not immediate parent for those field+ search_op = coll1->search("*", {},"", {"value.color"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);++ search_op = coll1->search("*", {},"", {"value.color", "value.r"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"value.r"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ results = search_op.get();+ ASSERT_EQ(2, results["facet_counts"].size());++ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);++ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][1]["counts"][1]["value"]);+}++TEST_F(CollectionFacetingTest, FacetingReturnParentDeepNested) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "product.specification.detail.width", "type": "int32", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "product" : {+ "specification": {+ "detail" : {+ "width": 25+ }+ }+ }+ })"_json;++ nlohmann::json doc2 = R"({+ "product" : {+ "specification": {+ "detail" : {+ "width": 30+ }+ }+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto search_op = coll1->search("*", {},"", {"product.specification.detail.width"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"product.specification.detail.width"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":30}}}", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":25}}}", results["facet_counts"][0]["counts"][1]["value"]);+}++TEST_F(CollectionFacetingTest, FacetingReturnParentObject) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value", "type": "object", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "value": {+ "color": "red",+ "r": 255,+ "g": 0,+ "b": 0+ }+ })"_json;++ nlohmann::json doc2 = R"({+ "value": {+ "color": "blue",+ "r": 0,+ "g": 0,+ "b": 255+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto search_op = coll1->search("*", {},"", {"value.color"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"value.color"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["value"]);- }++}