Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallan4e3374a945e8
diff --cc src/index.cpp
index ee4021a9,6a1c99bf..e4546983
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -1155,41 -1112,19 +1155,42 @@@ void Index::tokenize_string_array(cons
token_to_offsets[last_token].push_back(0);
}
}
++
void Index::initialize_facet_indexes(const field& facet_field) {
- if(facet_field.is_array()) {
- array_mapped_facet_t facet_array;
- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- facet_array[i] = new facet_map_t();
+ facet_index_v4->initialize(facet_field.name);
+}
+
+void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {
+ if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {
+ int32_t val = std::stoi(raw_value);
+ if (val < a_facet.stats.fvmin) {
+ a_facet.stats.fvmin = val;
}
- facet_index_v3.emplace(facet_field.name, facet_array);
- } else {
- array_mapped_single_val_facet_t facet_array;
- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- facet_array[i] = new single_val_facet_map_t();
+ if (val > a_facet.stats.fvmax) {
+ a_facet.stats.fvmax = val;
+ }
+ a_facet.stats.fvsum += val;
+ a_facet.stats.fvcount++;
+ } else if(field_type == field_types::INT64 || field_type == field_types::INT64_ARRAY) {
+ int64_t val = std::stoll(raw_value);
+ if(val < a_facet.stats.fvmin) {
+ a_facet.stats.fvmin = val;
}
- single_val_facet_index_v3.emplace(facet_field.name, facet_array);
+ if(val > a_facet.stats.fvmax) {
+ a_facet.stats.fvmax = val;
+ }
+ a_facet.stats.fvsum += val;
+ a_facet.stats.fvcount++;
+ } else if(field_type == field_types::FLOAT || field_type == field_types::FLOAT_ARRAY) {
+ float val = std::stof(raw_value);
+ if(val < a_facet.stats.fvmin) {
+ a_facet.stats.fvmin = val;
+ }
+ if(val > a_facet.stats.fvmax) {
+ a_facet.stats.fvmax = val;
+ }
+ a_facet.stats.fvsum += val;
+ a_facet.stats.fvcount++;
}
}
diff --cc test/collection_faceting_test.cpp
index 85ace93a,fec66c48..cadc95c8
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@@ -1607,272 -1675,3 +1679,272 @@@ TEST_F(CollectionFacetingTest, FacetOnA
}
}
}
+
+TEST_F(CollectionFacetingTest, FloatFieldValueTruncation) {
+ std::vector<field> fields = {
+ field("tags", field_types::STRING_ARRAY, true),
+ field("points", field_types::FLOAT, true),
+ };
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["tags"] = {"gamma"};
+ doc["points"] = 300;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("*", {},
+ "", {"points"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+
+ ASSERT_EQ("300", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
+}
+
+TEST_F(CollectionFacetingTest, FacetingReturnParent) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value.color", "type": "string", "optional": false, "facet": true },
+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "value": {
+ "color": "red",
+ "r": 255,
+ "g": 0,
+ "b": 0
+ }
+ })"_json;
+
+ nlohmann::json doc2 = R"({
+ "value": {
+ "color": "blue",
+ "r": 0,
+ "g": 0,
+ "b": 255
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_op = coll1->search("*", {},"", {"value.color"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"value.color"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["value"]);
+
+ //not passing facet_fields in facet_return_parent list will only return facet value, not immediate parent for those field
+ search_op = coll1->search("*", {},"", {"value.color"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);
+
+ search_op = coll1->search("*", {},"", {"value.color", "value.r"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"value.r"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ results = search_op.get();
+ ASSERT_EQ(2, results["facet_counts"].size());
+
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);
+
+ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][1]["counts"][1]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetingReturnParentDeepNested) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "product.specification.detail.width", "type": "int32", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "product" : {
+ "specification": {
+ "detail" : {
+ "width": 25
+ }
+ }
+ }
+ })"_json;
+
+ nlohmann::json doc2 = R"({
+ "product" : {
+ "specification": {
+ "detail" : {
+ "width": 30
+ }
+ }
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_op = coll1->search("*", {},"", {"product.specification.detail.width"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"product.specification.detail.width"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":30}}}", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":25}}}", results["facet_counts"][0]["counts"][1]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetingReturnParentObject) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value", "type": "object", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "value": {
+ "color": "red",
+ "r": 255,
+ "g": 0,
+ "b": 0
+ }
+ })"_json;
+
+ nlohmann::json doc2 = R"({
+ "value": {
+ "color": "blue",
+ "r": 0,
+ "g": 0,
+ "b": 255
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_op = coll1->search("*", {},"", {"value.color"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"value.color"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["value"]);
- }
++}