Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.24-changes' into v0.24-nested

Kishore Nalland748c90b4b40
diff --cc include/collection.h
index c99ca220,7bd40485..3dd7b746
--- a/include/collection.h
+++ b/include/collection.h
@@@ -471,11 -400,11 +471,11 @@@ public
const uint8_t* index_symbols,
size_t snippet_start_offset) ;
- void process_highlight_fields(const std::vector<std::string>& search_fields,
+ void process_highlight_fields(const std::vector<search_field_t>& search_fields,
- const spp::sparse_hash_set<std::string>& exclude_fields,
- const spp::sparse_hash_set<std::string>& include_fields,
- const string& highlight_fields,
- const std::string& highlight_full_fields,
+ const tsl::htrie_set<char>& exclude_fields,
+ const tsl::htrie_set<char>& include_fields,
+ const std::vector<std::string>& highlight_field_names,
+ const std::vector<std::string>& highlight_full_field_names,
const std::vector<enable_t>& infixes,
std::vector<std::string>& q_tokens,
const tsl::htrie_map<char, token_leaf>& qtoken_set,
diff --cc src/collection.cpp
index 83650ee7,b8c04310..42ec3a0b
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -1316,24 -1264,10 +1316,24 @@@ Option<nlohmann::json> Collection::sear
// handle which fields have to be highlighted
std::vector<highlight_field_t> highlight_items;
+ bool has_atleast_one_fully_highlighted_field = false;
+
+ std::vector<std::string> highlight_field_names;
+ StringUtils::split(highlight_fields, highlight_field_names, ",");
+
+ std::vector<std::string> highlight_full_field_names;
+ StringUtils::split(highlight_full_fields, highlight_full_field_names, ",");
if(query != "*") {
- process_highlight_fields(search_fields, include_fields_full, exclude_fields_full,
- process_highlight_fields(weighted_search_fields, include_fields, exclude_fields, highlight_fields, highlight_full_fields,
- infixes, q_tokens, search_params->qtoken_set, highlight_items);
++ process_highlight_fields(weighted_search_fields, include_fields_full, exclude_fields_full,
+ highlight_field_names, highlight_full_field_names, infixes, q_tokens,
+ search_params->qtoken_set, highlight_items);
+
+ for(auto& highlight_item: highlight_items) {
+ if(highlight_item.fully_highlighted) {
+ has_atleast_one_fully_highlighted_field = true;
+ }
+ }
}
nlohmann::json result = nlohmann::json::object();
@@@ -1795,11 -1665,11 +1797,11 @@@ void Collection::populate_text_match_in
info["fields_matched"] = ((match_score << 61) >> (61));
}
- void Collection::process_highlight_fields(const std::vector<std::string>& search_fields,
+ void Collection::process_highlight_fields(const std::vector<search_field_t>& search_fields,
- const spp::sparse_hash_set<std::string>& include_fields,
- const spp::sparse_hash_set<std::string>& exclude_fields,
- const string& highlight_fields,
- const std::string& highlight_full_fields,
+ const tsl::htrie_set<char>& include_fields,
+ const tsl::htrie_set<char>& exclude_fields,
+ const std::vector<std::string>& highlight_field_names,
+ const std::vector<std::string>& highlight_full_field_names,
const std::vector<enable_t>& infixes,
std::vector<std::string>& q_tokens,
const tsl::htrie_map<char, token_leaf>& qtoken_set,
@@@ -1828,9 -1695,9 +1830,9 @@@
}
}
- if(highlight_fields.empty()) {
+ if(highlight_field_names.empty()) {
for(size_t i = 0; i < search_fields.size(); i++) {
- const auto& field_name = search_fields[i];
+ const auto& field_name = search_fields[i].name;
if(exclude_fields.count(field_name) != 0) {
// should not pick excluded field for highlighting (only for implicit highlighting)
continue;
diff --cc test/collection_specific_more_test.cpp
index b4b6aeed,32be11ae..4c2e55c7
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@@ -1035,64 -992,43 +1035,105 @@@ TEST_F(CollectionSpecificMoreTest, Sear
ASSERT_EQ(1, results["hits"].size());
}
+TEST_F(CollectionSpecificMoreTest, UpsertUpdateEmplaceShouldAllRemoveIndex) {
- nlohmann::json schema = R"({
++ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title1", "type": "string", "optional": true},
+ {"name": "title2", "type": "string", "optional": true},
+ {"name": "title3", "type": "string", "optional": true}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto doc1 = R"({
+ "id": "0",
+ "title1": "Foo",
+ "title2": "Bar",
+ "title3": "Baz",
+ "data": "abcdefghijk"
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
+
+ // via upsert
+
+ auto doc_update = R"({
+ "id": "0",
+ "title2": "Bar",
+ "title3": "Baz"
+ })"_json;
+ ASSERT_TRUE(coll1->add(doc_update.dump(), UPSERT).ok());
+
+ auto results = coll1->search("foo", {"title1"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(0, results["found"].get<size_t>());
+
+ results = coll1->search("bar", {"title2"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(3, results["hits"][0]["document"].size());
+
+ // via update, existing index should not be removed because update can send partial doc
+
+ doc_update = R"({
+ "id": "0",
+ "title3": "Baz"
+ })"_json;
+ ASSERT_TRUE(coll1->add(doc_update.dump(), UPDATE).ok());
+
+ results = coll1->search("bar", {"title2"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ // via emplace, existing index should not be removed because emplace could send partial doc
+
+ doc_update = R"({
+ "id": "0"
+ })"_json;
+ ASSERT_TRUE(coll1->add(doc_update.dump(), EMPLACE).ok());
+
+ results = coll1->search("baz", {"title3"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+}
++
+ TEST_F(CollectionSpecificMoreTest, UnorderedWeightingOfFields) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "brand", "type": "string"},
+ {"name": "sku", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "42f05db9-373a-4372-9bd0-ff4b5aaba28d";
+ doc["brand"] = "brand";
+ doc["sku"] = "rgx761";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // with num_typos
+
+ auto res_op = coll1->search("rg0761", {"title","brand","sku"}, "", {}, {}, {2,2,0}, 10, 1,
+ FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {10, 7, 10});
+
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(0, res_op.get()["hits"].size());
+
+ // with prefix
+ res_op = coll1->search("rgx", {"title","brand","sku"}, "", {}, {}, {2,2,0}, 10, 1,
+ FREQUENCY, {true, true, false},
+ 0, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {10, 7, 10});
+
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(0, res_op.get()["hits"].size());
+ }