Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

[Bug] String array field update + faceting.

Kishore Nallan908a8b0026a5
diff --git a/include/collection.h b/include/collection.h
index ebfacb57..065ba6f8 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -296,7 +296,7 @@ public:
static constexpr const char* SEQ_ID_PREFIX = "$SI";
static constexpr const char* DOC_ID_PREFIX = "$DI";
- void facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,
+ bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,
std::string &value);
void aggregate_topster(size_t query_index, Topster &topster, Topster *index_topster) const;
diff --git a/include/topster.h b/include/topster.h
index 2e585a6b..bad3ea30 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -130,8 +130,6 @@ struct Topster {
// erase current min heap key from kv_map
kv_map.erase(old_min_heap_kv.distinct_key);
- // kv will be copied into the pointer at heap_op_index
- kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);
} else {
// kv is guaranteed to be > current min heap: kvs[0]
// create fresh topster for this distinct group key since it does not exist
@@ -159,11 +157,11 @@ struct Topster {
group_kv_map.erase(kvs[heap_op_index]->distinct_key);
kv_map.erase(kvs[heap_op_index]->distinct_key);
}
-
- // kv will be copied into the pointer at heap_op_index
- kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);
}
+ // kv will be copied into the pointer at heap_op_index
+ kv_map.emplace(kv->distinct_key, kvs[heap_op_index]);
+
} else { // not distinct
//LOG(INFO) << "Searching for key: " << kv->key;
@@ -191,9 +189,6 @@ struct Topster {
// replace existing kv and sift down
heap_op_index = existing_kv->array_index;
kv_map.erase(kvs[heap_op_index]->key);
-
- // kv will be copied into the pointer at heap_op_index
- kv_map.emplace(kv->key, kvs[heap_op_index]);
} else { // not duplicate
if(size < MAX_SIZE) {
@@ -208,10 +203,10 @@ struct Topster {
heap_op_index = 0;
kv_map.erase(kvs[heap_op_index]->key);
}
-
- // kv will be copied into the pointer at heap_op_index
- kv_map.emplace(kv->key, kvs[heap_op_index]);
}
+
+ // kv will be copied into the pointer at heap_op_index
+ kv_map.emplace(kv->key, kvs[heap_op_index]);
}
// we have to replace the existing element in the heap and sift down
diff --git a/src/collection.cpp b/src/collection.cpp
index aaa51843..78ff3069 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -224,7 +224,10 @@ void Collection::get_doc_changes(const nlohmann::json &document, nlohmann::json
for(auto it = document.begin(); it != document.end(); ++it) {
new_doc[it.key()] = it.value();
- if(old_doc.count(it.key()) != 0) {
+
+ // if the update document contains a field that exists in old, we record that (for delete + reindex)
+ bool field_exists_in_old_doc = (old_doc.count(it.key()) != 0);
+ if(field_exists_in_old_doc) {
// key exists in the stored doc, so it must be reindexed
// we need to check for this because a field can be optional
del_doc[it.key()] = old_doc[it.key()];
@@ -1173,7 +1176,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
std::string value;
- facet_value_to_string(a_facet, facet_count, document, value);
+ bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);
+
+ if(!facet_found) {
+ continue;
+ }
std::vector<std::string> tokens;
StringUtils::split(value, tokens, " ");
@@ -1281,9 +1288,27 @@ void Collection::aggregate_topster(size_t query_index, Topster &topster, Topster
}
}
-void Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,
+bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,
const nlohmann::json &document, std::string &value) {
+ if(document.count(a_facet.field_name) == 0) {
+ LOG(ERROR) << "Could not find field " << a_facet.field_name << " in document during faceting.";
+ LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;
+ LOG(ERROR) << "Actual document: " << document;
+ return false;
+ }
+
+ if(facet_schema.at(a_facet.field_name).is_array()) {
+ size_t array_sz = document[a_facet.field_name].size();
+ if(facet_count.array_pos >= array_sz) {
+ LOG(ERROR) << "Facet field array size " << array_sz << " lesser than array pos " << facet_count.array_pos
+ << " for facet field " << a_facet.field_name;
+ LOG(ERROR) << "Facet field type: " << facet_schema.at(a_facet.field_name).type;
+ LOG(ERROR) << "Actual document: " << document;
+ return false;
+ }
+ }
+
if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {
value = document[a_facet.field_name];
} else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {
@@ -1315,6 +1340,8 @@ void Collection::facet_value_to_string(const facet &a_facet, const facet_count_t
value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());
value = (value == "1") ? "true" : "false";
}
+
+ return true;
}
void Collection::highlight_result(const field &search_field,
diff --git a/src/index.cpp b/src/index.cpp
index 7de60861..4efa5575 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -104,11 +104,12 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
// initialize facet index since it will be updated as well during search indexing
// even if a field is optional, a facet position will be available in the vector for that field
+ // NOTE: Use of `emplace()` means that we will not replace existing facet values.
std::vector<std::vector<uint64_t>> values(facet_schema.size());
facet_index_v2.emplace(seq_id, values);
// assumes that validation has already been done
- for(const std::pair<std::string, field> & field_pair: search_schema) {
+ for(const auto& field_pair: search_schema) {
const std::string & field_name = field_pair.first;
if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {
@@ -239,8 +240,8 @@ Option<uint32_t> Index::validate_index_in_memory(const nlohmann::json &document,
return Option<>(400, "Default sorting field `" + default_sorting_field + "` exceeds maximum value of a float.");
}
- for(const std::pair<std::string, field> & field_pair: search_schema) {
- const std::string & field_name = field_pair.first;
+ for(const auto& field_pair: search_schema) {
+ const std::string& field_name = field_pair.first;
if((field_pair.second.optional || is_update) && document.count(field_name) == 0) {
continue;
@@ -357,6 +358,7 @@ void Index::scrub_reindex_doc(nlohmann::json& update_doc, nlohmann::json& del_do
if(exact_match) {
it = del_doc.erase(it);
+ update_doc.erase(field_name);
} else {
++it;
}
@@ -836,7 +838,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
if(query_token_positions.find(qtoken_pos.pos) == query_token_positions.end() ||
query_token_positions[qtoken_pos.pos].cost >= qtoken_pos.cost ) {
token_pos_cost_t ftoken_pos_cost = {field_token_index, qtoken_pos.cost};
- query_token_positions.emplace(qtoken_pos.pos, ftoken_pos_cost);
+ query_token_positions[qtoken_pos.pos] = ftoken_pos_cost;
}
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index efdc33b8..4d9e6f97 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1330,7 +1330,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
std::vector<field> fields = {
field("title", field_types::STRING, false),
- field("starring", field_types::STRING, false),
+ field("starring", field_types::STRING, true),
field("cast", field_types::STRING_ARRAY, false),
field("points", field_types::INT32, false)
};
@@ -1365,17 +1365,17 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
ASSERT_EQ(1, import_results[i].size());
}
- auto results = coll_mul_fields->search("*", query_fields, "", {}, sort_fields, 0, 30, 1, FREQUENCY, false).get();
+ auto results = coll_mul_fields->search("*", query_fields, "", {"starring"}, sort_fields, 0, 30, 1, FREQUENCY, false).get();
ASSERT_EQ(19, results["hits"].size());
ASSERT_EQ(19, coll_mul_fields->get_num_documents());
- results = coll_mul_fields->search("back again forest", query_fields, "", {}, sort_fields, 0, 30, 1, FREQUENCY, false).get();
+ results = coll_mul_fields->search("back again forest", query_fields, "", {"starring"}, sort_fields, 0, 30, 1, FREQUENCY, false).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_STREQ("Back Again Forest", coll_mul_fields->get("18").get()["title"].get<std::string>().c_str());
- results = coll_mul_fields->search("fifth", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ results = coll_mul_fields->search("fifth", query_fields, "", {"starring"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("The <mark>Fifth</mark> Harry", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
@@ -1432,7 +1432,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
ASSERT_TRUE(import_response["success"].get<bool>());
ASSERT_EQ(1, import_response["num_imported"].get<int>());
- results = coll_mul_fields->search("Good Will Hunting", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ results = coll_mul_fields->search("Good Will Hunting", query_fields, "", {"starring"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(70, results["hits"][0]["document"]["points"].get<uint32_t>());
// updating a document that does not exist should fail, others should succeed
@@ -1449,7 +1449,7 @@ TEST_F(CollectionTest, ImportDocumentsUpsert) {
ASSERT_STREQ("Could not find a document with id: 20", import_results[0]["error"].get<std::string>().c_str());
ASSERT_EQ(404, import_results[0]["code"].get<size_t>());
- results = coll_mul_fields->search("wake up harry", query_fields, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ results = coll_mul_fields->search("wake up harry", query_fields, "", {"starring"}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(64, results["hits"][0]["document"]["points"].get<uint32_t>());
// trying to create documents with existing IDs should fail
@@ -2510,7 +2510,7 @@ TEST_F(CollectionTest, UpdateDocument) {
auto add_op = coll1->add(doc.dump());
ASSERT_TRUE(add_op.ok());
- auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ auto res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
@@ -2518,6 +2518,25 @@ TEST_F(CollectionTest, UpdateDocument) {
ASSERT_STREQ("The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.",
res["hits"][0]["document"]["title"].get<std::string>().c_str());
+ // reindex the document entirely again verbatim and try querying
+ add_op = coll1->add(doc.dump(), UPSERT);
+ ASSERT_TRUE(add_op.ok());
+
+ res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
+
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(1, res["facet_counts"].size());
+ ASSERT_STREQ("tags", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
+ ASSERT_EQ(2, res["facet_counts"][0]["counts"].size());
+
+ ASSERT_STREQ("NEWS", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][0]["count"]);
+
+ ASSERT_STREQ("LAZY", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][1]["count"]);
+
// try changing the title and searching for an older token
doc["title"] = "The quick brown fox.";
add_op = coll1->add(doc.dump(), UPSERT);
@@ -2525,13 +2544,13 @@ TEST_F(CollectionTest, UpdateDocument) {
ASSERT_EQ(1, coll1->get_num_documents());
- res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
ASSERT_EQ(0, res["hits"].size());
- res = coll1->search("quick", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ res = coll1->search("quick", {"title"}, "", {"title"}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
@@ -2551,7 +2570,7 @@ TEST_F(CollectionTest, UpdateDocument) {
ASSERT_TRUE(add_op.ok());
// check for old tag
- res = coll1->search("NEWS", {"tags"}, "", {}, sort_fields, 0, 10, 1,
+ res = coll1->search("NEWS", {"tags"}, "", {"tags"}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
@@ -2573,7 +2592,7 @@ TEST_F(CollectionTest, UpdateDocument) {
add_op = coll1->add(doc3.dump(), UPDATE);
ASSERT_TRUE(add_op.ok());
- res = coll1->search("*", {"tags"}, "points: > 90", {}, sort_fields, 0, 10, 1,
+ res = coll1->search("*", {"tags"}, "points: > 90", {"tags"}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
@@ -2587,7 +2606,7 @@ TEST_F(CollectionTest, UpdateDocument) {
add_op = coll1->add(doc4.dump(), UPSERT, "100");
ASSERT_TRUE(add_op.ok());
- res = coll1->search("*", {"tags"}, "points: > 101", {}, sort_fields, 0, 10, 1,
+ res = coll1->search("*", {"tags"}, "points: > 101", {"tags"}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
@@ -2599,7 +2618,7 @@ TEST_F(CollectionTest, UpdateDocument) {
add_op = coll1->add(doc4.dump(), UPSERT, "100");
ASSERT_FALSE(add_op.ok());
- res = coll1->search("*", {"tags"}, "points: > 101", {}, sort_fields, 0, 10, 1,
+ res = coll1->search("*", {"tags"}, "points: > 101", {"tags"}, sort_fields, 0, 10, 1,
token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
@@ -2629,6 +2648,66 @@ TEST_F(CollectionTest, UpdateDocument) {
ASSERT_FALSE(add_op.ok());
ASSERT_EQ(400, add_op.code());
ASSERT_STREQ("The `id` should not be empty.", add_op.error().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionTest, UpdateDocumentSorting) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("tags", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "100";
+ doc1["title"] = "The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.";
+ doc1["tags"] = {"NEWS", "LAZY"};
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "101";
+ doc2["title"] = "The random sentence.";
+ doc2["tags"] = {"RANDOM"};
+ doc2["points"] = 101;
+
+ auto add_op = coll1->add(doc1.dump());
+ coll1->add(doc2.dump());
+
+ auto res = coll1->search("*", {"tags"}, "", {"tags"}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
+
+ ASSERT_EQ(2, res["hits"].size());
+ ASSERT_EQ(101, res["hits"][0]["document"]["points"].get<size_t>());
+ ASSERT_STREQ("101", res["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_EQ(100, res["hits"][1]["document"]["points"].get<size_t>());
+ ASSERT_STREQ("100", res["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ // now update doc1 points from 100 -> 1000 and it should bubble up
+ doc1["points"] = 1000;
+ coll1->add(doc1.dump(), UPDATE);
+
+ res = coll1->search("*", {"tags"}, "", {"tags"}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
+
+ ASSERT_EQ(2, res["hits"].size());
+ ASSERT_EQ(1000, res["hits"][0]["document"]["points"].get<size_t>());
+ ASSERT_STREQ("100", res["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_EQ(101, res["hits"][1]["document"]["points"].get<size_t>());
+ ASSERT_STREQ("101", res["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
}
TEST_F(CollectionTest, SearchHighlightFieldFully) {